//! The WireGuard data plane, driven over real iroh connections. //! //! Everything here is real except the packet interface: real agents, real //! control plane, real iroh data links, real WireGuard handshakes and //! encryption from `boringtun`. Only the TUN device is in memory, which is why //! the whole data plane can be tested with no privileges and without touching //! the host's network. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::net::Ipv4Addr; use std::sync::Arc; use std::time::Duration; use bytes::Bytes; use iroh::EndpointId; use tempfile::TempDir; use tsunagi::agent::Event; use tsunagi::dataplane::IpPlugin; use tsunagi::discovery::SharedMemoryDiscovery; use tsunagi::identity::{NetworkId, NetworkName, NetworkSecret}; use tsunagi::overlay::{ MemoryTun, MemoryTunFactory, OverlayError, TunDevice, TunFactory, TunRequest, }; use tsunagi::state::Ipv4Range; use tsunagi::testing::{network, settle, wait_event, wait_for_peers, wait_until}; use tsunagi::{Agent, NetworkStatus}; use tsunagi_wg_quic::{ WIREGUARD_PROTOCOL, WgAnnouncement, WgSecretKey, WireguardConfig, WireguardPlugin, }; /// Real QUIC at its minimum path MTU: loopback PMTU discovery must not hide /// failures that occur on ordinary Internet paths. The TUN MTU stays at 1280. fn config_with(root: &std::path::Path, discovery: &SharedMemoryDiscovery) -> tsunagi::AgentConfig { let mut config = tsunagi::testing::config_with(root, discovery); config.test_quic_transport = Some( iroh::endpoint::QuicTransportConfig::builder() .initial_mtu(1200) .min_mtu(1200) .mtu_discovery_config(None) .build(), ); config } /// A factory that claims the host and puts nothing on it. /// /// This is the case the missing-address report exists for: a provisioner /// that returned success without achieving it, or something outside that /// removed the address afterwards. An in-memory interface is *not* that /// case — it has no host side at all, and treating the two as one is what /// had the agent reporting a fault about `--no-tun` working as intended, /// and configuring host interfaces it never created. #[derive(Debug, Clone)] struct PretendHostTuns(MemoryTunFactory); impl TunFactory for PretendHostTuns { fn name(&self) -> &str { "pretend-host" } fn on_host(&self) -> bool { true } fn create<'a>( &'a self, request: TunRequest, ) -> tsunagi::BoxFuture<'a, Result, OverlayError>> { self.0.create(request) } fn reconfigure<'a>( &'a self, request: TunRequest, ) -> tsunagi::BoxFuture<'a, Result<(), OverlayError>> { self.0.reconfigure(request) } fn destroy<'a>(&'a self, name: &'a str) -> tsunagi::BoxFuture<'a, ()> { self.0.destroy(name) } } /// An agent with a WireGuard plugin backed by an in-memory packet interface. struct WgAgent { dir: TempDir, agent: Agent, plugin: Arc, tuns: MemoryTunFactory, } impl WgAgent { async fn spawn(discovery: &SharedMemoryDiscovery, tag: &str) -> Self { Self::spawn_with(discovery, tag, |config| config).await } async fn spawn_with( discovery: &SharedMemoryDiscovery, tag: &str, tune: impl FnOnce(WireguardConfig) -> WireguardConfig, ) -> Self { Self::spawn_full( discovery, tag, tune, Some(tsunagi::state::DEFAULT_IPV4_RANGE), ) .await } /// Starts an agent with an explicit overlay range, or none at all. async fn spawn_range( discovery: &SharedMemoryDiscovery, tag: &str, range: Option, ) -> Self { Self::spawn_full(discovery, tag, |config| config, range).await } async fn spawn_full( discovery: &SharedMemoryDiscovery, tag: &str, tune: impl FnOnce(WireguardConfig) -> WireguardConfig, range: Option, ) -> Self { let dir = TempDir::new().unwrap(); let (agent, plugin, tuns) = Self::open_with(dir.path(), discovery, tag, tune, range).await; Self { dir, agent, plugin, tuns, } } async fn open( root: &std::path::Path, discovery: &SharedMemoryDiscovery, tag: &str, tune: impl FnOnce(WireguardConfig) -> WireguardConfig, ) -> (Agent, Arc, MemoryTunFactory) { Self::open_with( root, discovery, tag, tune, Some(tsunagi::state::DEFAULT_IPV4_RANGE), ) .await } async fn open_with( root: &std::path::Path, discovery: &SharedMemoryDiscovery, tag: &str, tune: impl FnOnce(WireguardConfig) -> WireguardConfig, range: Option, ) -> (Agent, Arc, MemoryTunFactory) { let tuns = MemoryTunFactory::new(); let wg = tune( WireguardConfig::new(root.join("wireguard")) .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), ); let plugin = WireguardPlugin::open(wg).await.unwrap(); // The interface belongs to the agent now, so the tag names it // directly rather than prefixing one per network. let agent = Agent::spawn( config_with(root, discovery) .with_overlay_ipv4_range(range) .with_interface(Arc::new(tuns.clone()), tag, 1280) .with_plugin(plugin.clone() as Arc), ) .await .unwrap(); (agent, plugin, tuns) } /// An agent that refuses a direct data path to some peers. /// /// The one arrangement a single host cannot produce by itself: two /// agents that both reach a third and not each other. The control /// plane is untouched — they are members and they talk — only the /// direct data link is refused, which is the real-world case of a /// blocked or unreachable data path. async fn spawn_cut_off( discovery: &SharedMemoryDiscovery, tag: &str, blocked: Arc>>, ) -> Self { let dir = TempDir::new().unwrap(); let tuns = MemoryTunFactory::new(); let plugin = WireguardPlugin::open( WireguardConfig::new(dir.path().join("wireguard")) .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), ) .await .unwrap(); let agent = Agent::spawn( config_with(dir.path(), discovery) .with_overlay_ipv4_range(Some(tsunagi::state::DEFAULT_IPV4_RANGE)) .with_interface(Arc::new(tuns.clone()), tag, 1280) .with_unreachable_data_peers(blocked) .with_plugin(plugin.clone() as Arc), ) .await .unwrap(); Self { dir, agent, plugin, tuns, } } /// An agent whose interface claims to be on the host but is not. /// /// Used only by the missing-address test: everywhere else the in-memory /// interface is honest about what it is. async fn spawn_pretending_host(discovery: &SharedMemoryDiscovery, tag: &str) -> Self { let dir = TempDir::new().unwrap(); let tuns = MemoryTunFactory::new(); let plugin = WireguardPlugin::open( WireguardConfig::new(dir.path().join("wireguard")) .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), ) .await .unwrap(); let agent = Agent::spawn( config_with(dir.path(), discovery) .with_overlay_ipv4_range(Some(tsunagi::state::DEFAULT_IPV4_RANGE)) .with_interface(Arc::new(PretendHostTuns(tuns.clone())), tag, 1280) .with_plugin(plugin.clone() as Arc), ) .await .unwrap(); Self { dir, agent, plugin, tuns, } } fn endpoint_id(&self) -> EndpointId { self.agent.endpoint_id() } /// This agent's overlay address in a network. /// /// Allocated and signed at the system level, so it appears once the /// network has agreed on it rather than the moment the plugin starts. async fn overlay(&self, network: NetworkId) -> Ipv4Addr { wait_until("the network agreed an overlay address", || async { self.plugin.overview(network)?.overlay_address_v4 }) .await } /// The one in-memory packet interface this agent owns. /// /// Not per network: one agent has one interface, and which network a /// packet on it belongs to is decided by its address. async fn tun(&self, _network: NetworkId) -> Arc { let name = wait_until("the packet interface exists", || async { self.agent.overlay().map(|overlay| overlay.interface) }) .await; self.tuns.device(&name).expect("the device was created") } /// Waits until `count` tunnels have completed a WireGuard handshake. async fn wait_for_tunnels(&self, network: NetworkId, count: usize) { wait_until( &format!("{count} established WireGuard tunnels"), || async { let view = self.plugin.overview(network)?; (view.established_peers() == count).then_some(()) }, ) .await; } async fn shutdown(self) -> TempDir { self.agent.shutdown().await; self.dir } } /// Builds a minimal well-formed IPv4 packet. fn ipv4_packet(source: Ipv4Addr, destination: Ipv4Addr, payload: &[u8]) -> Bytes { let total = 20 + payload.len(); let mut packet = Vec::with_capacity(total); packet.push((4 << 4) | 5); // version 4, header length 5 words packet.push(0); // dscp/ecn packet.extend_from_slice(&(total as u16).to_be_bytes()); packet.extend_from_slice(&[0, 0]); // identification packet.extend_from_slice(&[0, 0]); // flags and fragment offset packet.push(64); // ttl packet.push(253); // an experimental protocol number packet.extend_from_slice(&[0, 0]); // checksum, not verified by the overlay packet.extend_from_slice(&source.octets()); packet.extend_from_slice(&destination.octets()); packet.extend_from_slice(payload); Bytes::from(packet) } #[tokio::test] async fn full_size_tcp_packets_cross_the_default_overlay() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-default-mtu"); let a = WgAgent::spawn(&discovery, "tmtua").await; let b = WgAgent::spawn(&discovery, "tmtub").await; let id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(id, 1).await; b.wait_for_tunnels(id, 1).await; let addr_a = a.overlay(id).await; let addr_b = b.overlay(id).await; let tun_a = a.tun(id).await; let tun_b = b.tun(id).await; assert_eq!(tun_a.mtu(), tsunagi_wg_quic::DEFAULT_MTU); for size in [1280, 1279, 1098, 1097, 64] { for (source, dest, from, to) in [ (addr_a, addr_b, &tun_a, &tun_b), (addr_b, addr_a, &tun_b, &tun_a), ] { let packet = tcp_packet(source, dest, size); from.push_from_os(packet.clone()); let received = tokio::time::timeout(Duration::from_secs(5), to.pop_to_os()) .await .expect("a full-size TCP packet must arrive at the default MTU") .unwrap(); assert_eq!(received, packet, "including DF, TCP flags and checksums"); } } for peer in [&a, &b] { let view = peer.plugin.overview(id).unwrap(); assert_eq!( view.peers[0] .tunnel .as_ref() .unwrap() .stats .dropped_oversize, 0 ); } a.shutdown().await; b.shutdown().await; } /// A checksummed IPv4/TCP segment with DF set, as a host's TCP stack sends it. fn tcp_packet(source: Ipv4Addr, destination: Ipv4Addr, size: usize) -> Bytes { fn checksum(bytes: &[u8]) -> u16 { let mut sum: u32 = bytes .chunks(2) .map(|c| u16::from_be_bytes([c[0], *c.get(1).unwrap_or(&0)]) as u32) .sum(); while sum > 0xffff { sum = (sum & 0xffff) + (sum >> 16); } !(sum as u16) } assert!(size >= 40); let mut packet = ipv4_packet(source, destination, &vec![0x5a; size - 20]).to_vec(); packet[6..8].copy_from_slice(&0x4000u16.to_be_bytes()); // don't fragment packet[9] = 6; // TCP packet[20..40].fill(0); packet[20..22].copy_from_slice(&40000u16.to_be_bytes()); packet[22..24].copy_from_slice(&22u16.to_be_bytes()); packet[24..28].copy_from_slice(&1u32.to_be_bytes()); packet[32] = 5 << 4; packet[33] = 0x18; // PSH, ACK packet[34..36].copy_from_slice(&65535u16.to_be_bytes()); let mut pseudo = Vec::from(&packet[12..20]); pseudo.extend_from_slice(&[0, 6]); pseudo.extend_from_slice(&((size - 20) as u16).to_be_bytes()); pseudo.extend_from_slice(&packet[20..]); packet[36..38].copy_from_slice(&checksum(&pseudo).to_be_bytes()); let ip_checksum = checksum(&packet[..20]); packet[10..12].copy_from_slice(&ip_checksum.to_be_bytes()); Bytes::from(packet) } #[tokio::test] async fn two_agents_carry_real_ip_packets_through_a_wireguard_tunnel() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-traffic"); let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); wait_for_peers(&a.agent, network_id, 1).await; // Both tunnels must actually handshake, not merely be configured. a.wait_for_tunnels(network_id, 1).await; b.wait_for_tunnels(network_id, 1).await; let addr_a = a.overlay(network_id).await; let addr_b = b.overlay(network_id).await; assert_ne!(addr_a, addr_b); // One shared range, agreed at the system level and signed by each // member, not derived from anything either protocol holds. let range = a.plugin.overview(network_id).unwrap().ipv4_range.unwrap(); assert!(range.contains(addr_a) && range.contains(addr_b)); let tun_a = a.tun(network_id).await; let tun_b = b.tun(network_id).await; // A real IP packet, encrypted by WireGuard, carried over iroh, decrypted // on the other side and handed to that host's packet interface. let payload = b"hello over the overlay"; tun_a.push_from_os(ipv4_packet(addr_a, addr_b, payload)); let received = tokio::time::timeout(tsunagi::testing::DEADLINE, tun_b.pop_to_os()) .await .expect("the packet should arrive") .expect("the interface should still be open"); assert_eq!(&received[20..], payload); assert_eq!(&received[12..16], &addr_a.octets(), "source preserved"); assert_eq!(&received[16..20], &addr_b.octets(), "destination preserved"); // And back the other way. tun_b.push_from_os(ipv4_packet(addr_b, addr_a, b"and back")); let back = tokio::time::timeout(tsunagi::testing::DEADLINE, tun_a.pop_to_os()) .await .expect("the reply should arrive") .unwrap(); assert_eq!(&back[20..], b"and back"); let view = a.plugin.overview(network_id).unwrap(); let tunnel = view.peers[0].tunnel.as_ref().unwrap(); assert!(tunnel.health.is_up()); assert!(tunnel.stats.tx_packets >= 1); assert!(tunnel.stats.rx_packets >= 1); assert_eq!(tunnel.stats.dropped_wrong_source, 0); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn a_peer_cannot_send_from_an_address_it_does_not_own() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-spoof"); let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; b.wait_for_tunnels(network_id, 1).await; let addr_a = a.overlay(network_id).await; let addr_b = b.overlay(network_id).await; let tun_a = a.tun(network_id).await; let tun_b = b.tun(network_id).await; // A sends a packet claiming to come from a third party's address. let someone_else = { let mut octets = addr_a.octets(); octets[3] ^= 0xff; Ipv4Addr::from(octets) }; tun_a.push_from_os(ipv4_packet(someone_else, addr_b, b"spoofed")); // B must drop it: the source is not the address A holds. // Counted on the interface, not on the tunnel: the protocol proved who // sent the packet, and whether that member may use the address it chose // is a question about a signed claim, which the system level holds. wait_until("the spoofed packet is dropped", || async { (b.agent.overlay()?.counters.wrong_source >= 1).then_some(()) }) .await; // A legitimate packet still goes through, so the tunnel is not broken. tun_a.push_from_os(ipv4_packet(addr_a, addr_b, b"honest")); let received = tokio::time::timeout(tsunagi::testing::DEADLINE, tun_b.pop_to_os()) .await .expect("the honest packet should arrive") .unwrap(); assert_eq!(&received[20..], b"honest"); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn the_overlay_uses_a_range_the_network_was_told_to_use() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-dual-stack"); let range = Some("10.77.0.0/16".parse::().unwrap()); let a = WgAgent::spawn_range(&discovery, "ta", range).await; let b = WgAgent::spawn_range(&discovery, "tb", range).await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; b.wait_for_tunnels(network_id, 1).await; let view_a = a.plugin.overview(network_id).unwrap(); let view_b = b.plugin.overview(network_id).unwrap(); let v4_a = view_a.overlay_address_v4.expect("ipv4 was configured"); let v4_b = view_b.overlay_address_v4.expect("ipv4 was configured"); assert_ne!(v4_a, v4_b); // Both inside the configured range. for addr in [v4_a, v4_b] { assert_eq!( u32::from(addr) & 0xffff_0000, u32::from(Ipv4Addr::new(10, 77, 0, 0)) ); } // Each side learned the other's address from the same signed state. assert_eq!(view_a.peers[0].overlay_address_v4, Some(v4_b)); assert_eq!(view_b.peers[0].overlay_address_v4, Some(v4_a)); let tun_a = a.tun(network_id).await; let tun_b = b.tun(network_id).await; // A real IPv4 packet through the same tunnel. tun_a.push_from_os(ipv4_packet(v4_a, v4_b, b"ipv4 over the overlay")); let received = tokio::time::timeout(tsunagi::testing::DEADLINE, tun_b.pop_to_os()) .await .expect("the IPv4 packet should arrive") .unwrap(); assert_eq!(received[0] >> 4, 4, "still an IPv4 packet"); assert_eq!(&received[12..16], &v4_a.octets()); assert_eq!(&received[16..20], &v4_b.octets()); assert_eq!(&received[20..], b"ipv4 over the overlay"); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn an_ipv4_source_a_peer_does_not_own_is_dropped() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-v4-spoof"); let range = Some("10.78.0.0/16".parse::().unwrap()); let a = WgAgent::spawn_range(&discovery, "ta", range).await; let b = WgAgent::spawn_range(&discovery, "tb", range).await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; b.wait_for_tunnels(network_id, 1).await; let v4_a = a .plugin .overview(network_id) .unwrap() .overlay_address_v4 .unwrap(); let v4_b = b .plugin .overview(network_id) .unwrap() .overlay_address_v4 .unwrap(); let tun_a = a.tun(network_id).await; let tun_b = b.tun(network_id).await; // A claims an IPv4 address that is not the one derived from its key. let forged = Ipv4Addr::from(u32::from(v4_a) ^ 0x0000_00ff); tun_a.push_from_os(ipv4_packet(forged, v4_b, b"spoofed v4")); wait_until("the spoofed IPv4 packet is dropped", || async { (b.agent.overlay()?.counters.wrong_source >= 1).then_some(()) }) .await; // The honest one still gets through. tun_a.push_from_os(ipv4_packet(v4_a, v4_b, b"honest v4")); let received = tokio::time::timeout(tsunagi::testing::DEADLINE, tun_b.pop_to_os()) .await .expect("the honest packet should arrive") .unwrap(); assert_eq!(&received[20..], b"honest v4"); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn a_joining_member_adopts_the_range_the_network_already_uses() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-range-adopted"); // The two were started with different ranges. Rather than misroute, they // converge on one, and every replica computes the same answer. let a = WgAgent::spawn_range(&discovery, "ta", Some("10.80.0.0/16".parse().unwrap())).await; let b = WgAgent::spawn_range(&discovery, "tb", Some("10.81.0.0/16".parse().unwrap())).await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; b.wait_for_tunnels(network_id, 1).await; let agreed = wait_until("both settle on one range", || async { let one = a.plugin.overview(network_id)?.ipv4_range?; let two = b.plugin.overview(network_id)?.ipv4_range?; (one == two).then_some(one) }) .await; // Whichever won, both hold an address inside it, and they differ. let view_a = wait_until("a has an address in the agreed range", || async { let view = a.plugin.overview(network_id)?; let address = view.overlay_address_v4?; agreed.contains(address).then_some(view) }) .await; let view_b = wait_until("b has an address in the agreed range", || async { let view = b.plugin.overview(network_id)?; let address = view.overlay_address_v4?; agreed.contains(address).then_some(view) }) .await; assert_ne!(view_a.overlay_address_v4, view_b.overlay_address_v4); // And each sees the other at the same address it sees for itself. let seen_b = wait_until("a sees b's address", || async { a.plugin .overview(network_id)? .peers .first()? .overlay_address_v4 }) .await; assert_eq!(Some(seen_b), view_b.overlay_address_v4); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn an_allocated_address_missing_from_the_host_is_reported() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-missing-address"); // An interface that says it is on the host and never carries the // address: a provisioner that reported a success it did not achieve, or // something outside that took the address away. Left unsaid, packets // leave with the wrong source and every peer drops them, which looks // like a broken network rather than a missing command. let a = WgAgent::spawn_pretending_host(&discovery, "ta").await; let b = WgAgent::spawn_pretending_host(&discovery, "tb").await; let mut events = a.agent.subscribe(); let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; let reason = wait_event(&mut events, |event| match event { Event::PluginError { reason, .. } if reason.contains("not on any") => Some(reason.clone()), _ => None, }) .await; let allocated = a .plugin .overview(network_id) .unwrap() .overlay_address_v4 .unwrap(); assert!( reason.contains(&allocated.to_string()), "unexpected: {reason}" ); // The agent owns the interface, so it is the agent that notices and the // report names the interface the address should have been on. assert!( reason.contains(&a.agent.overlay().unwrap().interface), "must name the interface: {reason}" ); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn an_interface_that_is_only_in_memory_is_not_reported_as_a_fault() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-memory-interface"); // `--no-tun` is an arrangement, not a failure: the tunnels run and the // packets move between agents, and nothing was ever going to put an // address on a host interface that does not exist. Complaining about it // sent people looking for something that had removed their address. let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let mut events = a.agent.subscribe(); let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; // An address was allocated, so the report had every other reason to fire. let address = a.overlay(network_id).await; assert!(tsunagi::state::DEFAULT_IPV4_RANGE.contains(address)); settle().await; let complaint = std::iter::from_fn(|| events.try_recv().ok()).find( |event| matches!(event, Event::PluginError { reason, .. } if reason.contains("not on any")), ); assert!(complaint.is_none(), "unexpected: {complaint:?}"); // And the interface says plainly what it is, which is what keeps the // resolver setting off a host interface of the same name. assert!(!a.agent.overlay().unwrap().on_host); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn leaving_a_network_takes_its_protocol_key_with_it() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-leave"); let agent = WgAgent::spawn(&discovery, "ta").await; let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); let before = wait_until("the plugin has a key for it", || async { Some(agent.plugin.overview(network_id)?.public_key) }) .await; agent.agent.leave_network(network_id).await.unwrap(); // Rejoining is joining, not resuming: the key was this agent's identity // in a network it left, and everyone there was told to let the address // it held go. Coming back with the same key would claim an identity the // network has already released. agent.agent.join_network(&name, &secret).await.unwrap(); let after = wait_until("the plugin has a key again", || async { Some(agent.plugin.overview(network_id)?.public_key) }) .await; assert_ne!(before, after, "a fresh key, not the released one"); agent.shutdown().await; } #[tokio::test] async fn an_address_is_kept_across_a_restart() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-address-persists"); let peer = WgAgent::spawn(&discovery, "tp").await; let subject = WgAgent::spawn(&discovery, "ts").await; let network_id = peer.agent.join_network(&name, &secret).await.unwrap(); subject.agent.join_network(&name, &secret).await.unwrap(); peer.wait_for_tunnels(network_id, 1).await; let before = wait_until("the subject has an address", || async { subject.plugin.overview(network_id)?.overlay_address_v4 }) .await; // The peer agrees about it. let seen_before = wait_until("the peer sees it", || async { peer.plugin .overview(network_id)? .peers .first()? .overlay_address_v4 }) .await; assert_eq!(seen_before, before); // Go away, come back. The address is a signed record, not a derivation // and not a session fact, so it survives. let dir = subject.shutdown().await; let (agent, plugin, _tuns) = WgAgent::open(dir.path(), &discovery, "ts", |config| config).await; let after = wait_until("the restarted agent has an address", || async { plugin.overview(network_id)?.overlay_address_v4 }) .await; assert_eq!( after, before, "a returning participant must reclaim the address it signed for" ); // And the peer still agrees, without having had to do anything. let seen_after = wait_until("the peer still agrees", || async { let seen = peer .plugin .overview(network_id)? .peers .first()? .overlay_address_v4?; (seen == after).then_some(seen) }) .await; assert_eq!(seen_after, after); agent.shutdown().await; peer.agent.shutdown().await; drop(agent); drop(dir); } #[tokio::test] async fn three_members_get_three_different_addresses() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-allocation"); let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let c = WgAgent::spawn(&discovery, "tc").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); c.agent.join_network(&name, &secret).await.unwrap(); for agent in [&a, &b, &c] { agent.wait_for_tunnels(network_id, 2).await; } // Everybody ends up knowing all three addresses, and they are distinct. let addresses = wait_until("everyone agrees on three addresses", || async { let mut all = std::collections::BTreeSet::new(); for agent in [&a, &b, &c] { let view = agent.plugin.overview(network_id)?; all.insert(view.overlay_address_v4?); for peer in &view.peers { all.insert(peer.overlay_address_v4?); } } (all.len() == 3).then_some(all) }) .await; let default_range = tsunagi::state::DEFAULT_IPV4_RANGE; for address in &addresses { assert!( default_range.contains(*address), "{address} outside the range" ); assert_ne!(address.octets()[3], 0); assert_ne!(address.octets()[3], 255); } a.shutdown().await; b.shutdown().await; c.shutdown().await; } #[tokio::test] async fn packets_for_an_unknown_address_are_counted_not_broadcast() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-unroutable"); let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); a.wait_for_tunnels(network_id, 1).await; let addr_a = a.overlay(network_id).await; let tun_a = a.tun(network_id).await; let tun_b = b.tun(network_id).await; // Nobody owns this address, so it must not be sent to anybody. let nowhere: Ipv4Addr = "192.0.2.111".parse().unwrap(); tun_a.push_from_os(ipv4_packet(addr_a, nowhere, b"lost")); wait_until("the packet is counted as unroutable", || async { let view = a.plugin.overview(network_id)?; let _ = view; (a.agent.overlay()?.counters.unroutable >= 1).then_some(()) }) .await; settle().await; assert!( tokio::time::timeout(Duration::from_millis(200), tun_b.pop_to_os()) .await .is_err(), "an unroutable packet must not reach another member" ); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn a_mesh_of_three_establishes_every_tunnel() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-mesh"); let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let c = WgAgent::spawn(&discovery, "tc").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); c.agent.join_network(&name, &secret).await.unwrap(); for agent in [&a, &b, &c] { wait_for_peers(&agent.agent, network_id, 2).await; // N - 1 tunnels, all handshaken. agent.wait_for_tunnels(network_id, 2).await; } // Everyone agrees on the subnet and nobody configured themselves. let mut addresses = Vec::new(); for agent in [&a, &b, &c] { let view = agent.plugin.overview(network_id).unwrap(); assert!( view.peers .iter() .all(|peer| peer.public_key != view.public_key) ); addresses.push(agent.overlay(network_id).await); } addresses.sort(); addresses.dedup(); assert_eq!(addresses.len(), 3, "every member has its own address"); // A packet from A reaches C directly, not via B. let addr_a = a.overlay(network_id).await; let addr_c = c.overlay(network_id).await; a.tun(network_id) .await .push_from_os(ipv4_packet(addr_a, addr_c, b"a to c")); let received = tokio::time::timeout( tsunagi::testing::DEADLINE, c.tun(network_id).await.pop_to_os(), ) .await .expect("the packet should arrive") .unwrap(); assert_eq!(&received[20..], b"a to c"); a.shutdown().await; b.shutdown().await; c.shutdown().await; } #[tokio::test] async fn a_departing_peer_loses_its_tunnel() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-departure"); let stayer = WgAgent::spawn(&discovery, "ta").await; let leaver = WgAgent::spawn(&discovery, "tb").await; let network_id = stayer.agent.join_network(&name, &secret).await.unwrap(); leaver.agent.join_network(&name, &secret).await.unwrap(); stayer.wait_for_tunnels(network_id, 1).await; let leaver_id = leaver.endpoint_id(); let mut events = stayer.agent.subscribe(); leaver.shutdown().await; wait_event(&mut events, |event| match event { Event::PeerDisconnected { peer, .. } if *peer == leaver_id => Some(()), _ => None, }) .await; wait_until("the tunnel is removed", || async { let view = stayer.plugin.overview(network_id)?; view.peers.is_empty().then_some(()) }) .await; // The interface itself stays; only the peer went. assert!(stayer.plugin.overview(network_id).is_some()); stayer.shutdown().await; } #[tokio::test] async fn two_networks_share_one_interface_with_keys_and_ranges_of_their_own() { // One agent, one interface — so two networks on it must use different // ranges, or an address would belong to both. Each network's range is // settled by whoever got there first, and the agent adopts what it // finds; see the routing table's own tests for the refusal when they // overlap. let discovery = SharedMemoryDiscovery::new(); let (name_a, secret_a) = network("wg-left"); let (name_b, secret_b) = network("wg-right"); let beta_range = Some("10.99.0.0/16".parse::().unwrap()); let hub = WgAgent::spawn(&discovery, "th").await; let left = WgAgent::spawn(&discovery, "tl").await; let right = WgAgent::spawn_range(&discovery, "tr", beta_range).await; // The other members settle each range before the hub joins, so it has // something to adopt rather than a default to collide with. let alpha = left.agent.join_network(&name_a, &secret_a).await.unwrap(); let beta = right.agent.join_network(&name_b, &secret_b).await.unwrap(); left.overlay(alpha).await; right.overlay(beta).await; hub.agent.join_network(&name_a, &secret_a).await.unwrap(); hub.agent.join_network(&name_b, &secret_b).await.unwrap(); hub.wait_for_tunnels(alpha, 1).await; hub.wait_for_tunnels(beta, 1).await; assert_ne!( hub.plugin.overview(alpha).unwrap().public_key, hub.plugin.overview(beta).unwrap().public_key, "one WireGuard identity per network, not one per host" ); assert_eq!( hub.tuns.devices().len(), 1, "one agent has one interface, whatever it is a member of" ); // Each network's address comes from its own range. let hub_alpha = wait_until("the hub settles into alpha's range", || async { let address = hub.plugin.overview(alpha)?.overlay_address_v4?; tsunagi::state::DEFAULT_IPV4_RANGE .contains(address) .then_some(address) }) .await; let hub_beta = wait_until("the hub settles into beta's range", || async { let address = hub.plugin.overview(beta)?.overlay_address_v4?; beta_range?.contains(address).then_some(address) }) .await; assert_ne!(hub_alpha, hub_beta); // Traffic in one overlay never surfaces in the other, though both cross // the same interface. let left_addr = left.overlay(alpha).await; hub.tun(alpha) .await .push_from_os(ipv4_packet(hub_alpha, left_addr, b"alpha only")); let seen = tokio::time::timeout( tsunagi::testing::DEADLINE, left.tun(alpha).await.pop_to_os(), ) .await .expect("the packet should arrive") .unwrap(); assert_eq!(&seen[20..], b"alpha only"); assert!( tokio::time::timeout( Duration::from_millis(200), right.tun(beta).await.pop_to_os() ) .await .is_err(), "the other overlay must see nothing" ); hub.shutdown().await; left.shutdown().await; right.shutdown().await; } #[tokio::test] async fn restarting_keeps_the_wireguard_identity_and_overlay_address() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-restart"); let peer = WgAgent::spawn(&discovery, "tp").await; let subject = WgAgent::spawn(&discovery, "ts").await; let network_id = peer.agent.join_network(&name, &secret).await.unwrap(); subject.agent.join_network(&name, &secret).await.unwrap(); peer.wait_for_tunnels(network_id, 1).await; let before = subject.plugin.overview(network_id).unwrap(); let dir = subject.shutdown().await; let (agent, plugin, _tuns) = WgAgent::open(dir.path(), &discovery, "ts", |config| config).await; let after = wait_until("the restarted plugin is ready", || async { plugin.overview(network_id) }) .await; assert_eq!(after.public_key, before.public_key); // The tunnel comes back on its own. wait_until("the tunnel is re-established", || async { let view = plugin.overview(network_id)?; (view.established_peers() == 1).then_some(()) }) .await; agent.shutdown().await; peer.shutdown().await; drop(agent); drop(dir); } #[tokio::test] async fn shutdown_removes_every_interface_the_plugin_created() { let discovery = SharedMemoryDiscovery::new(); let (name_a, secret_a) = network("wg-teardown-a"); let (name_b, secret_b) = network("wg-teardown-b"); let agent = WgAgent::spawn(&discovery, "ta").await; let alpha = agent.agent.join_network(&name_a, &secret_a).await.unwrap(); let beta = agent.agent.join_network(&name_b, &secret_b).await.unwrap(); agent.tun(alpha).await; agent.tun(beta).await; agent.agent.shutdown().await; assert!(agent.plugin.overview(alpha).is_none()); assert!(agent.plugin.overview(beta).is_none()); } #[tokio::test] async fn the_core_carries_the_payload_without_interpreting_it() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-opaque"); let a = WgAgent::spawn(&discovery, "ta").await; let b = WgAgent::spawn(&discovery, "tb").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); wait_for_peers(&a.agent, network_id, 1).await; let capability = wait_until("the peer's capability arrived", || async { let status: NetworkStatus = a.agent.network_status(network_id).await.ok()?; status .peers .first() .and_then(|peer| peer.capabilities.first().cloned()) }) .await; assert_eq!(capability.protocol, WIREGUARD_PROTOCOL); let view_b = b.plugin.overview(network_id).unwrap(); let expected = WgAnnouncement::new(network_id, &view_b.public_key) .encode() .unwrap(); assert_eq!(capability.data, expected); assert!(capability.data.len() < tsunagi::Limits::default().max_capability_data_len); a.shutdown().await; b.shutdown().await; } #[tokio::test] async fn an_announcement_for_another_network_never_reaches_a_tunnel() { let discovery = SharedMemoryDiscovery::new(); let name = NetworkName::new("wg-hijack").unwrap(); let secret = NetworkSecret::generate(); let victim = WgAgent::spawn(&discovery, "tv").await; let network_id = victim.agent.join_network(&name, &secret).await.unwrap(); // Addresses are not in an announcement any more, so there is no address // to forge. What is left to lie about is which network the key is for — // and a capability is scoped to the session it arrived on, so claiming // another network's is the shape a confused or hostile member takes. let attacker_key = WgSecretKey::generate().public(); let elsewhere = tsunagi::identity::NetworkKeys::derive( &NetworkName::new("somewhere-else").unwrap(), &NetworkSecret::generate(), ) .network_id(); let forged = WgAnnouncement::new(elsewhere, &attacker_key); let forger = Arc::new(ForgingPlugin { payload: std::sync::Mutex::new(Some(forged.encode().unwrap())), }); let attacker_dir = TempDir::new().unwrap(); let attacker = Agent::spawn( config_with(attacker_dir.path(), &discovery) .with_plugin(forger.clone() as Arc), ) .await .unwrap(); let mut events = victim.agent.subscribe(); attacker.join_network(&name, &secret).await.unwrap(); wait_for_peers(&victim.agent, network_id, 1).await; // Filtered on the reason: this agent reports other things too, and the // first plugin error to arrive is not necessarily this one. let reason = wait_event(&mut events, |event| match event { Event::PluginError { protocol, reason, .. } if protocol == WIREGUARD_PROTOCOL && reason.contains("different network") => { Some(reason.clone()) } _ => None, }) .await; assert!(reason.contains("different network"), "unexpected: {reason}"); settle().await; let view = victim.plugin.overview(network_id).unwrap(); assert!( view.peers .iter() .all(|peer| peer.public_key != attacker_key), "a rejected announcement must never become a tunnel" ); attacker.shutdown().await; victim.shutdown().await; drop(attacker_dir); } #[tokio::test] async fn a_peer_at_another_protocol_version_gets_no_data_plane_and_keeps_the_control_plane() { // Both sides must have the protocol at the same version. There is no // middle ground to negotiate: either the words mean the same thing at // both ends or they do not. let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-version"); let here = WgAgent::spawn(&discovery, "tvh").await; let network_id = here.agent.join_network(&name, &secret).await.unwrap(); let ahead = Arc::new(FromTheFuture); let dir = TempDir::new().unwrap(); let other = Agent::spawn( config_with(dir.path(), &discovery).with_plugin(ahead.clone() as Arc), ) .await .unwrap(); let mut events = here.agent.subscribe(); other.join_network(&name, &secret).await.unwrap(); wait_for_peers(&here.agent, network_id, 1).await; let reason = wait_event(&mut events, |event| match event { Event::PluginError { reason, .. } if reason.contains("version") => Some(reason.clone()), _ => None, }) .await; assert!( reason.contains("control plane is unaffected"), "the message should say what still works: {reason}" ); // No tunnel, and no attempt at one. settle().await; assert!( here.plugin.overview(network_id).unwrap().peers.is_empty(), "a version that cannot match must not become a tunnel" ); let status = here.agent.network_status(network_id).await.unwrap(); assert_eq!(status.peers.len(), 1, "the session is up all the same"); assert!( status.peers[0].protocols.is_empty(), "nothing was agreed with it: {:?}", status.peers[0].protocols ); // And the control plane carries a message to it regardless. assert_eq!( here.agent .broadcast( network_id, tsunagi::proto::ControlMessage::Ping { seq: 1, payload: b"still talking".to_vec(), }, ) .await .unwrap(), 1 ); other.shutdown().await; here.shutdown().await; drop(dir); } #[tokio::test] async fn agreement_turns_on_the_protocol_version_and_nothing_else() { // A different build is not a different protocol. What is compared is the // wire version and the name; everything else about a peer — its release, // and whatever opaque payload its announcement carries — has no say. let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-same-wire"); let here = WgAgent::spawn(&discovery, "tsw").await; let network_id = here.agent.join_network(&name, &secret).await.unwrap(); // Announces the right protocol at the right version, with a payload // this build has never seen. let stranger = Arc::new(ForgingPlugin { payload: std::sync::Mutex::new(Some(b"from some other release".to_vec())), }); let dir = TempDir::new().unwrap(); let other = Agent::spawn( config_with(dir.path(), &discovery).with_plugin(stranger.clone() as Arc), ) .await .unwrap(); other.join_network(&name, &secret).await.unwrap(); wait_for_peers(&here.agent, network_id, 1).await; // Agreed, because the wire matches. The payload is rejected by the // protocol afterwards, which is a separate matter and says nothing about // whether the two agreed to talk. wait_until("the protocol is agreed with it", || async { let status = here.agent.network_status(network_id).await.ok()?; status .peers .first()? .protocols .contains(&WIREGUARD_PROTOCOL.to_string()) .then_some(()) }) .await; other.shutdown().await; here.shutdown().await; drop(dir); } /// A plugin claiming the same protocol at a version this build does not speak. #[derive(Debug)] struct FromTheFuture; impl IpPlugin for FromTheFuture { fn protocol_id(&self) -> &str { WIREGUARD_PROTOCOL } fn protocol_version(&self) -> u16 { tsunagi_wg_quic::ANNOUNCEMENT_VERSION + 1 } fn local_capability( &self, _network: NetworkId, ) -> Result, tsunagi::dataplane::PluginError> { Ok(Some(tsunagi::dataplane::PluginCapability { protocol: WIREGUARD_PROTOCOL.to_string(), version: self.protocol_version(), enabled: true, data: Vec::new(), })) } fn on_peer_capability( &self, _network: NetworkId, _peer: EndpointId, _capability: &tsunagi::dataplane::PluginCapability, ) -> Result<(), tsunagi::dataplane::PluginError> { Ok(()) } fn on_peer_gone(&self, _network: NetworkId, _peer: EndpointId) {} fn on_network_deactivated(&self, _network: NetworkId) {} } /// A plugin that announces whatever bytes it is told to, under the WireGuard /// protocol id. Used to test what a hostile member can do. #[derive(Debug)] struct ForgingPlugin { payload: std::sync::Mutex>>, } impl IpPlugin for ForgingPlugin { fn protocol_id(&self) -> &str { WIREGUARD_PROTOCOL } /// The same version, so the announcement is looked at rather than /// dismissed for the wrong reason. fn protocol_version(&self) -> u16 { tsunagi_wg_quic::ANNOUNCEMENT_VERSION } fn local_capability( &self, _network: NetworkId, ) -> Result, tsunagi::dataplane::PluginError> { let payload = match self.payload.lock() { Ok(guard) => guard.clone(), Err(poisoned) => poisoned.into_inner().clone(), }; Ok(payload.map(|data| tsunagi::dataplane::PluginCapability { protocol: WIREGUARD_PROTOCOL.to_string(), // The version it actually speaks, so the payload is examined // rather than set aside for the wrong reason. version: tsunagi_wg_quic::ANNOUNCEMENT_VERSION, enabled: true, data, })) } fn on_peer_capability( &self, _network: NetworkId, _peer: EndpointId, _capability: &tsunagi::dataplane::PluginCapability, ) -> Result<(), tsunagi::dataplane::PluginError> { Ok(()) } fn on_peer_gone(&self, _network: NetworkId, _peer: EndpointId) {} fn on_network_deactivated(&self, _network: NetworkId) {} } #[tokio::test] async fn an_mtu_below_what_ipv4_guarantees_is_refused() { use tsunagi_wg_quic::{DEFAULT_MTU, MIN_MTU, WIREGUARD_OVERHEAD}; // 576 bytes is what every IPv4 host must be able to reassemble, so // nothing below it is worth offering. The floor used to be 1280 for // IPv6's sake; the overlay is IPv4 now and a relayed path with small // datagrams can be matched instead of warned about. let dir = TempDir::new().unwrap(); let result = WireguardPlugin::open(WireguardConfig::new(dir.path()).with_mtu(MIN_MTU - 1)).await; match result { Err(err) => { let text = err.to_string(); assert!(text.contains("576"), "unexpected message: {text}"); } Ok(_) => panic!("an MTU below what IPv4 guarantees must be refused"), } // The default leaves room below it now, which is the point of lowering // the floor, and a link has to carry it plus WireGuard's own overhead. const { assert!(DEFAULT_MTU > MIN_MTU) }; assert_eq!(WIREGUARD_OVERHEAD, 32); assert!( WireguardPlugin::open(WireguardConfig::new(dir.path().join("ok"))) .await .is_ok() ); } #[tokio::test] async fn a_peer_with_no_direct_path_is_reached_through_one_that_has_both() { // A and B can each reach C and not each other. Without a way through // the middle they are lost to one another while sitting in the same // mesh; with one, C carries their datagrams without being able to read // a byte of them — the WireGuard tunnel is still end to end. let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-relay"); let a_blocks = Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())); let b_blocks = Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())); let a = WgAgent::spawn_cut_off(&discovery, "tra", Arc::clone(&a_blocks)).await; let b = WgAgent::spawn_cut_off(&discovery, "trb", Arc::clone(&b_blocks)).await; let middle = WgAgent::spawn(&discovery, "trc").await; a_blocks.lock().unwrap().insert(b.endpoint_id()); b_blocks.lock().unwrap().insert(a.endpoint_id()); let network_id = middle.agent.join_network(&name, &secret).await.unwrap(); a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); // All three are members and all three talk: only the data path // between A and B is missing. wait_for_peers(&a.agent, network_id, 2).await; wait_for_peers(&b.agent, network_id, 2).await; middle.wait_for_tunnels(network_id, 2).await; let a_addr = a.overlay(network_id).await; let b_addr = b.overlay(network_id).await; assert_ne!(a_addr, b_addr); // A's tunnel to B comes up through C. wait_until("a's tunnel to b is established", || async { let view = a.plugin.overview(network_id)?; view.peers .iter() .find(|peer| peer.endpoint_id == b.endpoint_id())? .tunnel .as_ref() .map(|tunnel| tunnel.health.since_handshake)? .map(|_| ()) }) .await; let path = a .plugin .overview(network_id) .unwrap() .peers .iter() .find(|peer| peer.endpoint_id == b.endpoint_id()) .and_then(|peer| peer.tunnel.as_ref().map(|tunnel| tunnel.path.clone())) .unwrap_or_default(); assert!( path.contains("via"), "the path should say it goes through somebody: {path}" ); // And a real packet crosses: A's interface to B's interface, through C. a.tun(network_id) .await .push_from_os(tcp_packet(a_addr, b_addr, 1280)); let seen = tokio::time::timeout( tsunagi::testing::DEADLINE, b.tun(network_id).await.pop_to_os(), ) .await .expect("the packet should arrive") .unwrap(); assert_eq!(seen, tcp_packet(a_addr, b_addr, 1280)); a.shutdown().await; b.shutdown().await; middle.shutdown().await; } #[tokio::test] async fn a_chain_routes_through_two_transit_peers_without_touching_their_tuns() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-multihop"); let mut agents = Vec::new(); let mut blocked = Vec::new(); for index in 0..4 { let cuts = Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())); agents.push(WgAgent::spawn_cut_off(&discovery, &format!("mh{index}"), cuts.clone()).await); blocked.push(cuts); } // The authenticated control plane stays connected. The only available // data links are A--B--C--D; A and D need two transit routers. for (i, cuts) in blocked.iter().enumerate() { for (j, agent) in agents.iter().enumerate() { if i.abs_diff(j) > 1 { cuts.lock().unwrap().insert(agent.endpoint_id()); } } } let mut id = None; for agent in &agents { id = Some(agent.agent.join_network(&name, &secret).await.unwrap()); } let id = id.unwrap(); for agent in &agents { wait_for_peers(&agent.agent, id, 3).await; agent.wait_for_tunnels(id, 3).await; } let a = &agents[0]; let d = &agents[3]; let a_addr = a.overlay(id).await; let d_addr = d.overlay(id).await; let a_tun = a.tun(id).await; let d_tun = d.tun(id).await; for (from, to, source, destination) in [ (&a_tun, &d_tun, a_addr, d_addr), (&d_tun, &a_tun, d_addr, a_addr), ] { let packet = tcp_packet(source, destination, 1280); from.push_from_os(packet.clone()); let received = tokio::time::timeout(tsunagi::testing::DEADLINE, to.pop_to_os()) .await .unwrap() .unwrap(); assert_eq!(received, packet); } for middle in &agents[1..3] { assert!( middle .agent .network_status(id) .await .unwrap() .relay .forwarded > 0 ); assert!( tokio::time::timeout(Duration::from_millis(100), middle.tun(id).await.pop_to_os()) .await .is_err(), "transit ciphertext must never enter the intermediate host's TUN" ); } // A direct shortcut wins immediately; losing it restores the chain using // the same end-to-end tunnels and overlay addresses. for allow_direct in [true, false] { for (from, to) in [(0, 3), (3, 0)] { if allow_direct { blocked[from] .lock() .unwrap() .remove(&agents[to].endpoint_id()); } else { blocked[from] .lock() .unwrap() .insert(agents[to].endpoint_id()); } agents[from].agent.recheck_network(id).await.unwrap(); } for (from, to) in [(a, d), (d, a)] { wait_until("route follows the changed topology", || async { let view = from.plugin.overview(id)?; let tunnel = view .peers .iter() .find(|peer| peer.endpoint_id == to.endpoint_id())? .tunnel .as_ref()?; let relayed = tunnel.path.contains("relay 3 hops"); (tunnel.health.is_up() && if allow_direct { !tunnel.path.contains("relay") && !tunnel.path.contains("unreachable") } else { relayed }) .then_some(()) }) .await; } let packet = tcp_packet(a_addr, d_addr, 1280); a_tun.push_from_os(packet.clone()); assert_eq!( tokio::time::timeout(tsunagi::testing::DEADLINE, d_tun.pop_to_os()) .await .unwrap() .unwrap(), packet ); } for agent in agents { agent.shutdown().await; } } #[tokio::test] async fn lan_broadcast_fanout_is_scoped_opt_in_and_does_not_reflood() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("lan-broadcast"); let cuts = Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())); let a = WgAgent::spawn_cut_off(&discovery, "bca", cuts.clone()).await; let b = WgAgent::spawn(&discovery, "bcb").await; let c = WgAgent::spawn(&discovery, "bcc").await; cuts.lock().unwrap().insert(c.endpoint_id()); let id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); c.agent.join_network(&name, &secret).await.unwrap(); for agent in [&a, &b, &c] { agent.wait_for_tunnels(id, 2).await; } let addr = a.overlay(id).await; let at = a.tun(id).await; let bt = b.tun(id).await; let ct = c.tun(id).await; // Valid UDP discovery queries with game-like ports and an unchanged body. fn query(source: std::net::Ipv4Addr, destination: std::net::Ipv4Addr) -> Bytes { let mut bytes = vec![0u8; 36]; bytes[0] = 0x45; bytes[2..4].copy_from_slice(&36u16.to_be_bytes()); bytes[8] = 1; bytes[9] = 17; bytes[12..16].copy_from_slice(&source.octets()); bytes[16..20].copy_from_slice(&destination.octets()); bytes[20..22].copy_from_slice(&27015u16.to_be_bytes()); bytes[22..24].copy_from_slice(&6112u16.to_be_bytes()); bytes[24..26].copy_from_slice(&16u16.to_be_bytes()); bytes[28..].copy_from_slice(b"LAN GAME"); // UDP checksum zero is valid for IPv4. let sum: u32 = bytes[..20] .chunks_exact(2) .map(|w| u16::from_be_bytes([w[0], w[1]]) as u32) .sum(); let checksum = !((sum & 0xffff) + (sum >> 16)) as u16; bytes[10..12].copy_from_slice(&checksum.to_be_bytes()); Bytes::from(bytes) } let range = a.agent.network_status(id).await.unwrap().range.unwrap(); let directed = std::net::Ipv4Addr::from(u32::from(range.base) | (u32::MAX >> range.prefix_len)); for destination in [std::net::Ipv4Addr::BROADCAST, directed] { let packet = query(addr, destination); at.push_from_os(packet.clone()); for tun in [&bt, &ct] { assert_eq!( tokio::time::timeout(tsunagi::testing::DEADLINE, tun.pop_to_os()) .await .unwrap() .unwrap(), packet ); assert!( tokio::time::timeout(Duration::from_millis(100), tun.pop_to_os()) .await .is_err(), "each recipient gets one copy" ); } assert!( tokio::time::timeout(Duration::from_millis(100), at.pop_to_os()) .await .is_err(), "never reflect to the origin" ); } c.agent.set_broadcast(id, false).await.unwrap(); wait_until("broadcast opt-out reaches sender", || async { a.agent .network_status(id) .await .ok()? .peers .iter() .find(|peer| peer.endpoint_id == c.endpoint_id()) .filter(|peer| !peer.broadcast) .map(|_| ()) }) .await; let packet = query(addr, std::net::Ipv4Addr::BROADCAST); at.push_from_os(packet.clone()); assert_eq!( tokio::time::timeout(tsunagi::testing::DEADLINE, bt.pop_to_os()) .await .unwrap() .unwrap(), packet ); assert!( tokio::time::timeout(Duration::from_millis(150), ct.pop_to_os()) .await .is_err() ); // Disabling origin participation also stops outgoing copies. a.agent.set_broadcast(id, false).await.unwrap(); at.push_from_os(packet); assert!( tokio::time::timeout(Duration::from_millis(150), bt.pop_to_os()) .await .is_err() ); // Discovery's unicast reply still crosses the same multihop route. let reply = tcp_packet(c.overlay(id).await, addr, 1280); ct.push_from_os(reply.clone()); assert_eq!( tokio::time::timeout(tsunagi::testing::DEADLINE, at.pop_to_os()) .await .unwrap() .unwrap(), reply ); for agent in [a, b, c] { agent.shutdown().await; } }