From 7c176e0b7491af2efb93d07ff5467025c26b05f8 Mon Sep 17 00:00:00 2001 From: Ultradesu Date: Sat, 3 Oct 2026 13:43:01 +0100 Subject: [PATCH] Exit nodes on Windows: use a member as exit node via destination bypass routes Choosing an exit node on Windows did nothing: the using side was refused, because Windows has no per-user routing and the agent's own traffic would loop into the tunnel it carries. Now the overlay interface gets 0.0.0.0/1 and 128.0.0.0/1 routes (and ::/1, 8000::/1 as an IPv6 blackhole), and every underlay address the agent talks to directly gets a /32 route through the physical default gateway, marked with route metric 7373 so stale ones are found and removed. Bypass routes are written before the halves and removed after them; a failed step removes what was written. All routes are in the active store. The addresses come from the runtime: each network runtime hands the interface the verified direct IPv4 paths of its control connections and the relay hosts in use (Interface::set_underlay, refreshed every dial round); relay names are resolved when the rules are applied. ExitHostRules::needs_bypass (true only on Windows) gates it, so Linux and macOS plans never carry or re-apply on path changes. ExitHostPlan gains `bypass` and `bypass_hosts`; networks leaving forget their entry. Limits: a path that appears after the last apply goes through the tunnel until the next apply; data-plane-only addresses are not collected separately (the data links ride the same iroh paths). Nothing was run on Windows. Checks: cargo fmt --check, cargo clippy --locked --workspace --all-targets -D warnings and cargo test --locked --workspace --all-targets passed on macOS; scripts and ordering are unit-tested with an injected runner and resolver, and Interface::set_underlay with the mock host. Co-Authored-By: Claude Sonnet 5.5 --- crates/tsunagi-core/src/agent/mod.rs | 5 + crates/tsunagi-core/src/agent/network.rs | 73 ++++ .../src/overlay/hostrules/exit.rs | 26 ++ .../src/overlay/hostrules/macos_exit.rs | 1 + .../src/overlay/hostrules/windows.rs | 26 +- .../src/overlay/hostrules/windows_exit.rs | 396 ++++++++++++++++-- crates/tsunagi-core/src/overlay/interface.rs | 146 ++++++- docs/exit-node.md | 31 +- 8 files changed, 659 insertions(+), 45 deletions(-) diff --git a/crates/tsunagi-core/src/agent/mod.rs b/crates/tsunagi-core/src/agent/mod.rs index 546d642..18582cb 100644 --- a/crates/tsunagi-core/src/agent/mod.rs +++ b/crates/tsunagi-core/src/agent/mod.rs @@ -800,6 +800,11 @@ impl Agent { // itself stays, because the agent owns it and other networks may // still be using it. self.inner.routes.remove_network(network_id); + if let Some(interface) = self.inner.interface.get() { + interface + .set_underlay(network_id, Vec::new(), Vec::new()) + .await; + } if let Some(interface) = self.inner.interface.get() && let Err(err) = interface.sync_addresses().await { diff --git a/crates/tsunagi-core/src/agent/network.rs b/crates/tsunagi-core/src/agent/network.rs index ad8f089..02ec0a8 100644 --- a/crates/tsunagi-core/src/agent/network.rs +++ b/crates/tsunagi-core/src/agent/network.rs @@ -712,6 +712,34 @@ impl Runtime { self.start_dials(); // Last, so it carries what this round learned. self.introduce_peers(); + self.refresh_underlay().await; + } + + /// Hands the interface the underlay destinations this network depends + /// on: the verified direct IPv4 paths of every control connection, and + /// the relays in use. Only hosts that cannot exempt the agent from an + /// exit node's default route read it. + async fn refresh_underlay(&self) { + let Some(interface) = self.params.interface.clone() else { + return; + }; + let mut addresses = Vec::new(); + let mut hosts = Vec::new(); + for session in self.sessions.values() { + for path in snapshot_connection(&session.conn).paths { + match path.remote { + PathAddr::Ip(std::net::SocketAddr::V4(socket)) => addresses.push(*socket.ip()), + PathAddr::Relay(url) => hosts.extend(relay_host(&url)), + _ => {} + } + } + } + for url in self.params.adapter.snapshot().relay_urls { + hosts.extend(relay_host(&url)); + } + interface + .set_underlay(self.network_id, addresses, hosts) + .await; } fn add_candidate(&mut self, candidate: Candidate) { @@ -2448,3 +2476,48 @@ mod tests { assert_eq!(chosen_restored.copied(), Some("wg")); } } + +/// The host name of a relay URL such as `https://relay.example:443/`. +fn relay_host(url: &str) -> Option { + let rest = url.split_once("://").map_or(url, |(_, rest)| rest); + let authority = rest.split(['/', '?', '#']).next()?; + let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let host = if host.starts_with('[') { + host.split_once(']')?.0.trim_start_matches('[') + } else { + host.split(':').next()? + }; + (!host.is_empty() + && host.len() <= 253 + && host + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == ':')) + .then(|| host.to_ascii_lowercase()) +} + +#[cfg(test)] +mod relay_host_tests { + use super::relay_host; + + #[test] + fn the_host_is_taken_out_of_a_relay_url() { + assert_eq!( + relay_host("https://relay.example.com./").as_deref(), + Some("relay.example.com.") + ); + assert_eq!( + relay_host("https://Euw1-1.relay.iroh.network:443/x").as_deref(), + Some("euw1-1.relay.iroh.network") + ); + assert_eq!( + relay_host("http://192.0.2.7:3340").as_deref(), + Some("192.0.2.7") + ); + assert_eq!( + relay_host("https://[2001:db8::1]:443/").as_deref(), + Some("2001:db8::1") + ); + assert_eq!(relay_host("https:///").as_deref(), None); + assert_eq!(relay_host("https://bad host/").as_deref(), None); + } +} diff --git a/crates/tsunagi-core/src/overlay/hostrules/exit.rs b/crates/tsunagi-core/src/overlay/hostrules/exit.rs index bf35ab7..313703d 100644 --- a/crates/tsunagi-core/src/overlay/hostrules/exit.rs +++ b/crates/tsunagi-core/src/overlay/hostrules/exit.rs @@ -39,6 +39,13 @@ pub struct ExitHostPlan { /// reads it, to keep this agent's own traffic *to* the overlay inside it /// where the host has no per-user routing (macOS). pub overlay: Vec, + /// Underlay addresses this agent talks to directly and must keep reaching + /// outside the overlay's default route. Only filled while `client` is set + /// and only for hosts that report [`ExitHostRules::needs_bypass`]. + pub bypass: Vec, + /// Host names (relay servers) that must stay reachable the same way; the + /// host rules resolve them. + pub bypass_hosts: Vec, } impl ExitHostPlan { @@ -76,6 +83,13 @@ pub trait ExitHostRules: Send + Sync + std::fmt::Debug + 'static { /// when nothing is there: it runs on the shutdown path, and once at /// startup to take away what a crashed run left. fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()>; + + /// Whether using an exit node on this host needs the underlay addresses + /// the agent talks to, because the host cannot exempt the agent's own + /// traffic from the default route by user id (Windows). The default is no. + fn needs_bypass(&self) -> bool { + false + } } /// The tag every firewall object of one offered range carries. @@ -103,6 +117,7 @@ pub struct MockExitRules { calls: Arc>>, forwarding: Arc>>, failure: Arc>>, + needs_bypass: Arc>, } impl MockExitRules { @@ -118,6 +133,11 @@ impl MockExitRules { *lock(&self.forwarding) = forwarding; } + /// Whether this host asks for the underlay addresses, like Windows. + pub fn set_needs_bypass(&self, needs: bool) { + *lock(&self.needs_bypass) = needs; + } + /// Makes every step fail with this reason from now on. pub fn fail_with(&self, reason: Option) { *lock(&self.failure) = reason; @@ -142,6 +162,10 @@ fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { } impl ExitHostRules for MockExitRules { + fn needs_bypass(&self) -> bool { + *lock(&self.needs_bypass) + } + fn apply<'a>(&'a self, plan: &'a ExitHostPlan) -> BoxFuture<'a, ExitHostReport> { Box::pin(async move { lock(&self.calls).push(format!("apply:{}", plan.interface)); @@ -209,6 +233,7 @@ mod tests { offer: vec![range()], client: true, overlay: vec![range()], + ..Default::default() }; let report = rules.apply(&plan).await; assert_eq!(report.offer[0].1, RuleOutcome::Applied); @@ -224,6 +249,7 @@ mod tests { offer: vec![range()], client: false, overlay: Vec::new(), + ..Default::default() }) .await; assert_eq!(report.forwarding, Some(false)); diff --git a/crates/tsunagi-core/src/overlay/hostrules/macos_exit.rs b/crates/tsunagi-core/src/overlay/hostrules/macos_exit.rs index adb4532..a833cbb 100644 --- a/crates/tsunagi-core/src/overlay/hostrules/macos_exit.rs +++ b/crates/tsunagi-core/src/overlay/hostrules/macos_exit.rs @@ -623,6 +623,7 @@ destination: default\n\ offer: if offer { vec![range()] } else { Vec::new() }, client, overlay: vec![range()], + ..Default::default() } } diff --git a/crates/tsunagi-core/src/overlay/hostrules/windows.rs b/crates/tsunagi-core/src/overlay/hostrules/windows.rs index 57689d5..e9b5fb9 100644 --- a/crates/tsunagi-core/src/overlay/hostrules/windows.rs +++ b/crates/tsunagi-core/src/overlay/hostrules/windows.rs @@ -162,12 +162,36 @@ impl WindowsHostRules { } } +/// The IPv4 addresses a host name resolves to; empty when it does not. +fn resolve_host(host: &str) -> Vec { + use std::net::{IpAddr, ToSocketAddrs}; + (host, 443) + .to_socket_addrs() + .map(|addrs| { + addrs + .filter_map(|addr| match addr.ip() { + IpAddr::V4(ip) => Some(ip), + IpAddr::V6(_) => None, + }) + .collect() + }) + .unwrap_or_default() +} + impl ExitHostRules for WindowsHostRules { + fn needs_bypass(&self) -> bool { + true + } + fn apply<'a>(&'a self, plan: &'a ExitHostPlan) -> BoxFuture<'a, ExitHostReport> { Box::pin(async move { let job = plan.clone(); tokio::task::spawn_blocking(move || { - windows_exit::apply_plan(&|script| run_powershell_output(script), &job) + windows_exit::apply_plan( + &|script| run_powershell_output(script), + &resolve_host, + &job, + ) }) .await .unwrap_or_else(|err| { diff --git a/crates/tsunagi-core/src/overlay/hostrules/windows_exit.rs b/crates/tsunagi-core/src/overlay/hostrules/windows_exit.rs index e62068b..3245f12 100644 --- a/crates/tsunagi-core/src/overlay/hostrules/windows_exit.rs +++ b/crates/tsunagi-core/src/overlay/hostrules/windows_exit.rs @@ -23,24 +23,46 @@ //! //! # Using //! -//! Not available. Windows has no per-user or per-process routing, so the -//! agent's own QUIC, relay and DHT traffic cannot be kept out of a default -//! route through the overlay: it would loop into the tunnel it carries. Rather -//! than install something that stops the agent working, the using side -//! reports why it is not in place and changes nothing. +//! Windows has no per-user or per-process routing, so the Linux scheme (an +//! exemption for the agent's user) does not carry over. Instead the agent's +//! own underlay traffic is exempted by *destination*: +//! +//! ```text +//! New-NetRoute 0.0.0.0/1 on the overlay interface, metric 1 +//! New-NetRoute 128.0.0.0/1 on the overlay interface, metric 1 +//! New-NetRoute /32 via the physical default gateway, metric 7373 +//! ``` +//! +//! The halves out-rank the default route without replacing it, so every more +//! specific route (LAN, another VPN) keeps working. A `/32` per address the +//! agent talks to directly (the verified direct paths of its connections, and +//! the relays it uses, resolved when the rules are applied) keeps that traffic +//! on the physical network instead of looping into the tunnel it carries. The +//! metric 7373 marks those routes as ours, which is how stale ones are found +//! and removed. Everything is written to the active store, so a reboot clears +//! it. IPv6 is blackholed with `::/1` and `8000::/1` on the overlay interface, +//! which has no IPv6: applications fall back to IPv4 after a delay, where +//! Linux and macOS answer at once. The bypass follows the agent's paths as +//! they change; a path that appears after the last apply is reached through +//! the tunnel until the next one. +//! +//! The bypass routes are written before the halves, and removed after them. use super::RuleOutcome; +use std::net::Ipv4Addr; + use super::exit::{ExitHostPlan, ExitHostReport, exit_tag, exit_tag_prefix}; use crate::state::Ipv4Range; -/// Why the using side is not installed. -pub(super) const CLIENT_UNAVAILABLE: &str = "using an exit node is not available on Windows yet: \ - Windows cannot keep the agent's own traffic out of a default route through the overlay, \ - so it would loop"; +/// The route metric that marks a bypass route as this agent's. +const BYPASS_METRIC: u32 = 7373; /// Runs a PowerShell script and returns what it printed. pub(super) type RunScript<'a> = &'a dyn Fn(&str) -> Result; +/// Resolves a host name to its IPv4 addresses. +pub(super) type Resolve<'a> = &'a dyn Fn(&str) -> Vec; + /// A PowerShell single-quoted literal, with any embedded quote doubled. fn ps_literal(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) @@ -86,6 +108,125 @@ fn forwarding_script(interface: &str) -> String { ) } +/// Prints ` ` of the best default route that is +/// not through the overlay interface. +fn gateway_script(interface: &str) -> String { + let alias = ps_literal(interface); + format!( + "$ErrorActionPreference = 'Stop'\n\ + $alias = {alias}\n\ + $best = Get-NetRoute -DestinationPrefix '0.0.0.0/0' -AddressFamily IPv4 \ + -PolicyStore ActiveStore | Where-Object {{ $_.InterfaceAlias -ne $alias -and \ + $_.NextHop -ne '0.0.0.0' }} | Sort-Object {{ $_.RouteMetric + \ + (Get-NetIPInterface -InterfaceIndex $_.InterfaceIndex -AddressFamily IPv4).InterfaceMetric }} | \ + Select-Object -First 1\n\ + if (-not $best) {{ throw 'the host has no default gateway outside the overlay' }}\n\ + \"$($best.InterfaceIndex) $($best.NextHop)\"\n" + ) +} + +/// The default gateway as the script above printed it. +fn parse_gateway(text: &str) -> Option<(u32, Ipv4Addr)> { + let mut words = text.split_whitespace(); + let index = words.next()?.parse().ok()?; + let hop = words.next()?.parse().ok()?; + words.next().is_none().then_some((index, hop)) +} + +/// Replaces the bypass routes with one per address, through the gateway. +fn bypass_script(index: u32, gateway: Ipv4Addr, addresses: &[Ipv4Addr]) -> String { + let list: Vec = addresses + .iter() + .map(|address| ps_literal(&address.to_string())) + .collect(); + format!( + "$ErrorActionPreference = 'Stop'\n\ + Get-NetRoute -AddressFamily IPv4 -PolicyStore ActiveStore -ErrorAction SilentlyContinue | \ + Where-Object {{ $_.RouteMetric -eq {BYPASS_METRIC} -and $_.DestinationPrefix.EndsWith('/32') }} | \ + Remove-NetRoute -Confirm:$false -ErrorAction SilentlyContinue\n\ + foreach ($ip in @({})) {{\n\ + New-NetRoute -DestinationPrefix \"$ip/32\" -InterfaceIndex {index} -NextHop '{gateway}' \ + -RouteMetric {BYPASS_METRIC} -PolicyStore ActiveStore -ErrorAction SilentlyContinue | Out-Null\n\ + }}\n", + list.join(", ") + ) +} + +/// Sends everything else through the overlay interface. +fn halves_script(interface: &str) -> String { + let alias = ps_literal(interface); + format!( + "$ErrorActionPreference = 'Stop'\n\ + $alias = {alias}\n\ + foreach ($p in '0.0.0.0/1', '128.0.0.0/1') {{\n\ + Get-NetRoute -DestinationPrefix $p -InterfaceAlias $alias -PolicyStore ActiveStore \ + -ErrorAction SilentlyContinue | Remove-NetRoute -Confirm:$false -ErrorAction SilentlyContinue\n\ + New-NetRoute -DestinationPrefix $p -InterfaceAlias $alias -NextHop '0.0.0.0' \ + -RouteMetric 1 -PolicyStore ActiveStore | Out-Null\n\ + }}\n\ + foreach ($p in '::/1', '8000::/1') {{\n\ + Get-NetRoute -DestinationPrefix $p -InterfaceAlias $alias -PolicyStore ActiveStore \ + -ErrorAction SilentlyContinue | Remove-NetRoute -Confirm:$false -ErrorAction SilentlyContinue\n\ + New-NetRoute -DestinationPrefix $p -InterfaceAlias $alias -NextHop '::' \ + -RouteMetric 1 -PolicyStore ActiveStore -ErrorAction SilentlyContinue | Out-Null\n\ + }}\n" + ) +} + +/// Removes the halves and every bypass route. What is not there is the state +/// wanted. +fn remove_client_script(interface: &str) -> String { + let alias = ps_literal(interface); + format!( + "$ErrorActionPreference = 'SilentlyContinue'\n\ + $alias = {alias}\n\ + foreach ($p in '0.0.0.0/1', '128.0.0.0/1', '::/1', '8000::/1') {{\n\ + Get-NetRoute -DestinationPrefix $p -InterfaceAlias $alias -PolicyStore ActiveStore | \ + Remove-NetRoute -Confirm:$false\n\ + }}\n\ + Get-NetRoute -AddressFamily IPv4 -PolicyStore ActiveStore | \ + Where-Object {{ $_.RouteMetric -eq {BYPASS_METRIC} -and $_.DestinationPrefix.EndsWith('/32') }} | \ + Remove-NetRoute -Confirm:$false\n\ + exit 0\n" + ) +} + +/// Every address to keep direct: the ones given and what the names resolve +/// to, sorted and without duplicates. +fn bypass_addresses( + resolve: Resolve<'_>, + addresses: &[Ipv4Addr], + hosts: &[String], +) -> Vec { + let mut all: std::collections::BTreeSet = addresses.iter().copied().collect(); + for host in hosts { + all.extend(resolve(host)); + } + all.into_iter() + .filter(|a| !a.is_loopback() && !a.is_unspecified() && !a.is_multicast()) + .collect() +} + +fn apply_client( + run: RunScript<'_>, + resolve: Resolve<'_>, + plan: &ExitHostPlan, +) -> Result<(), String> { + let interface = plan.interface.as_str(); + let gateway = run(&gateway_script(interface))?; + let (index, hop) = parse_gateway(&gateway) + .ok_or_else(|| format!("cannot read the default gateway from `{}`", gateway.trim()))?; + let addresses = bypass_addresses(resolve, &plan.bypass, &plan.bypass_hosts); + run(&bypass_script(index, hop, &addresses))?; + if let Err(why) = run(&halves_script(interface)) { + // Do not leave a half-installed set: bypass routes alone are harmless + // but the next apply starts clean. + let _ = run(&remove_client_script(interface)); + return Err(why); + } + Ok(()) +} + fn parse_forwarding(text: &str) -> Option { match text.trim().to_ascii_lowercase().as_str() { "enabled" | "1" => Some(true), @@ -95,7 +236,11 @@ fn parse_forwarding(text: &str) -> Option { } /// Makes the host match the plan. -pub(super) fn apply_plan(run: RunScript<'_>, plan: &ExitHostPlan) -> ExitHostReport { +pub(super) fn apply_plan( + run: RunScript<'_>, + resolve: Resolve<'_>, + plan: &ExitHostPlan, +) -> ExitHostReport { let interface = plan.interface.as_str(); // What an earlier plan left and this one no longer wants. let pruned = run(&prune_script(interface, &plan.offer)); @@ -113,6 +258,16 @@ pub(super) fn apply_plan(run: RunScript<'_>, plan: &ExitHostPlan) -> ExitHostRep (*range, outcome) }) .collect(); + let client = if plan.client { + Some(match apply_client(run, resolve, plan) { + Ok(()) => RuleOutcome::Applied, + Err(why) => RuleOutcome::Failed(why), + }) + } else { + // Whatever an earlier plan left must not stay in force. + let _ = run(&remove_client_script(interface)); + None + }; let forwarding = if plan.offer.is_empty() { None } else { @@ -123,9 +278,7 @@ pub(super) fn apply_plan(run: RunScript<'_>, plan: &ExitHostPlan) -> ExitHostRep ExitHostReport { offer, forwarding, - client: plan - .client - .then(|| RuleOutcome::Failed(CLIENT_UNAVAILABLE.to_string())), + client, } } @@ -134,6 +287,9 @@ pub(super) fn clear_all(run: RunScript<'_>, interface: &str) { if let Err(why) = run(&prune_script(interface, &[])) { tracing::warn!(interface, "cannot remove the exit node NAT: {why}"); } + if let Err(why) = run(&remove_client_script(interface)) { + tracing::warn!(interface, "cannot remove the exit node routes: {why}"); + } } #[cfg(test)] @@ -154,6 +310,7 @@ mod tests { offer: vec![range()], client, overlay: vec![range()], + ..Default::default() } } @@ -210,14 +367,16 @@ mod tests { String::new() }) }; - let report = apply_plan(&run, &plan(false)); + let report = apply_plan(&run, &no_names, &plan(false)); assert_eq!(report.offer, vec![(range(), RuleOutcome::Applied)]); assert_eq!(report.forwarding, Some(false)); assert_eq!(report.client, None); let calls = calls.borrow(); assert!(calls[0].contains("StartsWith"), "{calls:?}"); assert!(calls[1].contains("New-NetNat"), "{calls:?}"); - assert!(calls[2].contains("Get-NetIPInterface"), "{calls:?}"); + // Without a client, the routes of an earlier plan are taken away. + assert!(calls[2].contains("RouteMetric -eq 7373"), "{calls:?}"); + assert!(calls[3].contains("Get-NetIPInterface"), "{calls:?}"); } #[test] @@ -229,39 +388,216 @@ mod tests { Ok(String::new()) } }; - let report = apply_plan(&run, &plan(false)); + let report = apply_plan(&run, &no_names, &plan(false)); let RuleOutcome::Failed(why) = &report.offer[0].1 else { panic!("{report:?}"); }; assert!(why.contains("already exists"), "{why}"); } + fn no_names(_: &str) -> Vec { + Vec::new() + } + + fn client_plan() -> ExitHostPlan { + ExitHostPlan { + interface: "tsun0".into(), + offer: Vec::new(), + client: true, + overlay: vec![range()], + bypass: vec![Ipv4Addr::new(138, 201, 61, 182)], + bypass_hosts: vec!["relay.example".into()], + } + } + #[test] - fn using_an_exit_node_is_refused_and_changes_nothing() { - let calls = RefCell::new(0usize); - let run = |script: &str| -> Result { - *calls.borrow_mut() += 1; - // No script may touch routes. - assert!(!script.contains("Route"), "{script}"); - Ok(String::new()) + fn the_gateway_is_read_from_the_script_output() { + assert_eq!( + parse_gateway("12 192.168.1.1\r\n"), + Some((12, Ipv4Addr::new(192, 168, 1, 1))) + ); + assert_eq!(parse_gateway(""), None); + assert_eq!(parse_gateway("12"), None); + assert_eq!(parse_gateway("12 gw"), None); + assert_eq!(parse_gateway("12 1.1.1.1 extra"), None); + let script = gateway_script("tsun0"); + assert!(script.contains("-ne $alias")); + assert!(script.contains("-ne '0.0.0.0'")); + } + + #[test] + fn bypass_routes_are_marked_host_routes_through_the_physical_gateway() { + let script = bypass_script( + 12, + Ipv4Addr::new(192, 168, 1, 1), + &[Ipv4Addr::new(1, 2, 3, 4), Ipv4Addr::new(5, 6, 7, 8)], + ); + assert!(script.contains("@('1.2.3.4', '5.6.7.8')"), "{script}"); + assert!(script.contains("-InterfaceIndex 12 -NextHop '192.168.1.1'")); + assert!(script.contains("-RouteMetric 7373")); + assert!(script.contains("-PolicyStore ActiveStore")); + // Stale marked routes go first, and only /32 ones with our metric. + assert!(script.find("Remove-NetRoute").unwrap() < script.find("New-NetRoute").unwrap()); + assert!(script.contains("RouteMetric -eq 7373")); + assert!(script.contains("EndsWith('/32')")); + } + + #[test] + fn the_halves_go_through_the_overlay_and_ipv6_is_blackholed() { + let script = halves_script("tsun0"); + assert!(script.contains("'0.0.0.0/1', '128.0.0.0/1'")); + assert!(script.contains("-InterfaceAlias $alias -NextHop '0.0.0.0'")); + assert!(script.contains("'::/1', '8000::/1'")); + assert!(script.contains("-PolicyStore ActiveStore")); + assert!( + !script.contains("0.0.0.0/0"), + "the default route is left alone" + ); + assert!(halves_script("x'y").contains("'x''y'")); + } + + #[test] + fn addresses_are_resolved_merged_and_cleaned() { + let resolve = |host: &str| -> Vec { + match host { + "relay.example" => vec![Ipv4Addr::new(9, 9, 9, 9), Ipv4Addr::new(1, 2, 3, 4)], + _ => Vec::new(), + } }; - let report = apply_plan(&run, &plan(true)); + let all = bypass_addresses( + &resolve, + &[ + Ipv4Addr::new(1, 2, 3, 4), + Ipv4Addr::LOCALHOST, + Ipv4Addr::UNSPECIFIED, + ], + &["relay.example".into(), "gone.example".into()], + ); + assert_eq!( + all, + vec![Ipv4Addr::new(1, 2, 3, 4), Ipv4Addr::new(9, 9, 9, 9)] + ); + } + + #[test] + fn the_bypass_is_written_before_the_halves_that_need_it() { + let calls = RefCell::new(Vec::new()); + let run = |script: &str| -> Result { + calls.borrow_mut().push(script.to_string()); + Ok(if script.contains("Select-Object -First 1") { + "12 192.168.1.1\n".to_string() + } else { + String::new() + }) + }; + let resolve = |_: &str| vec![Ipv4Addr::new(9, 9, 9, 9)]; + let report = apply_plan(&run, &resolve, &client_plan()); + assert_eq!(report.client, Some(RuleOutcome::Applied)); + let calls = calls.borrow(); + let at = |needle: &str| calls.iter().position(|c| c.contains(needle)).unwrap(); + assert!(at("Select-Object -First 1") < at("-RouteMetric 7373")); + assert!(at("-RouteMetric 7373") < at("'0.0.0.0/1', '128.0.0.0/1'")); + let bypass = &calls[at("-RouteMetric 7373")]; + assert!( + bypass.contains("'9.9.9.9'") && bypass.contains("'138.201.61.182'"), + "{bypass}" + ); + } + + #[test] + fn nothing_goes_through_the_overlay_when_the_bypass_cannot_be_written() { + let run = |script: &str| -> Result { + if script.contains("Select-Object -First 1") { + Ok("12 192.168.1.1".to_string()) + } else if script.contains("-RouteMetric 7373") && script.contains("foreach ($ip") { + Err("Access is denied".to_string()) + } else { + assert!(!script.contains("-NextHop '0.0.0.0'"), "{script}"); + Ok(String::new()) + } + }; + let report = apply_plan(&run, &no_names, &client_plan()); + assert_eq!( + report.client, + Some(RuleOutcome::Failed("Access is denied".into())) + ); + } + + #[test] + fn a_missing_gateway_is_a_failure_that_installs_no_route() { + let run = |script: &str| -> Result { + assert!(!script.contains("New-NetRoute"), "{script}"); + if script.contains("Select-Object -First 1") { + Err("the host has no default gateway outside the overlay".into()) + } else { + Ok(String::new()) + } + }; + let report = apply_plan(&run, &no_names, &client_plan()); let Some(RuleOutcome::Failed(why)) = report.client else { panic!("{report:?}"); }; - assert!(why.contains("not available on Windows"), "{why}"); - assert!(why.contains("loop"), "{why}"); + assert!(why.contains("no default gateway"), "{why}"); + } + + #[test] + fn a_failed_halves_step_removes_what_was_written() { + let calls = RefCell::new(Vec::new()); + let run = |script: &str| -> Result { + calls.borrow_mut().push(script.to_string()); + if script.contains("Select-Object -First 1") { + Ok("12 192.168.1.1".to_string()) + } else if script.contains("'128.0.0.0/1'") && script.contains("New-NetRoute") { + Err("boom".to_string()) + } else { + Ok(String::new()) + } + }; + let report = apply_plan(&run, &no_names, &client_plan()); + assert!(matches!(report.client, Some(RuleOutcome::Failed(_)))); + assert!(calls.borrow().last().unwrap().contains("Remove-NetRoute")); + } + + #[test] + fn dropping_the_client_removes_its_routes_and_clearing_does_too() { + let calls = RefCell::new(Vec::new()); + let run = |script: &str| -> Result { + calls.borrow_mut().push(script.to_string()); + Ok(String::new()) + }; + let report = apply_plan(&run, &no_names, &plan(false)); + assert_eq!(report.client, None); + assert!( + calls + .borrow() + .iter() + .any(|c| c.contains("RouteMetric -eq 7373")) + ); + let script = remove_client_script("tsun0"); + assert!(script.contains("'0.0.0.0/1', '128.0.0.0/1', '::/1', '8000::/1'")); + assert!(script.contains("exit 0")); + // Only routes on our interface, or marked ours, are touched. + assert!(script.contains("-InterfaceAlias $alias")); + calls.borrow_mut().clear(); + clear_all(&run, "tsun0"); + assert!( + calls + .borrow() + .iter() + .any(|c| c.contains("RouteMetric -eq 7373")) + ); } #[test] fn clearing_prunes_everything_for_the_interface() { - let seen = RefCell::new(String::new()); + let seen = RefCell::new(Vec::::new()); let run = |script: &str| -> Result { - *seen.borrow_mut() = script.to_string(); + seen.borrow_mut().push(script.to_string()); Ok(String::new()) }; clear_all(&run, "tsun0"); - assert!(seen.borrow().contains("$keep = @()")); - assert!(seen.borrow().contains("'tsunagi-exit:tsun0:'")); + let seen = seen.borrow(); + assert!(seen[0].contains("$keep = @()")); + assert!(seen[0].contains("'tsunagi-exit:tsun0:'")); } } diff --git a/crates/tsunagi-core/src/overlay/interface.rs b/crates/tsunagi-core/src/overlay/interface.rs index 1119724..6ca55f5 100644 --- a/crates/tsunagi-core/src/overlay/interface.rs +++ b/crates/tsunagi-core/src/overlay/interface.rs @@ -164,6 +164,17 @@ pub struct Interface { rules_report: std::sync::Mutex>, /// What the exit-node rules were last asked for, and what came of it. exit_state: std::sync::Mutex, + /// Where each network's runtime reaches its peers and relays outside the + /// overlay; read only by hosts that cannot exempt the agent from the + /// exit node's default route. + underlay: std::sync::Mutex>, +} + +/// Underlay destinations one network's runtime currently depends on. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct Underlay { + addresses: std::collections::BTreeSet, + hosts: std::collections::BTreeSet, } /// The exit-node host rules as this interface last left them. @@ -275,6 +286,7 @@ impl Interface { host_rules, rules_report: std::sync::Mutex::new(None), exit_state: std::sync::Mutex::new(ExitState::default()), + underlay: std::sync::Mutex::new(std::collections::HashMap::new()), }) } @@ -350,11 +362,20 @@ impl Interface { .collect(); ranges.sort_by_key(|range| (range.base, range.prefix_len)); ranges.dedup(); + let client = self.routes.exit_via().is_some(); + let overlay = self.routes.overlay_ranges(); + let (bypass, bypass_hosts) = if client && exit.needs_bypass() { + self.underlay_targets(&overlay) + } else { + (Vec::new(), Vec::new()) + }; let wanted = ExitHostPlan { interface: self.name.clone(), offer: ranges, - client: self.routes.exit_via().is_some(), - overlay: self.routes.overlay_ranges(), + client, + overlay, + bypass, + bypass_hosts, }; let first = !std::mem::replace(&mut self.exit_state().cleaned, true); @@ -426,6 +447,68 @@ impl Interface { state.report = Some(report); } + /// Tells the interface where a network's runtime reaches peers and + /// relays outside the overlay. When the host needs those kept out of the + /// exit node's default route and the set changed, the rules follow. + pub async fn set_underlay( + &self, + network: NetworkId, + addresses: impl IntoIterator, + hosts: impl IntoIterator, + ) { + let new = Underlay { + addresses: addresses.into_iter().collect(), + hosts: hosts.into_iter().collect(), + }; + let changed = { + let mut map = match self.underlay.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + if new == Underlay::default() { + map.remove(&network).is_some() + } else { + map.insert(network, new.clone()) != Some(new) + } + }; + let needed = changed + && self.routes.exit_via().is_some() + && self + .host_rules + .as_ref() + .and_then(|rules| rules.exit_rules()) + .is_some_and(|exit| exit.needs_bypass()); + if needed { + self.sync_exit_rules().await; + } + } + + /// Every underlay destination worth keeping direct, minus anything that + /// is the overlay itself, loopback or link-local. + fn underlay_targets( + &self, + overlay: &[crate::state::Ipv4Range], + ) -> (Vec, Vec) { + let map = match self.underlay.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + let mut addresses = std::collections::BTreeSet::new(); + let mut hosts = std::collections::BTreeSet::new(); + for underlay in map.values() { + addresses.extend(underlay.addresses.iter().copied().filter(|address| { + !address.is_loopback() + && !address.is_link_local() + && !address.is_multicast() + && !address.is_unspecified() + && !address.is_broadcast() + && !overlay.iter().any(|range| range.contains(*address)) + })); + hosts.extend(underlay.hosts.iter().cloned()); + } + (addresses.into_iter().collect(), hosts.into_iter().collect()) + } + fn exit_state(&self) -> std::sync::MutexGuard<'_, ExitState> { match self.exit_state.lock() { Ok(guard) => guard, @@ -1343,6 +1426,65 @@ mod tests { assert_eq!(interface.counters().received, 1); } + #[tokio::test] + async fn the_underlay_reaches_the_rules_only_for_hosts_that_need_it_and_only_while_using_one() { + let rules = MockHostRules::new(); + let (interface, routes, id) = managed_interface(&rules, false).await; + let relay = || vec!["relay.example".to_string()]; + let direct = || { + vec![ + Ipv4Addr::new(138, 201, 61, 182), + Ipv4Addr::new(10, 13, 37, 9), // the overlay itself + Ipv4Addr::LOCALHOST, + Ipv4Addr::new(169, 254, 1, 1), + ] + }; + + // A host that does not ask never gets them, so Linux and macOS do not + // re-apply their rules whenever a path moves. + routes.set_exit(id, using(peer(2))); + interface.set_underlay(id, direct(), relay()).await; + interface.sync_exit_rules().await; + let plan = rules.exit().installed().unwrap(); + assert!(plan.client); + assert!(plan.bypass.is_empty() && plan.bypass_hosts.is_empty()); + + // A host that does gets the usable ones, and follows changes. + rules.exit().set_needs_bypass(true); + interface.set_underlay(id, direct(), relay()).await; + interface.sync_exit_rules().await; + let plan = rules.exit().installed().unwrap(); + assert_eq!(plan.bypass, vec![Ipv4Addr::new(138, 201, 61, 182)]); + assert_eq!(plan.bypass_hosts, relay()); + let applies = || { + rules + .exit() + .calls() + .iter() + .filter(|c| c.starts_with("apply")) + .count() + }; + let before = applies(); + interface.set_underlay(id, direct(), relay()).await; + assert_eq!(applies(), before, "an unchanged set is not applied again"); + interface + .set_underlay(id, vec![Ipv4Addr::new(5, 6, 7, 8)], relay()) + .await; + assert_eq!(applies(), before + 1); + assert_eq!( + rules.exit().installed().unwrap().bypass, + vec![Ipv4Addr::new(5, 6, 7, 8)] + ); + + // Not using an exit node: nothing to keep direct. + routes.set_exit(id, Default::default()); + interface.sync_exit_rules().await; + interface + .set_underlay(id, vec![Ipv4Addr::new(1, 1, 1, 1)], relay()) + .await; + assert!(rules.exit().installed().is_none()); + } + #[tokio::test] async fn the_exit_rules_follow_the_routing_table_and_leave_with_the_interface() { let rules = MockHostRules::new(); diff --git a/docs/exit-node.md b/docs/exit-node.md index 1f977b2..b132e9e 100644 --- a/docs/exit-node.md +++ b/docs/exit-node.md @@ -82,18 +82,25 @@ is built from `pf` and `route`. The agent runs as root there. ## Windows -Offering only. Each offered range gets a WinNAT object -`tsunagi-exit::` (`New-NetNat`), removed with the offer, the -network and the agent. Forwarding is read for the overlay interface and never -changed; Windows forwards per interface, so enable it on the ones involved -(`Get-NetIPInterface | Set-NetIPInterface -Forwarding Enabled`, administrator -PowerShell). Some Windows versions allow one NAT per host; a second range then -reports its own failure. - -Using one is **not available**: Windows has no per-user routing, so the agent's -own traffic cannot be kept out of a default route through the overlay and would -loop. `tsng exit-node ` reports that and installs nothing. This was -written without a Windows host to run it on. +- **Offering:** each offered range gets a WinNAT object + `tsunagi-exit::` (`New-NetNat`), removed with the offer, the + network and the agent. Forwarding is read for the overlay interface and never + changed; Windows forwards per interface, so enable it on the ones involved + (`Get-NetIPInterface | Set-NetIPInterface -Forwarding Enabled`, administrator + PowerShell). Some Windows versions allow one NAT per host; a second range then + reports its own failure. +- **Using:** Windows has no per-user routing, so the agent's own traffic is kept + off the tunnel *by destination*. `0.0.0.0/1` and `128.0.0.0/1` go through the + overlay interface; a `/32` route with metric 7373 (the marker for "ours") + through the physical default gateway is added for every address the agent + talks to directly: verified direct paths of its connections, plus the relays + it uses, resolved when the rules are applied. The set follows path changes. A + path that appears after the last apply goes through the tunnel until the next + one. IPv6 is blackholed with `::/1` and `8000::/1` on the overlay interface, + so applications fall back to IPv4 after a delay rather than at once. + Everything is in the active store: a reboot clears it. +- Not verified on a live host when this was written. To inspect: + `Get-NetRoute -PolicyStore ActiveStore | ? { $_.RouteMetric -eq 7373 -or $_.InterfaceAlias -like 'tsun*' }`. ## What it does not do