diff --git a/README.md b/README.md index 0cc7308..45dc104 100644 --- a/README.md +++ b/README.md @@ -123,15 +123,23 @@ tsunagi tun-setup --network lab --secret "$SECRET" ``` ```text -# Network lab (jwc6dcrtmo5zzdk7f6wfpkcqrpvqwr6po7vz3q2fvttgolt4ijfa) # Interface tsunjwc6dcrtmo5, address fd80:1210:f724:f620:d1bb:f982:3b6e:19bd/64, mtu 1100 # Run once as root; then run `tsunagi up` as ab. sudo ip tuntap add dev tsunjwc6dcrtmo5 mode tun user ab -sudo ip -6 address add fd80:1210:f724:f620:d1bb:f982:3b6e:19bd/64 dev tsunjwc6dcrtmo5 sudo ip link set dev tsunjwc6dcrtmo5 mtu 1100 up +sudo sysctl -qw net.ipv6.conf.tsunjwc6dcrtmo5.keep_addr_on_down=1 +sudo ip -6 address add fd80:1210:f724:f620:d1bb:f982:3b6e:19bd/64 dev tsunjwc6dcrtmo5 nodad ``` +The order and the last two lines are not decoration. A persistent TUN +interface has **no carrier** until a process attaches to it, and Linux flushes +IPv6 addresses from an interface that loses carrier unless +`keep_addr_on_down` is set — so an address added without it disappears before +the agent ever starts. `nodad` is needed for the same reason: duplicate +address detection cannot finish without a carrier, and the address would sit +there tentative and unusable. + `user ab` is the point: the interface is persistent and owned by that user, so `tsunagi up` afterwards opens it with **no privileges and no capabilities at all**. The interface name and address are derived, so they are stable — the diff --git a/docs/wireguard.md b/docs/wireguard.md index c86f744..a263096 100644 --- a/docs/wireguard.md +++ b/docs/wireguard.md @@ -190,6 +190,11 @@ async fn main() -> Result<()> { * **Userspace costs CPU.** Kernel WireGuard is faster. A kernel backend could return behind the same boundary, but it would give up transport-provided NAT traversal unless paired with a local proxy. +* **A persistent TUN interface needs `keep_addr_on_down`.** Without a process + attached it has no carrier, and Linux then flushes its IPv6 addresses. The + setup printed by `tsunagi tun-setup` sets it; the agent checks the address is + present *and usable* — not tentative, not DAD-failed — before attaching, and + reports what it actually found. * **The agent cannot assign the overlay address itself.** The `tun` crate sets addresses through an IPv4-only ioctl, so the IPv6 overlay address must come from `ip -6 address add` or an equivalent. The agent verifies the address is diff --git a/src/bin/tsunagi.rs b/src/bin/tsunagi.rs index 9e233fa..1994671 100644 --- a/src/bin/tsunagi.rs +++ b/src/bin/tsunagi.rs @@ -328,10 +328,21 @@ async fn tun_setup(args: TunSetupArgs) -> Result<(), Box> println!("# Network {name} ({network})"); println!("# Interface {interface}, address {address}/{OVERLAY_PREFIX_LEN}, mtu {mtu}"); - println!("# Run once as root; then run `tsunagi up` as {user}.\n"); + println!("# Run once as root; then run `tsunagi up` as {user}."); + println!( + "#\n\ + # keep_addr_on_down matters: a persistent TUN interface has no carrier\n\ + # until a process attaches, and Linux flushes IPv6 addresses from an\n\ + # interface that loses carrier unless it is set. `nodad` matters for the\n\ + # same reason: duplicate address detection can never finish without a\n\ + # carrier, leaving the address tentative and unusable.\n" + ); println!("sudo ip tuntap add dev {interface} mode tun user {user}"); - println!("sudo ip -6 address add {address}/{OVERLAY_PREFIX_LEN} dev {interface}"); println!("sudo ip link set dev {interface} mtu {mtu} up"); + println!("sudo sysctl -qw net.ipv6.conf.{interface}.keep_addr_on_down=1"); + println!("sudo ip -6 address add {address}/{OVERLAY_PREFIX_LEN} dev {interface} nodad"); + println!("\n# To check it afterwards:"); + println!("ip -6 addr show dev {interface}"); println!("\n# To remove it again:"); println!("sudo ip link del dev {interface}"); Ok(()) diff --git a/src/dataplane/wireguard/mod.rs b/src/dataplane/wireguard/mod.rs index 71465d7..11d40ac 100644 --- a/src/dataplane/wireguard/mod.rs +++ b/src/dataplane/wireguard/mod.rs @@ -64,4 +64,7 @@ pub use store::WgKeyStore; pub use tun::{MemoryTun, MemoryTunFactory, TunDevice, TunFactory, TunRequest}; #[cfg(feature = "tun-device")] -pub use tun::{SystemTunFactory, interface_exists, interface_has_address, setup_commands}; +pub use tun::{ + Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6, + setup_commands, +}; diff --git a/src/dataplane/wireguard/tun.rs b/src/dataplane/wireguard/tun.rs index 5859eb2..f2fa481 100644 --- a/src/dataplane/wireguard/tun.rs +++ b/src/dataplane/wireguard/tun.rs @@ -182,7 +182,10 @@ impl TunFactory for MemoryTunFactory { } #[cfg(feature = "tun-device")] -pub use system::{SystemTunFactory, interface_exists, interface_has_address, setup_commands}; +pub use system::{ + Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6, + setup_commands, +}; #[cfg(feature = "tun-device")] mod system { @@ -271,38 +274,130 @@ mod system { std::path::Path::new(&format!("/sys/class/net/{name}")).exists() } - /// Whether an interface already carries an IPv6 address. + /// `IFA_F_TENTATIVE`: the address is not usable until DAD finishes, which + /// never happens on an interface with no carrier. + const IFA_F_TENTATIVE: u32 = 0x40; + /// `IFA_F_DADFAILED`: duplicate address detection rejected it. + const IFA_F_DADFAILED: u32 = 0x08; + + /// One IPv6 address assigned to an interface. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct Assigned { + /// The address. + pub address: Ipv6Addr, + /// Raw `IFA_F_*` flags as the kernel reports them. + pub flags: u32, + } + + impl Assigned { + /// Whether the address can actually carry traffic. + pub fn is_usable(&self) -> bool { + self.flags & (IFA_F_TENTATIVE | IFA_F_DADFAILED) == 0 + } + + /// A short explanation when it cannot. + pub fn why_unusable(&self) -> Option<&'static str> { + if self.flags & IFA_F_DADFAILED != 0 { + Some("duplicate address detection failed") + } else if self.flags & IFA_F_TENTATIVE != 0 { + Some( + "still tentative; duplicate address detection cannot finish \ + on an interface with no carrier, so add it with `nodad`", + ) + } else { + None + } + } + } + + /// Parses the IPv6 addresses of one interface out of `/proc/net/if_inet6`. + /// + /// Each line is `<32 hex address> + /// `, all hexadecimal. + pub fn parse_if_inet6(contents: &str, name: &str) -> Vec { + contents + .lines() + .filter_map(|line| { + let fields: Vec<&str> = line.split_whitespace().collect(); + if fields.len() < 6 || fields[5] != name { + return None; + } + let raw = <[u8; 16]>::try_from(hex::decode(fields[0]).ok()?.as_slice()).ok()?; + Some(Assigned { + address: Ipv6Addr::from(raw), + flags: u32::from_str_radix(fields[4], 16).unwrap_or(0), + }) + }) + .collect() + } + + /// The IPv6 addresses of an interface, or `None` if that cannot be read. /// /// Reads `/proc/net/if_inet6`, which needs no privileges. - pub fn interface_has_address(name: &str, address: Ipv6Addr) -> bool { - let Ok(contents) = std::fs::read_to_string("/proc/net/if_inet6") else { - // Cannot tell. Assume it is there rather than block on a guess. - return true; + pub fn interface_addresses(name: &str) -> Option> { + let contents = std::fs::read_to_string("/proc/net/if_inet6").ok()?; + Some(parse_if_inet6(&contents, name)) + } + + /// What is wrong with an interface's addressing, if anything. + pub(crate) fn check_address(name: &str, wanted: Ipv6Addr) -> Result<(), String> { + let Some(assigned) = interface_addresses(name) else { + // Cannot tell. Carry on rather than block on a guess. + return Ok(()); }; - let wanted = hex::encode(address.octets()); - contents.lines().any(|line| { - let mut fields = line.split_whitespace(); - let addr = fields.next().unwrap_or_default(); - let iface = fields.last().unwrap_or_default(); - addr.eq_ignore_ascii_case(&wanted) && iface == name - }) + match assigned.iter().find(|entry| entry.address == wanted) { + Some(entry) if entry.is_usable() => Ok(()), + Some(entry) => Err(format!( + "interface `{name}` has {wanted} but it is unusable: {}", + entry.why_unusable().unwrap_or("unknown reason") + )), + None => { + let present = if assigned.is_empty() { + "it currently has no IPv6 address at all".to_string() + } else { + format!( + "it currently has: {}", + assigned + .iter() + .map(|entry| entry.address.to_string()) + .collect::>() + .join(", ") + ) + }; + Err(format!( + "interface `{name}` has no {wanted} address, {present}" + )) + } + } } /// The commands a privileged user runs once to prepare an interface. + /// + /// The order and the two extra settings matter. A persistent TUN + /// interface has no carrier until a process attaches to it, and Linux + /// flushes IPv6 addresses from an interface that loses carrier unless + /// `keep_addr_on_down` is set — so an address added without it silently + /// disappears before the agent ever starts. `nodad` is needed for the same + /// reason: duplicate address detection can never finish with no carrier, + /// and the address would stay tentative and unusable. pub fn setup_commands(request: &TunRequest, user: &str) -> Vec { vec![ format!( "sudo ip tuntap add dev {} mode tun user {user}", request.name ), - format!( - "sudo ip -6 address add {}/{} dev {}", - request.address, request.prefix_len, request.name - ), format!( "sudo ip link set dev {} mtu {} up", request.name, request.mtu ), + format!( + "sudo sysctl -qw net.ipv6.conf.{}.keep_addr_on_down=1", + request.name + ), + format!( + "sudo ip -6 address add {}/{} dev {} nodad", + request.address, request.prefix_len, request.name + ), ] } @@ -335,6 +430,19 @@ mod system { Box::pin(async move { let existed = interface_exists(&request.name); + // Check before attaching. Opening and then dropping the + // device toggles the carrier, and with the default + // `keep_addr_on_down=0` that is enough to flush the very + // address we are looking for. + if existed && let Err(reason) = check_address(&request.name, request.address) { + let commands = setup_commands(&request, ¤t_user()).join("\n "); + return Err(PluginError::Unavailable(format!( + "{reason}.\nAssigning an IPv6 address needs privileges. \ + Remove the interface and prepare it again:\n sudo ip link del dev {}\n {commands}", + request.name + ))); + } + let mut config = tun::Configuration::default(); config.tun_name(&request.name); if existed { @@ -371,14 +479,11 @@ mod system { PluginError::Unavailable(hint) })?; - // Without its overlay address the interface can never receive - // anything, so say so instead of pretending to be up. - if !interface_has_address(&request.name, request.address) { + // An interface we just created has no address yet either. + if let Err(reason) = check_address(&request.name, request.address) { let commands = setup_commands(&request, ¤t_user()).join("\n "); return Err(PluginError::Unavailable(format!( - "interface `{}` has no {} address. Assigning an IPv6 address needs \ - privileges. Run:\n {commands}", - request.name, request.address + "{reason}.\nAssigning an IPv6 address needs privileges. Run:\n {commands}" ))); } @@ -393,3 +498,99 @@ mod system { } } } + +#[cfg(all(test, feature = "tun-device", target_os = "linux"))] +mod system_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use std::net::Ipv6Addr; + + use super::TunRequest; + use super::system::{interface_addresses, parse_if_inet6, setup_commands}; + + const SAMPLE: &str = "\ +fe800000000000008baaeb0c433b635a 04 40 20 80 tailscale0 +00000000000000000000000000000001 01 80 10 80 lo +fd559caf9652cb86321feac65c73bd82 05 40 00 80 tsun0 +fd559caf9652cb86321feac65c73bd83 05 40 00 40 tsun0 +fd559caf9652cb86321feac65c73bd84 05 40 00 08 tsun0 +"; + + #[test] + fn addresses_are_read_per_interface_with_their_flags() { + let found = parse_if_inet6(SAMPLE, "tsun0"); + assert_eq!(found.len(), 3); + assert_eq!( + found[0].address, + "fd55:9caf:9652:cb86:321f:eac6:5c73:bd82" + .parse::() + .unwrap() + ); + assert!(found[0].is_usable(), "permanent address is usable"); + + // Tentative: duplicate address detection never finishes without a + // carrier, so the address exists but cannot carry traffic. + assert!(!found[1].is_usable()); + assert!(found[1].why_unusable().unwrap().contains("tentative")); + + // Duplicate address detection failed outright. + assert!(!found[2].is_usable()); + assert!(found[2].why_unusable().unwrap().contains("duplicate")); + + assert!(parse_if_inet6(SAMPLE, "nosuchdev").is_empty()); + // An interface name that is a prefix of another must not match. + assert!(parse_if_inet6(SAMPLE, "tsun").is_empty()); + } + + #[test] + fn malformed_lines_are_skipped_rather_than_panicking() { + assert!(parse_if_inet6("", "tsun0").is_empty()); + assert!(parse_if_inet6("garbage", "tsun0").is_empty()); + assert!(parse_if_inet6("zz 01 40 00 80 tsun0", "tsun0").is_empty()); + assert!(parse_if_inet6("00 01 40 00 80 tsun0", "tsun0").is_empty()); + // Flags that do not parse fall back to zero rather than dropping the + // address, so a usable address is never hidden by a formatting change. + let odd = parse_if_inet6("00000000000000000000000000000001 01 80 10 zz lo", "lo"); + assert_eq!(odd.len(), 1); + assert!(odd[0].is_usable()); + } + + #[test] + fn loopback_is_found_on_this_host() { + // A real read of /proc/net/if_inet6: every Linux host has ::1 on lo. + let found = interface_addresses("lo").expect("/proc/net/if_inet6 should be readable"); + assert!( + found + .iter() + .any(|entry| entry.address == Ipv6Addr::LOCALHOST), + "expected ::1 on lo, got {found:?}" + ); + assert!( + interface_addresses("definitely-not-an-interface") + .unwrap() + .is_empty() + ); + } + + #[test] + fn the_setup_recipe_survives_a_carrier_drop() { + let request = TunRequest { + name: "tsun0".into(), + address: "fd00::1".parse().unwrap(), + prefix_len: 64, + mtu: 1100, + }; + let commands = setup_commands(&request, "someone"); + + // The interface must be up before the address is added, the address + // must survive losing carrier, and it must not wait for duplicate + // address detection that can never complete. + let joined = commands.join("\n"); + let up = joined.find("link set dev tsun0 mtu 1100 up").unwrap(); + let keep = joined.find("keep_addr_on_down=1").unwrap(); + let add = joined.find("address add fd00::1/64").unwrap(); + assert!(up < keep && keep < add, "wrong order:\n{joined}"); + assert!(joined.contains("nodad")); + assert!(joined.contains("user someone")); + } +}