Fix the TUN setup recipe: the overlay address was being flushed

The setup this tool printed did not work, and the agent then correctly
refused to start. A persistent TUN interface has no carrier until a process
attaches to it, and Linux flushes IPv6 addresses from an interface that
loses carrier unless net.ipv6.conf.<dev>.keep_addr_on_down is set, which it
is not by default. So `ip -6 address add` on a freshly created interface
silently lost the address before the agent ever ran.

The recipe now brings the link up first, sets keep_addr_on_down, and adds
the address with `nodad` — without which duplicate address detection can
never finish on an interface with no carrier and the address stays
tentative and unusable.

The agent's own retry loop made this worse: it attached, failed the address
check, dropped the device and toggled the carrier, which flushed the
address again. The check now runs before attaching to an existing
interface, so looking is not destructive.

Failures are self-diagnosing now: the check parses the IFA_F_* flags, tells
tentative and DAD-failed apart from missing, and lists the addresses the
interface actually has.

Four new tests, including one that reads this host's real /proc/net/if_inet6
and one that pins the ordering of the setup commands.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/bin/tsunagi.rs

@@ -328,10 +328,21 @@ async fn tun_setup(args: TunSetupArgs) -> Result<(), Box<dyn std::error::Error>>
 
     println!("# Network  {name} ({network})");
     println!("# Interface {interface}, address {address}/{OVERLAY_PREFIX_LEN}, mtu {mtu}");
-    println!("# Run once as root; then run `tsunagi up` as {user}.\n");
+    println!("# Run once as root; then run `tsunagi up` as {user}.");
+    println!(
+        "#\n\
+         # keep_addr_on_down matters: a persistent TUN interface has no carrier\n\
+         # until a process attaches, and Linux flushes IPv6 addresses from an\n\
+         # interface that loses carrier unless it is set. `nodad` matters for the\n\
+         # same reason: duplicate address detection can never finish without a\n\
+         # carrier, leaving the address tentative and unusable.\n"
+    );
     println!("sudo ip tuntap add dev {interface} mode tun user {user}");
-    println!("sudo ip -6 address add {address}/{OVERLAY_PREFIX_LEN} dev {interface}");
     println!("sudo ip link set dev {interface} mtu {mtu} up");
+    println!("sudo sysctl -qw net.ipv6.conf.{interface}.keep_addr_on_down=1");
+    println!("sudo ip -6 address add {address}/{OVERLAY_PREFIX_LEN} dev {interface} nodad");
+    println!("\n# To check it afterwards:");
+    println!("ip -6 addr show dev {interface}");
     println!("\n# To remove it again:");
     println!("sudo ip link del dev {interface}");
     Ok(())

src/dataplane/wireguard/mod.rs

@@ -64,4 +64,7 @@ pub use store::WgKeyStore;
 pub use tun::{MemoryTun, MemoryTunFactory, TunDevice, TunFactory, TunRequest};
 
 #[cfg(feature = "tun-device")]
-pub use tun::{SystemTunFactory, interface_exists, interface_has_address, setup_commands};
+pub use tun::{
+    Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6,
+    setup_commands,
+};

src/dataplane/wireguard/tun.rs

@@ -182,7 +182,10 @@ impl TunFactory for MemoryTunFactory {
 }
 
 #[cfg(feature = "tun-device")]
-pub use system::{SystemTunFactory, interface_exists, interface_has_address, setup_commands};
+pub use system::{
+    Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6,
+    setup_commands,
+};
 
 #[cfg(feature = "tun-device")]
 mod system {
@@ -271,38 +274,130 @@ mod system {
         std::path::Path::new(&format!("/sys/class/net/{name}")).exists()
     }
 
-    /// Whether an interface already carries an IPv6 address.
+    /// `IFA_F_TENTATIVE`: the address is not usable until DAD finishes, which
+    /// never happens on an interface with no carrier.
+    const IFA_F_TENTATIVE: u32 = 0x40;
+    /// `IFA_F_DADFAILED`: duplicate address detection rejected it.
+    const IFA_F_DADFAILED: u32 = 0x08;
+
+    /// One IPv6 address assigned to an interface.
+    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+    pub struct Assigned {
+        /// The address.
+        pub address: Ipv6Addr,
+        /// Raw `IFA_F_*` flags as the kernel reports them.
+        pub flags: u32,
+    }
+
+    impl Assigned {
+        /// Whether the address can actually carry traffic.
+        pub fn is_usable(&self) -> bool {
+            self.flags & (IFA_F_TENTATIVE | IFA_F_DADFAILED) == 0
+        }
+
+        /// A short explanation when it cannot.
+        pub fn why_unusable(&self) -> Option<&'static str> {
+            if self.flags & IFA_F_DADFAILED != 0 {
+                Some("duplicate address detection failed")
+            } else if self.flags & IFA_F_TENTATIVE != 0 {
+                Some(
+                    "still tentative; duplicate address detection cannot finish \
+                      on an interface with no carrier, so add it with `nodad`",
+                )
+            } else {
+                None
+            }
+        }
+    }
+
+    /// Parses the IPv6 addresses of one interface out of `/proc/net/if_inet6`.
+    ///
+    /// Each line is `<32 hex address> <ifindex> <prefixlen> <scope> <flags>
+    /// <device>`, all hexadecimal.
+    pub fn parse_if_inet6(contents: &str, name: &str) -> Vec<Assigned> {
+        contents
+            .lines()
+            .filter_map(|line| {
+                let fields: Vec<&str> = line.split_whitespace().collect();
+                if fields.len() < 6 || fields[5] != name {
+                    return None;
+                }
+                let raw = <[u8; 16]>::try_from(hex::decode(fields[0]).ok()?.as_slice()).ok()?;
+                Some(Assigned {
+                    address: Ipv6Addr::from(raw),
+                    flags: u32::from_str_radix(fields[4], 16).unwrap_or(0),
+                })
+            })
+            .collect()
+    }
+
+    /// The IPv6 addresses of an interface, or `None` if that cannot be read.
     ///
     /// Reads `/proc/net/if_inet6`, which needs no privileges.
-    pub fn interface_has_address(name: &str, address: Ipv6Addr) -> bool {
-        let Ok(contents) = std::fs::read_to_string("/proc/net/if_inet6") else {
-            // Cannot tell. Assume it is there rather than block on a guess.
-            return true;
+    pub fn interface_addresses(name: &str) -> Option<Vec<Assigned>> {
+        let contents = std::fs::read_to_string("/proc/net/if_inet6").ok()?;
+        Some(parse_if_inet6(&contents, name))
+    }
+
+    /// What is wrong with an interface's addressing, if anything.
+    pub(crate) fn check_address(name: &str, wanted: Ipv6Addr) -> Result<(), String> {
+        let Some(assigned) = interface_addresses(name) else {
+            // Cannot tell. Carry on rather than block on a guess.
+            return Ok(());
         };
-        let wanted = hex::encode(address.octets());
-        contents.lines().any(|line| {
-            let mut fields = line.split_whitespace();
-            let addr = fields.next().unwrap_or_default();
-            let iface = fields.last().unwrap_or_default();
-            addr.eq_ignore_ascii_case(&wanted) && iface == name
-        })
+        match assigned.iter().find(|entry| entry.address == wanted) {
+            Some(entry) if entry.is_usable() => Ok(()),
+            Some(entry) => Err(format!(
+                "interface `{name}` has {wanted} but it is unusable: {}",
+                entry.why_unusable().unwrap_or("unknown reason")
+            )),
+            None => {
+                let present = if assigned.is_empty() {
+                    "it currently has no IPv6 address at all".to_string()
+                } else {
+                    format!(
+                        "it currently has: {}",
+                        assigned
+                            .iter()
+                            .map(|entry| entry.address.to_string())
+                            .collect::<Vec<_>>()
+                            .join(", ")
+                    )
+                };
+                Err(format!(
+                    "interface `{name}` has no {wanted} address, {present}"
+                ))
+            }
+        }
     }
 
     /// The commands a privileged user runs once to prepare an interface.
+    ///
+    /// The order and the two extra settings matter. A persistent TUN
+    /// interface has no carrier until a process attaches to it, and Linux
+    /// flushes IPv6 addresses from an interface that loses carrier unless
+    /// `keep_addr_on_down` is set — so an address added without it silently
+    /// disappears before the agent ever starts. `nodad` is needed for the same
+    /// reason: duplicate address detection can never finish with no carrier,
+    /// and the address would stay tentative and unusable.
     pub fn setup_commands(request: &TunRequest, user: &str) -> Vec<String> {
         vec![
             format!(
                 "sudo ip tuntap add dev {} mode tun user {user}",
                 request.name
             ),
-            format!(
-                "sudo ip -6 address add {}/{} dev {}",
-                request.address, request.prefix_len, request.name
-            ),
             format!(
                 "sudo ip link set dev {} mtu {} up",
                 request.name, request.mtu
             ),
+            format!(
+                "sudo sysctl -qw net.ipv6.conf.{}.keep_addr_on_down=1",
+                request.name
+            ),
+            format!(
+                "sudo ip -6 address add {}/{} dev {} nodad",
+                request.address, request.prefix_len, request.name
+            ),
         ]
     }
 
@@ -335,6 +430,19 @@ mod system {
             Box::pin(async move {
                 let existed = interface_exists(&request.name);
 
+                // Check before attaching. Opening and then dropping the
+                // device toggles the carrier, and with the default
+                // `keep_addr_on_down=0` that is enough to flush the very
+                // address we are looking for.
+                if existed && let Err(reason) = check_address(&request.name, request.address) {
+                    let commands = setup_commands(&request, &current_user()).join("\n  ");
+                    return Err(PluginError::Unavailable(format!(
+                        "{reason}.\nAssigning an IPv6 address needs privileges. \
+                         Remove the interface and prepare it again:\n  sudo ip link del dev {}\n  {commands}",
+                        request.name
+                    )));
+                }
+
                 let mut config = tun::Configuration::default();
                 config.tun_name(&request.name);
                 if existed {
@@ -371,14 +479,11 @@ mod system {
                     PluginError::Unavailable(hint)
                 })?;
 
-                // Without its overlay address the interface can never receive
-                // anything, so say so instead of pretending to be up.
-                if !interface_has_address(&request.name, request.address) {
+                // An interface we just created has no address yet either.
+                if let Err(reason) = check_address(&request.name, request.address) {
                     let commands = setup_commands(&request, &current_user()).join("\n  ");
                     return Err(PluginError::Unavailable(format!(
-                        "interface `{}` has no {} address. Assigning an IPv6 address needs \
-                         privileges. Run:\n  {commands}",
-                        request.name, request.address
+                        "{reason}.\nAssigning an IPv6 address needs privileges. Run:\n  {commands}"
                     )));
                 }
 
@@ -393,3 +498,99 @@ mod system {
         }
     }
 }
+
+#[cfg(all(test, feature = "tun-device", target_os = "linux"))]
+mod system_tests {
+    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
+
+    use std::net::Ipv6Addr;
+
+    use super::TunRequest;
+    use super::system::{interface_addresses, parse_if_inet6, setup_commands};
+
+    const SAMPLE: &str = "\
+fe800000000000008baaeb0c433b635a 04 40 20 80 tailscale0
+00000000000000000000000000000001 01 80 10 80       lo
+fd559caf9652cb86321feac65c73bd82 05 40 00 80 tsun0
+fd559caf9652cb86321feac65c73bd83 05 40 00 40 tsun0
+fd559caf9652cb86321feac65c73bd84 05 40 00 08 tsun0
+";
+
+    #[test]
+    fn addresses_are_read_per_interface_with_their_flags() {
+        let found = parse_if_inet6(SAMPLE, "tsun0");
+        assert_eq!(found.len(), 3);
+        assert_eq!(
+            found[0].address,
+            "fd55:9caf:9652:cb86:321f:eac6:5c73:bd82"
+                .parse::<Ipv6Addr>()
+                .unwrap()
+        );
+        assert!(found[0].is_usable(), "permanent address is usable");
+
+        // Tentative: duplicate address detection never finishes without a
+        // carrier, so the address exists but cannot carry traffic.
+        assert!(!found[1].is_usable());
+        assert!(found[1].why_unusable().unwrap().contains("tentative"));
+
+        // Duplicate address detection failed outright.
+        assert!(!found[2].is_usable());
+        assert!(found[2].why_unusable().unwrap().contains("duplicate"));
+
+        assert!(parse_if_inet6(SAMPLE, "nosuchdev").is_empty());
+        // An interface name that is a prefix of another must not match.
+        assert!(parse_if_inet6(SAMPLE, "tsun").is_empty());
+    }
+
+    #[test]
+    fn malformed_lines_are_skipped_rather_than_panicking() {
+        assert!(parse_if_inet6("", "tsun0").is_empty());
+        assert!(parse_if_inet6("garbage", "tsun0").is_empty());
+        assert!(parse_if_inet6("zz 01 40 00 80 tsun0", "tsun0").is_empty());
+        assert!(parse_if_inet6("00 01 40 00 80 tsun0", "tsun0").is_empty());
+        // Flags that do not parse fall back to zero rather than dropping the
+        // address, so a usable address is never hidden by a formatting change.
+        let odd = parse_if_inet6("00000000000000000000000000000001 01 80 10 zz lo", "lo");
+        assert_eq!(odd.len(), 1);
+        assert!(odd[0].is_usable());
+    }
+
+    #[test]
+    fn loopback_is_found_on_this_host() {
+        // A real read of /proc/net/if_inet6: every Linux host has ::1 on lo.
+        let found = interface_addresses("lo").expect("/proc/net/if_inet6 should be readable");
+        assert!(
+            found
+                .iter()
+                .any(|entry| entry.address == Ipv6Addr::LOCALHOST),
+            "expected ::1 on lo, got {found:?}"
+        );
+        assert!(
+            interface_addresses("definitely-not-an-interface")
+                .unwrap()
+                .is_empty()
+        );
+    }
+
+    #[test]
+    fn the_setup_recipe_survives_a_carrier_drop() {
+        let request = TunRequest {
+            name: "tsun0".into(),
+            address: "fd00::1".parse().unwrap(),
+            prefix_len: 64,
+            mtu: 1100,
+        };
+        let commands = setup_commands(&request, "someone");
+
+        // The interface must be up before the address is added, the address
+        // must survive losing carrier, and it must not wait for duplicate
+        // address detection that can never complete.
+        let joined = commands.join("\n");
+        let up = joined.find("link set dev tsun0 mtu 1100 up").unwrap();
+        let keep = joined.find("keep_addr_on_down=1").unwrap();
+        let add = joined.find("address add fd00::1/64").unwrap();
+        assert!(up < keep && keep < add, "wrong order:\n{joined}");
+        assert!(joined.contains("nodad"));
+        assert!(joined.contains("user someone"));
+    }
+}