diff --git a/Cargo.lock b/Cargo.lock index 52c2407..01991bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ dependencies = [ "ip_network", "ip_network_table", "libc", - "nix", + "nix 0.31.3", "parking_lot", "portable-atomic", "rand_core 0.6.4", @@ -307,6 +307,15 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "caps" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1ddba47aba30b6a889298ad0109c3b8dcb0e8fc993b459daa7067d46f865e0" +dependencies = [ + "libc", +] + [[package]] name = "cc" version = "1.4.7" @@ -2065,6 +2074,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "netlink-proto" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e03ae6247c6cdb499e8f14e98c72b83954d85822b687056e3a21150b438f2b" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.9.0", + "netlink-sys 0.9.0", + "thiserror 2.0.20", +] + [[package]] name = "netlink-sys" version = "0.8.8" @@ -2085,8 +2109,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10" dependencies = [ "bytes", + "futures-util", "libc", "log", + "tokio", ] [[package]] @@ -2109,7 +2135,7 @@ dependencies = [ "netdev 0.46.3", "netlink-packet-core 0.8.2", "netlink-packet-route 0.31.0", - "netlink-proto", + "netlink-proto 0.12.2", "netlink-sys 0.8.8", "noq-udp", "objc2-core-foundation", @@ -2127,6 +2153,18 @@ dependencies = [ "wmi", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -2748,6 +2786,24 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "rtnetlink" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f60a3ee149005c5819e03e3d73ba4a470bc18b52b0c8422cfe1bea2cae63c6" +dependencies = [ + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.9.0", + "netlink-packet-route 0.33.0", + "netlink-proto 0.13.0", + "netlink-sys 0.9.0", + "nix 0.30.1", + "thiserror 2.0.20", + "tokio", +] + [[package]] name = "rusqlite" version = "0.40.2" @@ -3602,10 +3658,12 @@ version = "0.1.0" dependencies = [ "boringtun", "bytes", + "caps", "clap", "data-encoding", "directories", "fs4", + "futures-util", "hex", "hkdf", "hmac 0.13.0", @@ -3613,6 +3671,7 @@ dependencies = [ "netwatch", "postcard", "rand", + "rtnetlink", "rusqlite", "serde", "sha2", @@ -3639,7 +3698,7 @@ dependencies = [ "ipnet", "libc", "log", - "nix", + "nix 0.31.3", "serde_json", "thiserror 2.0.20", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 9fd1be1..e1a9871 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ cli = ["dep:clap", "dep:tracing-subscriber", "tokio/signal", "tun-device"] # A real TUN device, so the WireGuard plugin can carry actual IP traffic. # Needs CAP_NET_ADMIN at run time; without it the plugin still runs and its # in-memory device can be used for tests. -tun-device = ["dep:tun"] +tun-device = ["dep:tun", "dep:rtnetlink", "dep:caps", "dep:futures-util"] [[bin]] name = "tsunagi" @@ -49,6 +49,14 @@ bytes = "1.12.1" boringtun = { version = "0.7.1", default-features = false } tun = { version = "0.8", features = ["async"], optional = true } +# Linux-only interface provisioning. `rtnetlink` configures the interface in +# process, so no `ip` invocation is ever needed; `caps` keeps CAP_NET_ADMIN +# out of the effective set except during the moments it is used. +[target.'cfg(target_os = "linux")'.dependencies] +rtnetlink = { version = "0.23", optional = true } +caps = { version = "0.5", optional = true } +futures-util = { version = "0.3", default-features = false, optional = true } + [dev-dependencies] tokio = { version = "1.53", features = ["rt", "rt-multi-thread", "sync", "time", "macros", "process"] } tempfile = "3.24" diff --git a/README.md b/README.md index 9ef0505..aa3b8c4 100644 --- a/README.md +++ b/README.md @@ -146,19 +146,57 @@ No vote is involved — see [docs/sync-model.md](docs/sync-model.md). Because the address is allocated at run time rather than derived, it is not -known until the agent has started and agreed with its peers, so -`tsunagi tun-setup` cannot print it in advance. The agent prints the exact -`ip address add` command once it has one, and keeps saying so until the -address is actually on an interface — without it, packets leave with the -wrong source address and every peer drops them. +known until the agent has started and agreed with its peers. On Linux the +agent assigns it itself as soon as it has one; elsewhere `tsunagi tun-setup` +prints the `ip address add` line once the claim is in `state.sqlite`. -## Running unprivileged +## Privileges -The agent does not need to run as root. Creating a network interface and -giving it an address do need privileges, but they are a **one-time setup step** -that can be done separately. +On Linux the agent **manages its own overlay interface**. It creates the TUN +interface, sets the MTU, brings it up and assigns both overlay addresses, all +over netlink in process — no `ip` invocation, no shell, nothing that a +remote peer could influence. -Ask the agent what it needs, then run that once as root: +That needs `CAP_NET_ADMIN`, granted once: + +```bash +sudo setcap cap_net_admin+p /usr/local/bin/tsunagi +``` + +`+p` rather than `+ep`: the capability is then *permitted* but not +*effective*, and the agent raises it only around the handful of netlink calls +that need it — a few milliseconds at startup, and again if its address +allocation changes. Everything else, including every byte from the network, +is handled with it lowered. `+ep` works too; the agent lowers it on the way +in. + +`tsunagi doctor` says which of these applies on the host it runs on. + +### It cleans up after itself + +The interface is tied to an open file descriptor and is deliberately **not** +made persistent, so the kernel removes it when the agent exits — on a clean +shutdown, on a panic, on `SIGKILL`, on power loss alike. Keeping it is what +would take an action; removing it is the default. + +If something is left behind anyway — an interface made by an older version's +manual recipe, or one from a run killed in the instant between creating it and +recording it — the next start **replaces it**, along with any stale addresses +it carried. Two things are never touched: + +* an interface that is not a TUN, because the name colliding with somebody's + bridge is not a reason to destroy the bridge; +* a TUN that another process is holding open, because that is a working + overlay belonging to somebody else — most likely a second agent on this + host, which should be given a different `--wg-prefix`. + +Both of those refuse with an explanation rather than guessing. + +### Running without the capability + +`--interface attach` (or `auto`, which falls back on its own) opens an +interface prepared beforehand and needs **no privileges at all**. Ask the +agent what to run: ```bash tsunagi tun-setup --network lab --secret "$SECRET" @@ -174,55 +212,44 @@ sudo sysctl -qw net.ipv6.conf.tsunjwc6dcrtmo5.keep_addr_on_down=1 sudo ip -6 address add fd80:1210:f724:f620:d1bb:f982:3b6e:19bd/64 dev tsunjwc6dcrtmo5 nodad ``` -With IPv4 enabled, `tun-setup` adds an `ip address add` line for the overlay -IPv4 address too — but only once there is one to print. Unlike the IPv6 -address, the IPv4 address is not derived from the keys: it is allocated at run -time and signed (see [docs/sync-model.md](docs/sync-model.md)), so it exists -only after the agent has run once. `tun-setup` then reads it back out of -`state.sqlite`, which does not disturb a running agent, and includes it from -then on. Until then the first `tsunagi up` prints the exact command for the -address it was given. +`user ab` is the point: the interface is persistent and owned by that user, so +`tsunagi up` afterwards opens it with no privileges and no capabilities. -That IPv4 line needs no `keep_addr_on_down` and no `nodad`: Linux keeps IPv4 -addresses on an interface that loses carrier, and IPv4 has no duplicate -address detection to stall. Adding it once is enough. +The last two settings are what the managed path does not need. A persistent +TUN has **no carrier** until a process attaches to it; Linux flushes IPv6 +addresses from an interface that loses carrier unless `keep_addr_on_down` is +set, and duplicate address detection can never finish without carrier, so the +address would sit there tentative and unusable without `nodad`. An interface +the agent creates and holds open has carrier for its whole life, so neither +applies. IPv4 needs neither in either case: Linux keeps IPv4 addresses across +carrier loss and IPv4 has no duplicate address detection. + +With IPv4 enabled, `tun-setup` adds an `ip address add` line once there is an +address to print — it reads the signed claim back out of `state.sqlite`, which +does not disturb a running agent. The MTU is 1280 because that is the minimum IPv6 requires (RFC 8200). Linux disables IPv6 entirely on an interface below it — the per-device -`/proc/sys/net/ipv6` entries vanish and `ip -6 address add` fails with +`/proc/sys/net/ipv6` entries vanish and adding an address fails with `Invalid argument` — so a smaller MTU cannot work at all. The agent refuses one rather than letting it fail later. -The order and the last two lines are not decoration. A persistent TUN -interface has **no carrier** until a process attaches to it, and Linux flushes -IPv6 addresses from an interface that loses carrier unless -`keep_addr_on_down` is set — so an address added without it disappears before -the agent ever starts. `nodad` is needed for the same reason: duplicate -address detection cannot finish without a carrier, and the address would sit -there tentative and unusable. - -`user ab` is the point: the interface is persistent and owned by that user, so -`tsunagi up` afterwards opens it with **no privileges and no capabilities at -all**. The interface name and address are derived, so they are stable — the -setup survives restarts and only has to be redone if the network name, the -secret or this agent's WireGuard key changes. - -Other ways, and their trade-offs: +### Summary | approach | agent runs as | notes | |---|---|---| -| `tsunagi tun-setup` (above) | ordinary user, no capabilities | recommended | -| `sudo setcap cap_net_admin+ep ./tsunagi` | ordinary user, one capability | the agent can then create the interface itself, but **still cannot assign the IPv6 address** (see below), so the `ip -6 address add` line is needed anyway. The capability is lost on every rebuild or copy. | -| systemd service | `User=`, `AmbientCapabilities=CAP_NET_ADMIN` | same caveat about the address | -| plain `sudo tsunagi up` | root | everything works, nothing is isolated | +| `setcap cap_net_admin+p` | ordinary user, one capability | recommended on Linux: nothing to prepare, nothing left behind. Lost on every rebuild or copy of the binary. | +| systemd service | `User=`, `AmbientCapabilities=CAP_NET_ADMIN` | the same, for an installed service | +| `tsunagi tun-setup` then `--interface attach` | ordinary user, no capabilities | one privileged setup per host; needed on macOS and Windows, where no provisioner is implemented yet | +| `sudo tsunagi up` | root | everything works, nothing is isolated | +| `--no-tun` | ordinary user, no capabilities | tunnels run and handshake, traffic never reaches the OS | -**Known limitation.** The agent cannot assign the IPv6 overlay address itself: -the `tun` crate sets addresses through an IPv4-only ioctl, so an IPv6 address -has to come from `ip -6 address add` or an equivalent. Rather than start with -an interface that could never receive anything, the agent checks for the -address in `/proc/net/if_inet6` and refuses with the exact command to run. -Doing it in-process would mean talking netlink directly, which is possible but -not implemented. +**Not implemented yet.** macOS and Windows have no provisioner: both need +real platform work — `utun` and `SystemConfiguration` on one, the IP Helper +API and a Wintun adapter on the other. On those the agent says so and falls +back to attaching to a prepared interface. The decision logic that says *what* +to change is shared and tested on every platform; only the execution is +per-platform. ## Checks diff --git a/docs/wireguard.md b/docs/wireguard.md index fa99dfa..8f21026 100644 --- a/docs/wireguard.md +++ b/docs/wireguard.md @@ -267,22 +267,85 @@ async fn main() -> Result<()> { * **Userspace costs CPU.** Kernel WireGuard is faster. A kernel backend could return behind the same boundary, but it would give up transport-provided NAT traversal unless paired with a local proxy. -* **A persistent TUN interface needs `keep_addr_on_down`.** Without a process - attached it has no carrier, and Linux then flushes its IPv6 addresses. The - setup printed by `tsunagi tun-setup` sets it; the agent checks the address is - present *and usable* — not tentative, not DAD-failed — before attaching, and - reports what it actually found. This applies to IPv6 only: IPv4 addresses - survive carrier loss, so the overlay IPv4 address is added once and stays. -* **The overlay IPv4 address is not derivable, so `tun-setup` cannot print it - on a fresh state directory.** It is allocated and signed at run time, so the - first `tsunagi up` is what names it; afterwards `tun-setup` reads it back - from `state.sqlite` — a read that takes no directory lock and so does not - disturb a running agent — and includes the `ip address add` line. -* **The agent cannot assign the overlay address itself.** The `tun` crate sets - addresses through an IPv4-only ioctl, so the IPv6 overlay address must come - from `ip -6 address add` or an equivalent. The agent verifies the address is - present, via `/proc/net/if_inet6`, and refuses with the exact command rather - than running an interface that could never receive anything. Doing it - in-process would mean speaking netlink, which is not implemented. +* **The interface is managed, not prepared.** On Linux the agent creates the + TUN interface and configures it over netlink, in process. See + *Provisioning* below. +* **No provisioner exists for macOS or Windows yet.** There the agent attaches + to an interface prepared by hand and says so. * **The system interface path is not exercised by the default suite**, because it needs privileges. Everything else about the data plane is. + + +## Provisioning the interface + +Everything the printed `ip` recipe used to do happens in process now, over +netlink. The shape of it is a reconciliation rather than a sequence of +commands: the agent is handed a plan — name, MTU, the addresses the interface +should carry and no others — observes what is actually on the host, and +applies the difference. Running it twice changes nothing the second time. + +The decision of *what* to change is +`dataplane::wireguard::provision::plan_changes`: pure, platform-independent +and unit-tested on every platform. Only the execution is behind +`InterfaceProvisioner`, which has three implementations — netlink on Linux, a +`MockProvisioner` over a pretend host for the tests, and one that refuses with +an explanation everywhere else. + +### Cleanup is the default, not an action + +The interface is created by opening `/dev/net/tun` and is **not** made +persistent, so the kernel destroys it when the last descriptor closes. A clean +shutdown, a panic, `SIGKILL` and a power cut all leave the same amount behind: +nothing. There is no path by which a dead agent leaves an interface, because +keeping one alive is what requires a live process. + +This also removes the two settings the manual recipe needed. `keep_addr_on_down` +existed only because an interface nobody held open lost carrier and had its +IPv6 addresses flushed; `nodad` only because duplicate address detection +cannot finish without carrier. An interface held open for its whole life has +carrier for its whole life. + +### Repairing what an older run left + +Two things can still be sitting on the name: an interface created persistent +by the old recipe, and — narrowly — one from a run killed between `TUNSETIFF` +and the agent recording it. Both are replaced, which discards their stale +addresses with them. + +The two refusals are the interesting part: + +* **A link that is not a TUN is never touched.** The name is derived from the + network id, so colliding with a real device is unlikely rather than + impossible, and deleting somebody's bridge is not a recoverable mistake. +* **A TUN another process holds open is never deleted.** Carrier is the + signal: a TUN has it exactly while something is attached. An attached one is + a working overlay, almost certainly a second agent on this host, and it is + told to use a different `--wg-prefix` instead. + +### Privilege + +`CAP_NET_ADMIN` is required and is kept out of the *effective* set except +around the netlink calls that need it. `setcap cap_net_admin+p` leaves it +permitted but not effective at exec, which is the resting state; the agent +raises it for a few milliseconds at startup and again when its address +allocation changes. + +Two facts shape how that is done. Capabilities on Linux are **per thread**, +and netlink checks the credentials of whichever thread calls `sendmsg` — +which, with an async client, is the connection task and not the caller. So +raising a capability around an `await` would be wrong in the way that works +until the scheduler moves the task. + +Therefore: all netlink work runs on one dedicated thread with a current-thread +runtime, where nothing is polled outside a `block_on`, and the capability is +raised immediately before that call and lowered immediately after. Opening the +TUN descriptor is the other privileged act; it is a synchronous call with no +`await` between the guard and the release, so it stays on its own thread by +construction. + +### What it cannot be told to do + +Nothing here takes a name, an address or a command from the network. The +interface name is derived from the network id, the addresses come from the +local plugin and the signed allocation records, and no external program is +executed at any point — there is no `ip`, no shell and no `PATH` involved. diff --git a/src/bin/tsunagi.rs b/src/bin/tsunagi.rs index 4c2b91f..104524e 100644 --- a/src/bin/tsunagi.rs +++ b/src/bin/tsunagi.rs @@ -224,6 +224,15 @@ struct UpArgs { #[arg(long)] no_tun: bool, + /// How the overlay interface is obtained. + /// + /// `managed` has the agent create and configure it itself, which needs + /// CAP_NET_ADMIN and cleans up on exit. `attach` opens an interface that + /// was prepared beforehand (see `tsunagi tun-setup`) and needs no + /// privileges. `auto` manages it when it can and attaches when it cannot. + #[arg(long, value_enum, default_value_t = InterfaceMode::Auto)] + interface: InterfaceMode, + /// Interface name prefix for the WireGuard data plane. #[arg(long, default_value = "tsun")] wg_prefix: String, @@ -533,16 +542,33 @@ async fn doctor(paths: PathArgs) -> Result<(), Box> { #[cfg(not(feature = "tun-device"))] println!(" interfaces not built in (enable the `tun-device` feature)"); - #[cfg(unix)] + #[cfg(feature = "tun-device")] { - // Creating a network interface needs CAP_NET_ADMIN, which in practice - // means root unless capabilities were granted explicitly. - let euid = std::fs::metadata("/proc/self").ok().map(|_| ()); - let _ = euid; - println!( - " privileges creating an interface needs CAP_NET_ADMIN; \ - use --no-tun to run without it" - ); + use tsunagi::dataplane::wireguard::{Privilege, probe_net_admin}; + match probe_net_admin() { + Privilege::Available => { + println!(" privileges CAP_NET_ADMIN held"); + println!( + " interface managed by the agent: created on start, \ + removed on exit" + ); + } + Privilege::Missing(reason) => { + println!(" privileges no CAP_NET_ADMIN ({reason})"); + println!(" interface must be prepared first; run `tsunagi tun-setup`"); + println!( + " to manage it {}", + Privilege::how_to_grant(&program_path()) + ); + } + Privilege::Unsupported => { + println!( + " privileges managing interfaces is not implemented on {} yet", + std::env::consts::OS + ); + println!(" interface must be prepared first; run `tsunagi tun-setup`"); + } + } } println!("\nlocal addresses"); @@ -556,6 +582,14 @@ async fn doctor(paths: PathArgs) -> Result<(), Box> { Ok(()) } +/// This program's path, for an instruction the user can paste. +fn program_path() -> String { + std::env::current_exe() + .ok() + .and_then(|path| path.to_str().map(str::to_string)) + .unwrap_or_else(|| "tsunagi".to_string()) +} + async fn netwatch_addresses() -> Vec { // Best effort; used for diagnostics only. let state = netwatch::interfaces::State::new().await; @@ -600,7 +634,7 @@ async fn up(args: UpArgs) -> Result<(), Box> { let tun_factory: Arc = if args.no_tun { Arc::new(MemoryTunFactory::new()) } else { - system_tun_factory()? + system_tun_factory(args.interface)? }; let mut wg = WireguardConfig::new(paths.state_dir.join("wireguard")) .with_interface_prefix(args.wg_prefix.clone()); @@ -832,14 +866,69 @@ async fn stop_signal() -> &'static str { } } +/// How the overlay interface is obtained. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +enum InterfaceMode { + /// Manage it when possible, attach to a prepared one otherwise. + Auto, + /// Create and configure it in process. Needs CAP_NET_ADMIN. + Managed, + /// Open an interface prepared beforehand. Needs no privileges. + Attach, +} + +/// Builds the interface factory for the chosen mode. +/// +/// The managed path is preferred because it is the one that cleans up after +/// itself: the interface is tied to an open file descriptor, so it goes away +/// when the agent does, however the agent goes away. #[cfg(feature = "tun-device")] -fn system_tun_factory() -> Result, Box> { +fn system_tun_factory( + mode: InterfaceMode, +) -> Result, Box> { use tsunagi::dataplane::wireguard::SystemTunFactory; - Ok(Arc::new(SystemTunFactory::new())) + + if mode == InterfaceMode::Attach { + return Ok(Arc::new(SystemTunFactory::new())); + } + + match managed_tun_factory() { + Ok(factory) => Ok(factory), + Err(err) if mode == InterfaceMode::Managed => Err(err), + Err(err) => { + tracing::warn!( + "{err} Falling back to attaching to a prepared interface; \ + `tsunagi tun-setup` prints how to make one." + ); + Ok(Arc::new(SystemTunFactory::new())) + } + } +} + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +fn managed_tun_factory() -> Result, Box> { + use tsunagi::dataplane::wireguard::{ManagedTunFactory, NetlinkProvisioner}; + let provisioner = NetlinkProvisioner::new()?; + Ok(Arc::new(ManagedTunFactory::new(Arc::new(provisioner)))) +} + +/// There is no provisioner for this platform yet. +/// +/// Refused here rather than at the first packet, so `auto` falls back to +/// attaching and `--interface managed` says plainly why it cannot. +#[cfg(all(feature = "tun-device", not(target_os = "linux")))] +fn managed_tun_factory() -> Result, Box> { + Err(format!( + "managing the overlay interface is not implemented on {} yet.", + std::env::consts::OS + ) + .into()) } #[cfg(not(feature = "tun-device"))] -fn system_tun_factory() -> Result, Box> { +fn system_tun_factory( + _mode: InterfaceMode, +) -> Result, Box> { Err("this build has no interface support; rebuild with the `tun-device` feature or pass --no-tun".into()) } diff --git a/src/dataplane/wireguard/mod.rs b/src/dataplane/wireguard/mod.rs index eba2c32..56e69f0 100644 --- a/src/dataplane/wireguard/mod.rs +++ b/src/dataplane/wireguard/mod.rs @@ -34,9 +34,11 @@ //! //! The only part that needs privileges is the packet interface. With //! [`tun::MemoryTunFactory`] the whole data plane — handshake, encryption, -//! routing, address ownership — runs and is tested with no privileges at all; -//! `SystemTunFactory` swaps in a real interface when you want traffic to -//! reach the operating system. +//! routing, address ownership — runs and is tested with no privileges at all. +//! For real traffic there are two ways in: [`provision::ManagedTunFactory`], +//! where the agent creates and configures the interface itself over netlink +//! and removes it again on exit, and `SystemTunFactory`, which attaches to an +//! interface somebody else prepared and needs no privileges. //! //! See `docs/wireguard.md` for the full picture. @@ -47,6 +49,7 @@ pub mod keys; pub mod overlay; pub mod packet; pub mod plugin; +pub mod provision; pub mod store; pub mod tun; @@ -61,9 +64,17 @@ pub use plugin::{ DEFAULT_MTU, MIN_MTU, NetworkOverview, PeerOverview, WIREGUARD_OVERHEAD, WIREGUARD_PROTOCOL, WireguardConfig, WireguardPlugin, }; +pub use provision::{ + Changes, InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, ManagedTunFactory, + MockHost, MockProvisioner, Privilege, Provisioned, UnsupportedProvisioner, plan_changes, + probe_net_admin, +}; pub use store::WgKeyStore; pub use tun::{MemoryTun, MemoryTunFactory, TunDevice, TunFactory, TunRequest, address_is_local}; +#[cfg(all(feature = "tun-device", target_os = "linux"))] +pub use provision::NetlinkProvisioner; + #[cfg(feature = "tun-device")] pub use tun::{ Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6, diff --git a/src/dataplane/wireguard/plugin.rs b/src/dataplane/wireguard/plugin.rs index e52a045..1b76e66 100644 --- a/src/dataplane/wireguard/plugin.rs +++ b/src/dataplane/wireguard/plugin.rs @@ -211,6 +211,12 @@ struct NetworkState { ipv4_range: Option, /// The address last reported as missing, so it is said once, not forever. reported_missing_v4: Option, + /// What was last applied to the host interface. + /// + /// The overlay IPv4 address is allocated at run time and can change while + /// the agent runs, so the interface has to be brought back in line + /// without being recreated — recreating it would drop every tunnel. + applied: Option, } #[derive(Debug, Default)] @@ -444,6 +450,7 @@ impl Worker { allocations: HashMap::new(), ipv4_range: None, reported_missing_v4: None, + applied: None, }); } @@ -456,29 +463,34 @@ impl Worker { Ok(true) } + /// What the host interface for a network should look like. + fn desired_request(&self, state: &NetworkState, network: NetworkId) -> TunRequest { + let own_range = state.ipv4_range; + TunRequest { + name: state.interface.clone(), + address: overlay_address(network, &state.key.public()), + prefix_len: OVERLAY_PREFIX_LEN, + address_v4: state.allocations.get(&self.local_id()).copied(), + prefix_len_v4: own_range.map_or(0, |range| range.prefix_len), + mtu: self.config.mtu, + } + } + /// Creates the packet interface and starts the WireGuard device. async fn ensure_device(&self, network: NetworkId) -> Result<(), PluginError> { - let (name, key, own_v4, own_range) = { + let (request, key, own_range) = { let shared = self.lock_shared(); match shared.networks.get(&network) { Some(state) if state.device.is_none() => ( - state.interface.clone(), + self.desired_request(state, network), state.key.clone(), - state.allocations.get(&self.local_id()).copied(), state.ipv4_range, ), _ => return Ok(()), } }; - let request = TunRequest { - name: name.clone(), - address: overlay_address(network, &key.public()), - prefix_len: OVERLAY_PREFIX_LEN, - address_v4: own_v4, - prefix_len_v4: own_range.map_or(0, |range| range.prefix_len), - mtu: self.config.mtu, - }; + let applied = request.clone(); let tun = self.tun_factory.create(request).await?; let device = Arc::new(WireguardDevice::start(network, key, tun, own_range)); @@ -486,10 +498,44 @@ impl Worker { if let Some(state) = shared.networks.get_mut(&network) { state.interface = device.interface().to_string(); state.device = Some(device); + state.applied = Some(applied); } Ok(()) } + /// Brings a live interface back in line after the overlay changed its + /// mind about this agent's address. + async fn ensure_addresses(&self, network: NetworkId) { + let wanted = { + let shared = self.lock_shared(); + match shared.networks.get(&network) { + Some(state) if state.device.is_some() => { + let wanted = self.desired_request(state, network); + if state.applied.as_ref() == Some(&wanted) { + return; + } + wanted + } + _ => return, + } + }; + + match self.tun_factory.reconfigure(wanted.clone()).await { + Ok(()) => { + let mut shared = self.lock_shared(); + if let Some(state) = shared.networks.get_mut(&network) { + state.applied = Some(wanted); + } + } + Err(err) => { + // Not fatal: the tunnels keep running on the addresses that + // are there, and the next reconciliation tries again. + tracing::warn!(%err, "cannot update the overlay interface addresses"); + self.report(network, err); + } + } + } + /// Brings the running tunnels in line with what is known. /// /// A peer gets a tunnel once both halves have arrived: its announcement, @@ -601,10 +647,17 @@ impl Worker { } /// Removes a network's interface and tunnels, keeping its key. - fn teardown(&self, network: NetworkId) { + async fn teardown(&self, network: NetworkId) { // Dropping the state drops the device, which stops its tasks and - // closes the packet interface. - self.lock_shared().networks.remove(&network); + // closes the packet interface. Closing it is already enough for the + // kernel to remove an interface this agent created; the explicit + // destroy makes that immediate and definite rather than dependent on + // the last reader letting go. + let removed = self.lock_shared().networks.remove(&network); + if let Some(state) = removed { + drop(state.device); + self.tun_factory.destroy(&state.interface).await; + } } fn known_networks(&self) -> Vec { @@ -649,12 +702,12 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { } Command::Teardown(network) => { pending.remove(&network); - worker.teardown(network); + worker.teardown(network).await; continue; } Command::Stop(reply) => { for network in worker.known_networks() { - worker.teardown(network); + worker.teardown(network).await; } let _ = reply.send(()); return; @@ -670,6 +723,7 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { }, if wait_until.is_some() => { deadline = None; for network in std::mem::take(&mut pending) { + worker.ensure_addresses(network).await; worker.sync(network); } } @@ -680,6 +734,7 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { if let Err(err) = worker.ensure_device(network).await { tracing::debug!(%err, "packet interface still unavailable"); } + worker.ensure_addresses(network).await; worker.sync(network); } } diff --git a/src/dataplane/wireguard/provision/factory.rs b/src/dataplane/wireguard/provision/factory.rs new file mode 100644 index 0000000..4fd1fc9 --- /dev/null +++ b/src/dataplane/wireguard/provision/factory.rs @@ -0,0 +1,201 @@ +//! The adapter between the plugin's view of a packet interface and the +//! host-management view. +//! +//! The plugin asks a [`TunFactory`] for a device and knows nothing else. This +//! factory answers by reconciling the host — creating the interface, fixing +//! up whatever an earlier run left behind, assigning the addresses — and +//! handing back the device that came out of it. + +use std::sync::Arc; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::super::config::Cidr; +use super::super::tun::{TunDevice, TunFactory, TunRequest}; +use super::{InterfacePlan, InterfaceProvisioner}; + +/// Turns a [`TunRequest`] into the plan for a host interface. +fn plan_for(request: &TunRequest) -> Result { + let mut addresses = vec![Cidr::new(request.address.into(), request.prefix_len)?]; + if let Some(address) = request.address_v4 { + addresses.push(Cidr::new(address.into(), request.prefix_len_v4)?); + } + Ok(InterfacePlan::new( + request.name.clone(), + request.mtu, + addresses, + )) +} + +/// A [`TunFactory`] backed by an [`InterfaceProvisioner`]. +#[derive(Debug)] +pub struct ManagedTunFactory { + provisioner: Arc, +} + +impl ManagedTunFactory { + /// Wraps a provisioner. + pub fn new(provisioner: Arc) -> Self { + Self { provisioner } + } + + /// The provisioner underneath. + pub fn provisioner(&self) -> &Arc { + &self.provisioner + } +} + +impl TunFactory for ManagedTunFactory { + fn name(&self) -> &str { + self.provisioner.name() + } + + fn create<'a>( + &'a self, + request: TunRequest, + ) -> BoxFuture<'a, Result, PluginError>> { + Box::pin(async move { + let plan = plan_for(&request)?; + let provisioned = self.provisioner.reconcile(&plan).await?; + tracing::info!( + interface = %plan.name, + changes = %provisioned.changes.summary(), + "overlay interface reconciled" + ); + provisioned.device.ok_or_else(|| { + // Reaching here would mean the interface already existed and + // was held open by us, which cannot be true on the path that + // creates a device. + PluginError::Other(format!( + "interface `{}` was reconciled but no device came back", + plan.name + )) + }) + }) + } + + fn reconfigure<'a>(&'a self, request: TunRequest) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + let plan = plan_for(&request)?; + let provisioned = self.provisioner.reconcile(&plan).await?; + if !provisioned.changes.is_empty() { + tracing::info!( + interface = %plan.name, + changes = %provisioned.changes.summary(), + "overlay interface updated" + ); + } + Ok(()) + }) + } + + fn destroy<'a>(&'a self, name: &'a str) -> BoxFuture<'a, ()> { + Box::pin(async move { + match self.provisioner.remove(name).await { + Ok(()) => tracing::info!(interface = %name, "overlay interface removed"), + Err(err) => { + tracing::warn!(interface = %name, %err, "cannot remove the overlay interface") + } + } + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use std::net::{Ipv4Addr, Ipv6Addr}; + + use super::super::{LinkKind, MockHost, MockProvisioner}; + use super::*; + + fn request(v4: Option) -> TunRequest { + TunRequest { + name: "tsunfactory".into(), + address: Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + prefix_len: 64, + address_v4: v4, + prefix_len_v4: 24, + mtu: 1280, + } + } + + #[tokio::test] + async fn creating_a_device_provisions_the_host_and_returns_it() { + let provisioner = Arc::new(MockProvisioner::default()); + let factory = ManagedTunFactory::new(provisioner.clone()); + + let device = factory + .create(request(Some(Ipv4Addr::new(10, 13, 37, 69)))) + .await + .unwrap(); + assert_eq!(device.name(), "tsunfactory"); + assert_eq!(device.mtu(), 1280); + + let state = provisioner.host().get("tsunfactory").unwrap(); + assert_eq!(state.kind, LinkKind::Tun); + assert_eq!(state.mtu, 1280); + assert_eq!(state.addresses.len(), 2, "both families are assigned"); + } + + #[tokio::test] + async fn a_reallocated_address_is_applied_to_the_live_interface() { + let provisioner = Arc::new(MockProvisioner::default()); + let factory = ManagedTunFactory::new(provisioner.clone()); + factory + .create(request(Some(Ipv4Addr::new(10, 13, 37, 69)))) + .await + .unwrap(); + + factory + .reconfigure(request(Some(Ipv4Addr::new(10, 13, 37, 70)))) + .await + .unwrap(); + + let state = provisioner.host().get("tsunfactory").unwrap(); + assert!(state.attached, "the interface was not recreated"); + let addresses: Vec = state + .addresses + .iter() + .map(|entry| entry.to_string()) + .collect(); + assert!( + addresses.contains(&"10.13.37.70/24".to_string()), + "{addresses:?}" + ); + assert!( + !addresses.contains(&"10.13.37.69/24".to_string()), + "{addresses:?}" + ); + } + + #[tokio::test] + async fn destroying_takes_the_interface_off_the_host() { + let provisioner = Arc::new(MockProvisioner::default()); + let factory = ManagedTunFactory::new(provisioner.clone()); + factory.create(request(None)).await.unwrap(); + + factory.destroy("tsunfactory").await; + assert!(provisioner.host().names().is_empty()); + } + + #[tokio::test] + async fn a_host_that_cannot_be_provisioned_fails_the_create() { + let host = MockHost::new(); + host.insert( + "tsunfactory", + super::super::InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: Vec::new(), + }, + ); + let factory = ManagedTunFactory::new(Arc::new(MockProvisioner::new(host))); + let err = factory.create(request(None)).await.unwrap_err(); + assert!(err.to_string().contains("bridge"), "{err}"); + } +} diff --git a/src/dataplane/wireguard/provision/linux.rs b/src/dataplane/wireguard/provision/linux.rs new file mode 100644 index 0000000..0748446 --- /dev/null +++ b/src/dataplane/wireguard/provision/linux.rs @@ -0,0 +1,605 @@ +//! Managing the overlay interface on Linux, through netlink. +//! +//! Everything the old printed recipe did — `ip tuntap add`, `ip link set`, +//! `ip address add` — happens here instead, in this process, over a netlink +//! socket. No `ip` binary is invoked, so nothing this module does can be +//! influenced by `PATH`, by a shell, or by anything a remote peer said. +//! +//! # The interface cleans up after itself +//! +//! The TUN interface is created by opening `/dev/net/tun` and is **not** +//! made persistent, so the kernel destroys it the moment the last file +//! descriptor closes. That covers the ordinary exit, a panic, a `SIGKILL` +//! and a power loss equally: there is no path by which a dead agent leaves an +//! interface behind, because keeping it alive is what needs an action, not +//! removing it. +//! +//! It also means the interface has carrier for its whole life, which removes +//! the two settings the manual recipe needed. `keep_addr_on_down` was only +//! needed because an interface nobody held open lost carrier and had its IPv6 +//! addresses flushed; `nodad` only because duplicate address detection cannot +//! finish without carrier. +//! +//! What can still be left behind is an interface from *before* this change — +//! one created persistent by the old recipe — or one from a run killed in the +//! window between `TUNSETIFF` and this module recording it. Those are found +//! at startup and replaced; see +//! [`plan_changes`](super::plan_changes) for the rules that decide it. +//! +//! # Threads +//! +//! Capabilities on Linux are per thread, and netlink checks the credentials +//! of whichever thread calls `sendmsg` — which, with an async netlink client, +//! is the connection task rather than the caller. Raising `CAP_NET_ADMIN` +//! around an `await` would therefore be both wrong and unsound in the "works +//! until the scheduler moves the task" sense. +//! +//! So all netlink work happens on one dedicated thread running a +//! current-thread runtime. Nothing is polled outside a `block_on`, the +//! capability is raised immediately before that call and lowered immediately +//! after, and the connection task lives and dies inside it. + +use std::net::IpAddr; +use std::sync::{Arc, Mutex, mpsc}; + +use futures_util::TryStreamExt; +// Through rtnetlink's own re-export, so the packet types can never drift out +// of step with the client that sends them. +use rtnetlink::packet_route::address::{AddressAttribute, AddressMessage}; +use rtnetlink::packet_route::link::{InfoKind, LinkAttribute, LinkFlags, LinkInfo, LinkMessage}; +use rtnetlink::{LinkMessageBuilder, LinkUnspec}; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::super::config::Cidr; +use super::super::tun::{TunDevice, TunRequest}; +use super::privilege::{NetAdmin, Privilege, probe_net_admin}; +use super::{ + InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned, plan_changes, +}; + +/// A request to the netlink thread. +enum Command { + /// What does this interface look like right now? + Observe(String, Reply), + /// Remove this interface. + Delete(String, Reply<()>), + /// Apply MTU, link state and addresses. + Configure(Box, Reply<()>), + /// Stop the thread. + Stop, +} + +type Reply = mpsc::Sender>; + +/// The configuration half of a reconciliation. +struct Configure { + name: String, + mtu: Option, + bring_up: bool, + add: Vec, + remove: Vec, +} + +/// Manages the overlay interface with netlink. +#[derive(Debug)] +pub struct NetlinkProvisioner { + commands: mpsc::Sender, + worker: Mutex>>, + /// Interfaces this process created, so they are adjusted rather than + /// replaced. See [`plan_changes`]. + ours: Mutex>, + /// Devices kept alive for as long as the interface should exist. Dropping + /// one is what removes the interface from the kernel. + held: Mutex)>>, +} + +impl NetlinkProvisioner { + /// Starts the netlink thread, after checking this process can use it. + pub fn new() -> Result { + match probe_net_admin() { + Privilege::Available => {} + Privilege::Missing(reason) => { + return Err(PluginError::Unavailable(format!( + "{reason}. {}", + Privilege::how_to_grant(¤t_program()) + ))); + } + Privilege::Unsupported => { + return Err(PluginError::Unavailable( + "interface management is not compiled in".to_string(), + )); + } + } + + let (commands, requests) = mpsc::channel(); + let worker = std::thread::Builder::new() + .name("tsunagi-netlink".to_string()) + .spawn(move || netlink_thread(requests)) + .map_err(|err| { + PluginError::Unavailable(format!("cannot start the netlink thread: {err}")) + })?; + + Ok(Self { + commands, + worker: Mutex::new(Some(worker)), + ours: Mutex::new(Vec::new()), + held: Mutex::new(Vec::new()), + }) + } + + fn call( + &self, + make: impl FnOnce(Reply) -> Command, + ) -> Result { + let (reply_tx, reply_rx) = mpsc::channel(); + self.commands + .send(make(reply_tx)) + .map_err(|_| PluginError::Unavailable("the netlink thread has stopped".to_string()))?; + reply_rx.recv().map_err(|_| { + PluginError::Unavailable("the netlink thread stopped mid-request".to_string()) + })? + } + + fn is_ours(&self, name: &str) -> bool { + lock(&self.ours).iter().any(|owned| owned == name) + } + + /// Opens the TUN interface, which is what creates it. + /// + /// Synchronous on purpose: the capability guard is raised and lowered + /// without an `await` in between, so it cannot outlive this thread. + fn create_device(&self, plan: &InterfacePlan) -> Result, PluginError> { + let request = TunRequest::bare(plan.name.clone(), plan.mtu); + let _guard = NetAdmin::acquire()?; + super::super::tun::open_tun(&request, false) + } +} + +impl Drop for NetlinkProvisioner { + fn drop(&mut self) { + let _ = self.commands.send(Command::Stop); + if let Some(worker) = lock(&self.worker).take() { + let _ = worker.join(); + } + } +} + +impl InterfaceProvisioner for NetlinkProvisioner { + fn name(&self) -> &str { + "netlink" + } + + fn reconcile<'a>( + &'a self, + plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let current = self.call(|reply| Command::Observe(plan.name.clone(), reply))?; + let changes = plan_changes(¤t, plan, self.is_ours(&plan.name))?; + + if changes.delete_link { + tracing::info!( + interface = %plan.name, + "removing an abandoned interface left by an earlier run" + ); + self.call(|reply| Command::Delete(plan.name.clone(), reply))?; + } + + let device = if changes.create_link { + let device = self.create_device(plan)?; + lock(&self.ours).push(plan.name.clone()); + lock(&self.held).push((plan.name.clone(), Arc::clone(&device))); + Some(device) + } else { + None + }; + + if changes.set_mtu.is_some() + || changes.bring_up + || !changes.add.is_empty() + || !changes.remove.is_empty() + { + let configure = Box::new(Configure { + name: plan.name.clone(), + mtu: changes.set_mtu, + bring_up: changes.bring_up, + add: changes.add.clone(), + remove: changes.remove.clone(), + }); + // A failure here leaves an interface that exists but cannot + // carry traffic, which is worse than none at all, so it is + // taken back down rather than left as a trap. + if let Err(err) = self.call(move |reply| Command::Configure(configure, reply)) { + if changes.create_link { + lock(&self.held).retain(|(held, _)| held != &plan.name); + lock(&self.ours).retain(|owned| owned != &plan.name); + } + return Err(err); + } + } + + Ok(Provisioned { changes, device }) + }) + } + + fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + // Dropping the device is what removes the interface; the explicit + // delete is only so it is gone by the time this returns rather + // than whenever the last reader lets go. + lock(&self.held).retain(|(held, _)| held != name); + lock(&self.ours).retain(|owned| owned != name); + self.call(|reply| Command::Delete(name.to_string(), reply)) + }) + } +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn current_program() -> String { + std::env::current_exe() + .ok() + .and_then(|path| path.to_str().map(str::to_string)) + .unwrap_or_else(|| "tsunagi".to_string()) +} + +/// The netlink thread. +/// +/// It owns a current-thread runtime, so nothing is polled except inside the +/// `block_on` below — which is what makes the capability window exact. +fn netlink_thread(requests: mpsc::Receiver) { + // A binary granted `cap_net_admin+ep` starts with the capability + // effective. Lower it immediately so that even this thread only has it + // during the calls that need it. + NetAdmin::lower(); + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(err) => { + // Answer every request with the reason rather than hanging. + while let Ok(command) = requests.recv() { + let message = format!("the netlink thread has no runtime: {err}"); + match command { + Command::Observe(_, reply) => { + let _ = reply.send(Err(PluginError::Unavailable(message))); + } + Command::Delete(_, reply) | Command::Configure(_, reply) => { + let _ = reply.send(Err(PluginError::Unavailable(message))); + } + Command::Stop => return, + } + } + return; + } + }; + + while let Ok(command) = requests.recv() { + match command { + Command::Stop => return, + // Reading the interface list needs no privilege at all. + Command::Observe(name, reply) => { + let _ = reply.send(runtime.block_on(observe(&name))); + } + Command::Delete(name, reply) => { + let result = NetAdmin::acquire().and_then(|guard| { + let result = runtime.block_on(delete_link(&name)); + drop(guard); + result + }); + let _ = reply.send(result); + } + Command::Configure(configure, reply) => { + let result = NetAdmin::acquire().and_then(|guard| { + let result = runtime.block_on(configure_link(&configure)); + drop(guard); + result + }); + let _ = reply.send(result); + } + } + } +} + +/// Opens a netlink connection and runs one unit of work over it. +async fn with_netlink(work: impl FnOnce(rtnetlink::Handle) -> F) -> Result +where + F: std::future::Future>, +{ + let (connection, handle, _messages) = rtnetlink::new_connection() + .map_err(|err| PluginError::Unavailable(format!("cannot open netlink: {err}")))?; + let pump = tokio::spawn(connection); + let result = work(handle).await; + pump.abort(); + result +} + +/// `fe80::/10`, which the kernel assigns on its own. +fn is_link_local(addr: IpAddr) -> bool { + match addr { + IpAddr::V4(addr) => addr.is_link_local(), + IpAddr::V6(addr) => (addr.segments()[0] & 0xffc0) == 0xfe80, + } +} + +/// Whether a name belongs to a TUN interface, from sysfs. +/// +/// A fallback for the case where the kernel does not report `IFLA_LINKINFO` +/// for the link. Reading sysfs needs no privileges. +fn is_tun_in_sysfs(name: &str) -> bool { + std::path::Path::new(&format!("/sys/class/net/{name}/tun_flags")).exists() +} + +fn link_kind(message: &LinkMessage, name: &str) -> LinkKind { + for attribute in &message.attributes { + if let LinkAttribute::LinkInfo(infos) = attribute { + for info in infos { + if let LinkInfo::Kind(kind) = info { + return match kind { + InfoKind::Tun => LinkKind::Tun, + other => LinkKind::Foreign(format!("{other:?}").to_lowercase()), + }; + } + } + } + } + if is_tun_in_sysfs(name) { + LinkKind::Tun + } else { + // No `IFLA_LINKINFO` and no `tun_flags`: a plain device such as an + // ethernet port. Unknown rather than ours, so it is left alone. + LinkKind::Foreign("non-tun".to_string()) + } +} + +async fn observe(name: &str) -> Result { + with_netlink(|handle| async move { + let mut links = handle.link().get().match_name(name.to_string()).execute(); + let message = match links.try_next().await { + Ok(Some(message)) => message, + Ok(None) => return Ok(InterfaceState::absent()), + Err(err) => { + // "No such device" is the expected answer on a clean host, so + // it is not an error; anything else is. + if !std::path::Path::new(&format!("/sys/class/net/{name}")).exists() { + return Ok(InterfaceState::absent()); + } + return Err(PluginError::Unavailable(format!( + "cannot read interface `{name}`: {err}" + ))); + } + }; + + let index = message.header.index; + let flags = message.header.flags; + let mtu = message + .attributes + .iter() + .find_map(|attribute| match attribute { + LinkAttribute::Mtu(mtu) => Some(*mtu), + _ => None, + }) + .unwrap_or(0); + + let mut addresses = Vec::new(); + let mut stream = handle + .address() + .get() + .set_link_index_filter(index) + .execute(); + while let Some(message) = stream.try_next().await.map_err(|err| { + PluginError::Unavailable(format!("cannot read the addresses of `{name}`: {err}")) + })? { + if let Some(cidr) = address_of(&message) + && !is_link_local(cidr.addr) + { + addresses.push(cidr); + } + } + addresses.sort(); + + Ok(InterfaceState { + kind: link_kind(&message, name), + // `IFF_LOWER_UP` is carrier, and a TUN has carrier exactly while + // a process holds it open. + attached: flags.contains(LinkFlags::LowerUp), + up: flags.contains(LinkFlags::Up), + mtu, + addresses, + }) + }) + .await +} + +/// The address a message carries, preferring `IFA_LOCAL`. +/// +/// For a point-to-point interface `IFA_ADDRESS` is the *peer* address, so +/// taking it would compare the wrong thing. +fn address_of(message: &AddressMessage) -> Option { + let mut address = None; + for attribute in &message.attributes { + match attribute { + AddressAttribute::Local(addr) => return cidr(*addr, message.header.prefix_len), + AddressAttribute::Address(addr) => address = Some(*addr), + _ => {} + } + } + address.and_then(|addr| cidr(addr, message.header.prefix_len)) +} + +fn cidr(addr: IpAddr, prefix_len: u8) -> Option { + Cidr::new(addr, prefix_len).ok() +} + +async fn delete_link(name: &str) -> Result<(), PluginError> { + with_netlink(|handle| async move { + let mut links = handle.link().get().match_name(name.to_string()).execute(); + let index = match links.try_next().await { + Ok(Some(message)) => message.header.index, + // Already gone, which is the outcome asked for. + Ok(None) | Err(_) => return Ok(()), + }; + handle.link().del(index).execute().await.map_err(|err| { + PluginError::Unavailable(format!("cannot remove interface `{name}`: {err}")) + }) + }) + .await +} + +async fn configure_link(configure: &Configure) -> Result<(), PluginError> { + let name = configure.name.as_str(); + with_netlink(|handle| async move { + let mut links = handle.link().get().match_name(name.to_string()).execute(); + let index = links + .try_next() + .await + .ok() + .flatten() + .map(|message| message.header.index) + .ok_or_else(|| { + PluginError::Unavailable(format!( + "interface `{name}` disappeared before it could be configured" + )) + })?; + + if configure.mtu.is_some() || configure.bring_up { + let mut builder = LinkMessageBuilder::::new().index(index); + if let Some(mtu) = configure.mtu { + builder = builder.mtu(mtu); + } + if configure.bring_up { + builder = builder.up(); + } + handle + .link() + .set(builder.build()) + .execute() + .await + .map_err(|err| { + PluginError::Unavailable(format!("cannot configure interface `{name}`: {err}")) + })?; + } + + for cidr in &configure.remove { + // Delete the exact message the kernel holds rather than a + // reconstruction of it, so the family and flags always match. + let mut stream = handle + .address() + .get() + .set_link_index_filter(index) + .execute(); + let mut target: Option = None; + while let Ok(Some(message)) = stream.try_next().await { + if address_of(&message) == Some(*cidr) { + target = Some(message); + break; + } + } + drop(stream); + if let Some(message) = target { + handle + .address() + .del(message) + .execute() + .await + .map_err(|err| { + PluginError::Unavailable(format!( + "cannot remove {cidr} from interface `{name}`: {err}" + )) + })?; + } + } + + for cidr in &configure.add { + handle + .address() + .add(index, cidr.addr, cidr.prefix_len) + .execute() + .await + .map_err(|err| { + PluginError::Unavailable(format!( + "cannot add {cidr} to interface `{name}`: {err}" + )) + })?; + } + + Ok(()) + }) + .await +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[test] + fn link_local_addresses_are_not_ours_to_manage() { + // The kernel assigns these itself when the link comes up. Treating + // them as unplanned would make every reconciliation try to delete one. + assert!(is_link_local(IpAddr::V6(Ipv6Addr::new( + 0xfe80, 0, 0, 0, 0, 0, 0, 1 + )))); + assert!(is_link_local(IpAddr::V4(Ipv4Addr::new(169, 254, 1, 1)))); + assert!(!is_link_local(IpAddr::V6(Ipv6Addr::new( + 0xfd00, 0, 0, 0, 0, 0, 0, 1 + )))); + assert!(!is_link_local(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 1)))); + } + + #[tokio::test] + async fn observing_a_name_nothing_uses_reports_it_absent() { + let state = observe("tsunagi-no-such-interface").await.unwrap(); + assert_eq!(state.kind, LinkKind::Absent); + assert!(state.addresses.is_empty()); + } + + #[tokio::test] + async fn observing_the_loopback_interface_decodes_what_the_kernel_reports() { + // Reading the interface table needs no privileges, so this runs + // everywhere and checks the netlink decoding against a real kernel + // rather than against a fixture. + let state = observe("lo").await.unwrap(); + assert!( + matches!(state.kind, LinkKind::Foreign(_)), + "loopback is not a tun: {:?}", + state.kind + ); + assert!(state.up, "loopback is up"); + assert!(state.mtu >= 1280, "decoded an mtu: {}", state.mtu); + assert!( + state + .addresses + .iter() + .any(|cidr| cidr.addr == IpAddr::V4(Ipv4Addr::LOCALHOST)), + "127.0.0.1 is decoded: {:?}", + state.addresses + ); + assert!( + !state.addresses.iter().any(|cidr| is_link_local(cidr.addr)), + "link-local addresses are filtered out: {:?}", + state.addresses + ); + } + + #[test] + fn an_interface_that_is_not_a_tun_is_reported_foreign() { + let message = LinkMessage::default(); + // No `IFLA_LINKINFO` and a name with no `tun_flags` in sysfs. + assert!(matches!( + link_kind(&message, "definitely-not-an-interface"), + LinkKind::Foreign(_) + )); + } +} diff --git a/src/dataplane/wireguard/provision/mock.rs b/src/dataplane/wireguard/provision/mock.rs new file mode 100644 index 0000000..2a4b720 --- /dev/null +++ b/src/dataplane/wireguard/provision/mock.rs @@ -0,0 +1,336 @@ +//! An in-memory host, so provisioning is tested without touching this one. +//! +//! [`MockHost`] is a pretend `/sys/class/net`: a test can seed it with the +//! leftovers of a crashed run, or with somebody else's bridge, then check +//! what the provisioner did about it. It is also what the platforms that have +//! no provisioner yet are wired to in their own tests. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::super::config::Cidr; +use super::super::tun::{MemoryTun, MemoryTunFactory, TunFactory, TunRequest}; +use super::{ + Changes, InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned, + plan_changes, +}; + +/// A pretend host with interfaces on it. +#[derive(Debug, Clone, Default)] +pub struct MockHost { + links: Arc>>, +} + +impl MockHost { + /// An empty host. + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + match self.links.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + /// Puts an interface on the host. + pub fn insert(&self, name: impl Into, state: InterfaceState) { + self.lock().insert(name.into(), state); + } + + /// Seeds the leftovers of a run that died: a TUN nobody holds open, still + /// carrying whatever addresses it had. + pub fn insert_stale_tun(&self, name: impl Into, addresses: Vec) { + self.insert( + name, + InterfaceState { + kind: LinkKind::Tun, + attached: false, + up: true, + mtu: 1280, + addresses, + }, + ); + } + + /// The state of an interface, if it exists. + pub fn get(&self, name: &str) -> Option { + self.lock().get(name).cloned() + } + + /// The names currently on the host. + pub fn names(&self) -> Vec { + let mut names: Vec = self.lock().keys().cloned().collect(); + names.sort(); + names + } +} + +/// Applies plans to a [`MockHost`]. +#[derive(Debug, Clone)] +pub struct MockProvisioner { + host: MockHost, + ours: Arc>>, + /// The devices handed out, so a test can drive packets through them. + devices: MemoryTunFactory, + /// Set to fail every call, to exercise the error path. + failure: Option, +} + +impl Default for MockProvisioner { + fn default() -> Self { + Self::new(MockHost::new()) + } +} + +impl MockProvisioner { + /// A provisioner over a host. + pub fn new(host: MockHost) -> Self { + Self { + host, + ours: Arc::new(Mutex::new(Vec::new())), + devices: MemoryTunFactory::new(), + failure: None, + } + } + + /// A provisioner that refuses everything, with this reason. + pub fn failing(reason: impl Into) -> Self { + Self { + host: MockHost::new(), + ours: Arc::new(Mutex::new(Vec::new())), + devices: MemoryTunFactory::new(), + failure: Some(reason.into()), + } + } + + /// The host it applies to. + pub fn host(&self) -> &MockHost { + &self.host + } + + /// The device created for an interface name, if any. + pub fn device(&self, name: &str) -> Option> { + self.devices.device(name) + } + + fn owned(&self) -> std::sync::MutexGuard<'_, Vec> { + match self.ours.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + fn apply(&self, plan: &InterfacePlan) -> Result { + if let Some(reason) = &self.failure { + return Err(PluginError::Unavailable(reason.clone())); + } + + let ours = self.owned().iter().any(|name| name == &plan.name); + let current = self + .host + .get(&plan.name) + .unwrap_or_else(InterfaceState::absent); + let changes = plan_changes(¤t, plan, ours)?; + + let mut state = current; + if changes.delete_link { + self.host.lock().remove(&plan.name); + state = InterfaceState::absent(); + } + if changes.create_link { + state = InterfaceState { + kind: LinkKind::Tun, + // Creating it means holding it open, so it has carrier. + attached: true, + up: false, + mtu: 1500, + addresses: Vec::new(), + }; + self.owned().push(plan.name.clone()); + } + if let Some(mtu) = changes.set_mtu { + state.mtu = mtu; + } + if changes.bring_up { + state.up = true; + } + state + .addresses + .retain(|addr| !changes.remove.contains(addr)); + state.addresses.extend(changes.add.iter().copied()); + state.addresses.sort(); + state.addresses.dedup(); + self.host.insert(plan.name.clone(), state); + + Ok(changes) + } +} + +impl InterfaceProvisioner for MockProvisioner { + fn name(&self) -> &str { + "mock" + } + + fn reconcile<'a>( + &'a self, + plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let changes = self.apply(plan)?; + let device = if changes.create_link { + Some( + self.devices + .create(TunRequest::bare(plan.name.clone(), plan.mtu)) + .await?, + ) + } else { + None + }; + Ok(Provisioned { changes, device }) + }) + } + + fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + self.host.lock().remove(name); + self.owned().retain(|owned| owned != name); + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + fn v6() -> Cidr { + Cidr { + addr: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1)), + prefix_len: 64, + } + } + + fn v4(last: u8) -> Cidr { + Cidr { + addr: IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)), + prefix_len: 24, + } + } + + fn plan(addresses: Vec) -> InterfacePlan { + InterfacePlan::new("tsunmock", 1280, addresses) + } + + #[tokio::test] + async fn reconciling_an_empty_host_creates_a_configured_interface() { + let provisioner = MockProvisioner::default(); + let plan = plan(vec![v6(), v4(69)]); + let changes = provisioner.reconcile(&plan).await.unwrap().changes; + assert!(changes.create_link); + + let state = provisioner.host().get("tsunmock").unwrap(); + assert_eq!(state.kind, LinkKind::Tun); + assert!(state.up); + assert_eq!(state.mtu, 1280); + assert_eq!(state.addresses, vec![v4(69), v6()]); + } + + #[tokio::test] + async fn reconciling_is_idempotent() { + let provisioner = MockProvisioner::default(); + let plan = plan(vec![v6(), v4(69)]); + provisioner.reconcile(&plan).await.unwrap(); + let second = provisioner.reconcile(&plan).await.unwrap().changes; + assert!(second.is_empty(), "{second:?}"); + } + + #[tokio::test] + async fn a_crashed_run_is_repaired_on_the_next_start() { + let host = MockHost::new(); + // What the previous run left: the interface, with the address it had + // been allocated back then. + host.insert_stale_tun("tsunmock", vec![v4(178)]); + let provisioner = MockProvisioner::new(host); + + let changes = provisioner + .reconcile(&plan(vec![v6(), v4(69)])) + .await + .unwrap() + .changes; + assert!(changes.delete_link && changes.create_link); + + let state = provisioner.host().get("tsunmock").unwrap(); + assert_eq!( + state.addresses, + vec![v4(69), v6()], + "the stale address is gone and the current one is there" + ); + assert!(state.attached, "the new interface is held open by us"); + } + + #[tokio::test] + async fn a_changed_allocation_is_applied_without_recreating_the_interface() { + let provisioner = MockProvisioner::default(); + provisioner + .reconcile(&plan(vec![v6(), v4(69)])) + .await + .unwrap(); + + // The overlay agreed on a different address for us while running. + let changes = provisioner + .reconcile(&plan(vec![v6(), v4(70)])) + .await + .unwrap() + .changes; + assert!( + !changes.delete_link && !changes.create_link, + "recreating would drop every tunnel" + ); + assert_eq!(changes.add, vec![v4(70)]); + assert_eq!(changes.remove, vec![v4(69)]); + assert_eq!( + provisioner.host().get("tsunmock").unwrap().addresses, + vec![v4(70), v6()] + ); + } + + #[tokio::test] + async fn removing_takes_the_interface_off_the_host_and_is_idempotent() { + let provisioner = MockProvisioner::default(); + provisioner.reconcile(&plan(vec![v6()])).await.unwrap(); + assert_eq!(provisioner.host().names(), vec!["tsunmock".to_string()]); + + provisioner.remove("tsunmock").await.unwrap(); + assert!(provisioner.host().names().is_empty()); + provisioner.remove("tsunmock").await.unwrap(); + } + + #[tokio::test] + async fn a_foreign_interface_makes_reconciling_fail_and_changes_nothing() { + let host = MockHost::new(); + host.insert( + "tsunmock", + InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: vec![v4(1)], + }, + ); + let provisioner = MockProvisioner::new(host); + assert!(provisioner.reconcile(&plan(vec![v6()])).await.is_err()); + + let state = provisioner.host().get("tsunmock").unwrap(); + assert_eq!(state.kind, LinkKind::Foreign("bridge".into())); + assert_eq!(state.addresses, vec![v4(1)], "left exactly as it was"); + } +} diff --git a/src/dataplane/wireguard/provision/mod.rs b/src/dataplane/wireguard/provision/mod.rs new file mode 100644 index 0000000..f5d93b9 --- /dev/null +++ b/src/dataplane/wireguard/provision/mod.rs @@ -0,0 +1,440 @@ +//! Bringing a host interface into the state the overlay needs. +//! +//! The agent used to print a list of `ip` commands and ask a human to run +//! them. That is fragile in exactly the way hand-held setup always is: the +//! interface does not survive a reboot, a changed address allocation needs +//! another manual round, and a crashed run leaves a half-configured interface +//! that the next run then trips over. +//! +//! So the agent does it itself, and the shape of that is a *reconciliation*: +//! it is handed an [`InterfacePlan`] describing what the interface should look +//! like, it observes what is actually there, and it applies the difference. +//! Running it twice changes nothing the second time, and running it after a +//! crash repairs whatever was left behind. +//! +//! # Why this is a trait +//! +//! Every platform does this differently — netlink on Linux, `SystemConfiguration` +//! on macOS, the Windows IP Helper API — while the *decision* of what to change +//! is the same everywhere. So the decision lives in [`plan_changes`], which is +//! pure and tested on every platform, and only the execution is behind +//! [`InterfaceProvisioner`]. +//! +//! Three implementations: +//! +//! * `NetlinkProvisioner` on Linux, which needs `CAP_NET_ADMIN`. +//! * [`MockProvisioner`], an in-memory host used by the tests. +//! * [`UnsupportedProvisioner`] elsewhere, which fails with an explanation +//! and a pointer at the manual route rather than pretending to work. +//! +//! # What it is not allowed to do +//! +//! Nothing here takes a name, an address or a command from the network. The +//! interface name is derived from the network id, the addresses come from the +//! local plugin, and an interface this agent did not create is never deleted +//! or reconfigured — see [`plan_changes`]. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::config::Cidr; +use super::tun::TunDevice; + +mod mock; +pub use mock::{MockHost, MockProvisioner}; + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +mod linux; +#[cfg(all(feature = "tun-device", target_os = "linux"))] +pub use linux::NetlinkProvisioner; + +mod privilege; +pub use privilege::{Privilege, probe_net_admin}; + +mod unsupported; +pub use unsupported::UnsupportedProvisioner; + +mod factory; +pub use factory::ManagedTunFactory; + +/// What an interface should look like. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InterfacePlan { + /// Interface name. Derived from the network id, never from the network. + pub name: String, + /// Interface MTU. + pub mtu: u32, + /// Every address the interface should carry, and no others. + pub addresses: Vec, +} + +impl InterfacePlan { + /// Builds a plan, normalising the address list. + pub fn new(name: impl Into, mtu: u32, addresses: Vec) -> Self { + let unique: BTreeSet = addresses.into_iter().collect(); + Self { + name: name.into(), + mtu, + addresses: unique.into_iter().collect(), + } + } +} + +/// What kind of link is sitting on a name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LinkKind { + /// Nothing is there. + Absent, + /// A TUN interface. + Tun, + /// Something else entirely — a bridge, a physical device, a VPN from + /// another program. Never ours to touch. + Foreign(String), +} + +/// What an interface currently looks like. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InterfaceState { + /// What kind of link holds the name. + pub kind: LinkKind, + /// Whether a process is attached to the TUN, which the kernel reports as + /// carrier. A leftover interface from a crashed run has none. + pub attached: bool, + /// Whether the link is administratively up. + pub up: bool, + /// The current MTU. + pub mtu: u32, + /// The addresses currently assigned. + pub addresses: Vec, +} + +impl InterfaceState { + /// The state of a name nothing is using. + pub fn absent() -> Self { + Self { + kind: LinkKind::Absent, + attached: false, + up: false, + mtu: 0, + addresses: Vec::new(), + } + } +} + +/// The steps that turn an [`InterfaceState`] into an [`InterfacePlan`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Changes { + /// Remove a stale interface left behind by an earlier run. + pub delete_link: bool, + /// Create the interface. + pub create_link: bool, + /// Set the MTU, when it is not already right. + pub set_mtu: Option, + /// Bring the link up. + pub bring_up: bool, + /// Addresses to add. + pub add: Vec, + /// Addresses to remove, because the plan no longer contains them. + pub remove: Vec, +} + +impl Changes { + /// Whether anything at all needs doing. + pub fn is_empty(&self) -> bool { + !self.delete_link + && !self.create_link + && self.set_mtu.is_none() + && !self.bring_up + && self.add.is_empty() + && self.remove.is_empty() + } + + /// A one-line summary for a log line. + pub fn summary(&self) -> String { + if self.is_empty() { + return "already as planned".to_string(); + } + let mut parts = Vec::new(); + if self.delete_link { + parts.push("remove a stale interface".to_string()); + } + if self.create_link { + parts.push("create the interface".to_string()); + } + if let Some(mtu) = self.set_mtu { + parts.push(format!("set mtu {mtu}")); + } + if self.bring_up { + parts.push("bring it up".to_string()); + } + for address in &self.add { + parts.push(format!("add {address}")); + } + for address in &self.remove { + parts.push(format!("remove {address}")); + } + parts.join(", ") + } +} + +/// Decides what to change, or refuses. +/// +/// `ours` says whether this process created the interface in its current run. +/// It is the whole reason this can be safe: an interface we made is adjusted +/// in place, and an interface we did not make is only ever *replaced* when it +/// is plainly abandoned. +/// +/// The refusals matter more than the changes: +/// +/// * A link that is not a TUN is never touched. Deriving the name from the +/// network id makes a collision with a real device unlikely, not impossible, +/// and destroying somebody's bridge because it happened to share a name is +/// not a recoverable mistake. +/// * A TUN with a process attached to it is never deleted. It is somebody +/// else's working interface — most likely another agent on this host — and +/// yanking it out from under them would break a running overlay. +/// +/// What is left is a TUN with nothing attached, which is precisely the +/// footprint of a run that died: those are removed and rebuilt. +pub fn plan_changes( + current: &InterfaceState, + plan: &InterfacePlan, + ours: bool, +) -> Result { + let mut changes = Changes::default(); + + match ¤t.kind { + LinkKind::Foreign(kind) => { + return Err(PluginError::Unavailable(format!( + "`{}` already exists and is a {kind} interface, not one of ours. \ + Refusing to touch it. Run with a different interface prefix.", + plan.name + ))); + } + LinkKind::Tun if !ours && current.attached => { + return Err(PluginError::Unavailable(format!( + "`{}` already exists and another process is attached to it. \ + That is most likely a second agent on this host in the same \ + network; give one of them a different interface prefix.", + plan.name + ))); + } + LinkKind::Tun if !ours => { + // Abandoned: a TUN with no carrier is one nothing holds open. It + // is either a leftover from a run that died or an interface made + // by the old manual recipe. Either way it is replaced, which also + // discards whatever stale addresses it carried. + changes.delete_link = true; + changes.create_link = true; + } + LinkKind::Tun => {} + LinkKind::Absent => changes.create_link = true, + } + + if changes.create_link { + // A fresh interface starts down, with the kernel default MTU and no + // addresses, so everything in the plan has to be applied. + changes.set_mtu = Some(plan.mtu); + changes.bring_up = true; + changes.add = plan.addresses.clone(); + return Ok(changes); + } + + if current.mtu != plan.mtu { + changes.set_mtu = Some(plan.mtu); + } + if !current.up { + changes.bring_up = true; + } + + let wanted: BTreeSet = plan.addresses.iter().copied().collect(); + let present: BTreeSet = current.addresses.iter().copied().collect(); + changes.add = wanted.difference(&present).copied().collect(); + changes.remove = present.difference(&wanted).copied().collect(); + + Ok(changes) +} + +/// The result of a reconciliation. +pub struct Provisioned { + /// What was changed to get here. + pub changes: Changes, + /// The packet interface, when this call is what created it. + /// + /// Creating the interface and configuring it are the same privileged act + /// and belong together, so the provisioner owns both. A reconciliation + /// that only adjusted an interface already in place returns `None`. + pub device: Option>, +} + +impl std::fmt::Debug for Provisioned { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Provisioned") + .field("changes", &self.changes) + .field("device", &self.device.as_ref().map(|device| device.name())) + .finish() + } +} + +/// Applies an [`InterfacePlan`] to the host. +/// +/// Implementations are expected to be idempotent: calling [`reconcile`] twice +/// with the same plan changes nothing the second time. +/// +/// [`reconcile`]: InterfaceProvisioner::reconcile +pub trait InterfaceProvisioner: Send + Sync + std::fmt::Debug + 'static { + /// A short name used in diagnostics. + fn name(&self) -> &str; + + /// Brings the interface in line with the plan, and reports what it did. + fn reconcile<'a>( + &'a self, + plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result>; + + /// Removes an interface this provisioner created. + /// + /// Removing one that is already gone succeeds: this runs on the shutdown + /// path, where the interface having vanished is the desired outcome. + fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), PluginError>>; +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + fn v6(last: u16) -> Cidr { + Cidr { + addr: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, last)), + prefix_len: 64, + } + } + + fn v4(last: u8) -> Cidr { + Cidr { + addr: IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)), + prefix_len: 24, + } + } + + fn plan() -> InterfacePlan { + InterfacePlan::new("tsuntest", 1280, vec![v6(1), v4(69)]) + } + + #[test] + fn an_absent_interface_is_created_and_fully_configured() { + let changes = plan_changes(&InterfaceState::absent(), &plan(), false).unwrap(); + assert!(changes.create_link); + assert!(!changes.delete_link); + assert_eq!(changes.set_mtu, Some(1280)); + assert!(changes.bring_up); + assert_eq!(changes.add, vec![v4(69), v6(1)]); + assert!(changes.remove.is_empty()); + } + + #[test] + fn an_abandoned_tun_from_a_crashed_run_is_replaced() { + // The footprint of a run that died: the interface is still there, it + // still has its old addresses, and nothing holds it open. + let current = InterfaceState { + kind: LinkKind::Tun, + attached: false, + up: true, + mtu: 1280, + addresses: vec![v4(178)], + }; + let changes = plan_changes(¤t, &plan(), false).unwrap(); + assert!(changes.delete_link, "the stale interface goes"); + assert!(changes.create_link); + // Replacing it discards the stale address, so it need not be removed + // one by one. + assert_eq!(changes.add, vec![v4(69), v6(1)]); + assert!(changes.remove.is_empty()); + } + + #[test] + fn a_foreign_interface_is_never_touched() { + let current = InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: vec![v4(1)], + }; + let err = plan_changes(¤t, &plan(), false).unwrap_err(); + let message = err.to_string(); + assert!(message.contains("bridge"), "{message}"); + assert!(message.contains("Refusing to touch it"), "{message}"); + } + + #[test] + fn a_tun_another_process_is_using_is_never_deleted() { + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: true, + mtu: 1280, + addresses: vec![v6(1)], + }; + let err = plan_changes(¤t, &plan(), false).unwrap_err(); + assert!(err.to_string().contains("another process"), "{err}"); + } + + #[test] + fn our_own_interface_is_adjusted_in_place() { + // The live case: our address allocation changed while running. The + // interface must not be recreated, or every tunnel on it would drop. + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: true, + mtu: 1280, + addresses: vec![v6(1), v4(178)], + }; + let changes = plan_changes(¤t, &plan(), true).unwrap(); + assert!(!changes.delete_link && !changes.create_link); + assert_eq!(changes.add, vec![v4(69)]); + assert_eq!(changes.remove, vec![v4(178)]); + } + + #[test] + fn reconciling_an_interface_that_already_matches_changes_nothing() { + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: true, + mtu: 1280, + addresses: vec![v4(69), v6(1)], + }; + let changes = plan_changes(¤t, &plan(), true).unwrap(); + assert!(changes.is_empty(), "{changes:?}"); + assert_eq!(changes.summary(), "already as planned"); + } + + #[test] + fn a_wrong_mtu_or_a_down_link_is_corrected() { + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: false, + mtu: 1500, + addresses: vec![v4(69), v6(1)], + }; + let changes = plan_changes(¤t, &plan(), true).unwrap(); + assert_eq!(changes.set_mtu, Some(1280)); + assert!(changes.bring_up); + assert!(changes.add.is_empty() && changes.remove.is_empty()); + } + + #[test] + fn a_plan_deduplicates_and_orders_its_addresses() { + let plan = InterfacePlan::new("x", 1280, vec![v6(1), v4(69), v6(1)]); + assert_eq!(plan.addresses, vec![v4(69), v6(1)]); + } +} diff --git a/src/dataplane/wireguard/provision/privilege.rs b/src/dataplane/wireguard/provision/privilege.rs new file mode 100644 index 0000000..882ada2 --- /dev/null +++ b/src/dataplane/wireguard/provision/privilege.rs @@ -0,0 +1,169 @@ +//! Holding `CAP_NET_ADMIN` for as short a time as possible. +//! +//! Creating a TUN interface and assigning addresses to it needs +//! `CAP_NET_ADMIN`, and there is no way around that on Linux. What *is* in +//! our control is how long the process can actually use it. +//! +//! Linux splits capabilities into sets. The *permitted* set is what a process +//! may use; the *effective* set is what it may use **right now**. A process +//! can lower a capability out of effective and raise it back later, but it +//! can never add to permitted. So the agent keeps `CAP_NET_ADMIN` out of the +//! effective set and raises it only around the handful of netlink calls that +//! need it, which is a few milliseconds at startup and again whenever the +//! address allocation changes. +//! +//! Grant it with: +//! +//! ```text +//! sudo setcap cap_net_admin+p /usr/local/bin/tsunagi +//! ``` +//! +//! `+p` rather than `+ep`: with `+p` the capability is permitted but not +//! effective at exec, which is exactly the resting state this module wants. +//! `+ep` also works — [`NetAdmin::acquire`] lowers it on the way in. +//! +//! # Capabilities are per thread +//! +//! `capset` affects the calling thread only, so raising one inside an async +//! block would be a bug the moment the task migrated to another worker. Every +//! caller here runs on a single dedicated thread; see +//! [`linux`](super::linux). + +/// Whether this process can configure interfaces. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Privilege { + /// `CAP_NET_ADMIN` is available. The agent manages the interface itself. + Available, + /// It is not, with a description of what was found. + Missing(String), + /// This platform has no provisioner yet. + Unsupported, +} + +impl Privilege { + /// Whether interfaces can be managed. + pub fn is_available(&self) -> bool { + matches!(self, Privilege::Available) + } + + /// How to obtain it, for a diagnostic. + pub fn how_to_grant(program: &str) -> String { + format!( + "Grant it once with `sudo setcap cap_net_admin+p {program}` \ + and the agent manages its own interface. Without it, prepare the \ + interface by hand with `tsunagi tun-setup`." + ) + } +} + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +pub use linux_impl::{NetAdmin, probe_net_admin}; + +#[cfg(not(all(feature = "tun-device", target_os = "linux")))] +pub use other_impl::probe_net_admin; + +#[cfg(not(all(feature = "tun-device", target_os = "linux")))] +mod other_impl { + use super::Privilege; + + /// Whether this process can configure interfaces. + pub fn probe_net_admin() -> Privilege { + Privilege::Unsupported + } +} + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +mod linux_impl { + use caps::{CapSet, Capability}; + + use super::Privilege; + use crate::dataplane::PluginError; + + /// Whether this thread holds `CAP_NET_ADMIN` in its permitted set. + pub fn probe_net_admin() -> Privilege { + match caps::has_cap(None, CapSet::Permitted, Capability::CAP_NET_ADMIN) { + Ok(true) => Privilege::Available, + Ok(false) => Privilege::Missing( + "this process does not hold CAP_NET_ADMIN, so it cannot create \ + or configure a network interface" + .to_string(), + ), + Err(err) => { + Privilege::Missing(format!("cannot read this process's capabilities: {err}")) + } + } + } + + /// `CAP_NET_ADMIN`, raised for as long as this value is alive. + /// + /// Dropping it lowers the capability again, including on the error paths, + /// which is the point of it being a guard rather than a pair of calls. + #[derive(Debug)] + pub struct NetAdmin { + /// Whether this guard is the one that raised it, and so the one that + /// must lower it. Nested acquisition leaves the inner guard inert. + raised: bool, + } + + impl NetAdmin { + /// Raises `CAP_NET_ADMIN` into the effective set. + pub fn acquire() -> Result { + let already = caps::has_cap(None, CapSet::Effective, Capability::CAP_NET_ADMIN) + .map_err(|err| { + PluginError::Unavailable(format!("cannot read capabilities: {err}")) + })?; + if already { + return Ok(Self { raised: false }); + } + caps::raise(None, CapSet::Effective, Capability::CAP_NET_ADMIN).map_err(|err| { + PluginError::Unavailable(format!( + "cannot raise CAP_NET_ADMIN: {err}. {}", + Privilege::how_to_grant("tsunagi") + )) + })?; + Ok(Self { raised: true }) + } + + /// Lowers `CAP_NET_ADMIN` out of the effective set of this thread. + /// + /// Called on the way in as well as on the way out, so that a binary + /// granted `cap_net_admin+ep` — which starts with it effective — still + /// spends almost all of its life unable to use it. + pub fn lower() { + let _ = caps::drop(None, CapSet::Effective, Capability::CAP_NET_ADMIN); + } + } + + impl Drop for NetAdmin { + fn drop(&mut self) { + if self.raised { + Self::lower(); + } + } + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + #[test] + fn the_grant_instructions_name_the_program() { + let text = Privilege::how_to_grant("/usr/local/bin/tsunagi"); + assert!(text.contains("setcap cap_net_admin+p /usr/local/bin/tsunagi")); + assert!(text.contains("tun-setup"), "the fallback is offered too"); + } + + #[test] + fn probing_says_something_definite_about_this_host() { + // Whatever the answer is, it must be one of the three, and a missing + // capability must come with a reason rather than a bare `false`. + match probe_net_admin() { + Privilege::Available => {} + Privilege::Missing(reason) => assert!(!reason.is_empty()), + Privilege::Unsupported => {} + } + } +} diff --git a/src/dataplane/wireguard/provision/unsupported.rs b/src/dataplane/wireguard/provision/unsupported.rs new file mode 100644 index 0000000..e630f96 --- /dev/null +++ b/src/dataplane/wireguard/provision/unsupported.rs @@ -0,0 +1,80 @@ +//! The provisioner for platforms that do not have one yet. +//! +//! macOS and Windows both need real work here — `utun` plus the +//! `SystemConfiguration` framework on one, the IP Helper API and a Wintun +//! adapter on the other — and neither is written. Rather than let the agent +//! come up and fail obscurely at the first packet, this refuses at the point +//! of provisioning and says what to do instead. + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::{InterfacePlan, InterfaceProvisioner, Provisioned}; + +/// Refuses to provision, with an explanation. +#[derive(Debug, Clone)] +pub struct UnsupportedProvisioner { + platform: &'static str, +} + +impl Default for UnsupportedProvisioner { + fn default() -> Self { + Self::new() + } +} + +impl UnsupportedProvisioner { + /// A provisioner naming the platform it is standing in for. + pub fn new() -> Self { + Self { + platform: std::env::consts::OS, + } + } + + fn refusal(&self) -> PluginError { + PluginError::Unavailable(format!( + "managing the overlay interface is not implemented on {} yet. \ + Prepare the interface by hand — `tsunagi tun-setup` prints what to run — \ + and the agent will attach to it.", + self.platform + )) + } +} + +impl InterfaceProvisioner for UnsupportedProvisioner { + fn name(&self) -> &str { + "unsupported" + } + + fn reconcile<'a>( + &'a self, + _plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { Err(self.refusal()) }) + } + + fn remove<'a>(&'a self, _name: &'a str) -> BoxFuture<'a, Result<(), PluginError>> { + // Nothing was ever created, so there is nothing to clean up and no + // reason to fail a shutdown path. + Box::pin(async move { Ok(()) }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + #[tokio::test] + async fn it_refuses_with_a_usable_pointer_and_still_cleans_up_quietly() { + let provisioner = UnsupportedProvisioner::new(); + let plan = InterfacePlan::new("tsuntest", 1280, Vec::new()); + let err = provisioner.reconcile(&plan).await.unwrap_err(); + let message = err.to_string(); + assert!(message.contains("tun-setup"), "{message}"); + assert!(message.contains(std::env::consts::OS), "{message}"); + + provisioner.remove("tsuntest").await.unwrap(); + } +} diff --git a/src/dataplane/wireguard/tun.rs b/src/dataplane/wireguard/tun.rs index 3318972..69e33f4 100644 --- a/src/dataplane/wireguard/tun.rs +++ b/src/dataplane/wireguard/tun.rs @@ -10,7 +10,8 @@ //! is what the test suite uses, so the entire data plane — handshake, //! encryption, routing — is exercised without touching the host. //! * `SystemTun`, behind the `tun-device` feature, is a real TUN interface. -//! Creating one needs `CAP_NET_ADMIN` on Linux or the equivalent elsewhere. +//! Creating one needs `CAP_NET_ADMIN`; attaching to one somebody else +//! prepared needs nothing. use std::net::Ipv6Addr; use std::sync::Arc; @@ -21,7 +22,7 @@ use crate::BoxFuture; use crate::dataplane::PluginError; /// What a device should look like once created. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct TunRequest { /// Interface name to ask for. pub name: String, @@ -37,6 +38,23 @@ pub struct TunRequest { pub mtu: u32, } +impl TunRequest { + /// A request carrying nothing but a name and an MTU. + /// + /// Used where the addresses have already been applied to the host, so the + /// device itself only needs opening. + pub fn bare(name: impl Into, mtu: u32) -> Self { + Self { + name: name.into(), + address: Ipv6Addr::UNSPECIFIED, + prefix_len: 0, + address_v4: None, + prefix_len_v4: 0, + mtu, + } + } +} + /// A packet interface. /// /// `recv` yields packets the operating system wants sent; `send` delivers @@ -66,6 +84,28 @@ pub trait TunFactory: Send + Sync + std::fmt::Debug + 'static { &'a self, request: TunRequest, ) -> BoxFuture<'a, Result, PluginError>>; + + /// Applies a changed request to an interface that already exists. + /// + /// The overlay IPv4 address is allocated at run time, so it can change + /// while the agent runs. A factory that manages the host applies that to + /// the live interface, without recreating it: recreating would drop every + /// tunnel riding on it. + /// + /// The default does nothing, which is right for a factory that only + /// attaches to an interface somebody else prepared. + fn reconfigure<'a>(&'a self, _request: TunRequest) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { Ok(()) }) + } + + /// Removes an interface this factory created. + /// + /// Runs on the teardown path, so it reports rather than fails: there is + /// nothing useful to do about a failure at that point, and an interface + /// that is already gone is the desired outcome anyway. + fn destroy<'a>(&'a self, _name: &'a str) -> BoxFuture<'a, ()> { + Box::pin(async move {}) + } } /// An in-memory packet interface. @@ -196,6 +236,8 @@ impl TunFactory for MemoryTunFactory { } } +#[cfg(feature = "tun-device")] +pub(crate) use system::open_tun; #[cfg(feature = "tun-device")] pub use system::{ Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6, @@ -223,11 +265,16 @@ mod system { /// this user. This is the recommended way to run the agent unprivileged. /// * **Create** it here, which needs `CAP_NET_ADMIN`. /// - /// Either way the overlay address has to be assigned by something - /// privileged: assigning an IPv6 address to an interface is not something - /// this crate's dependencies can do, so the agent checks that it is there - /// and says exactly what to run if it is not, rather than coming up in a - /// state where no traffic could ever arrive. + /// `SystemTunFactory` is the **attach** path, for a host where the agent + /// has no privileges at all: the interface and its addresses were put + /// there by something else, so it checks they are present and says + /// exactly what to run if they are not, rather than coming up in a state + /// where no traffic could ever arrive. + /// + /// The other path is + /// [`ManagedTunFactory`](super::super::provision::ManagedTunFactory), + /// where the agent creates and configures the interface itself. That is + /// the default on Linux and needs no preparation at all. pub struct SystemTun { name: String, mtu: u32, @@ -426,6 +473,60 @@ mod system { commands } + /// Opens the TUN interface, creating it if it is not already there. + /// + /// Synchronous, and deliberately so: on the managed path the caller holds + /// a capability guard across this call, and a guard must not span an + /// `await` because Linux capabilities are per thread. + /// + /// `attach_only` says the interface already exists and was prepared by + /// something else, so nothing beyond `TUNSETIFF` is issued — reconfiguring + /// it would need exactly the privileges that path is avoiding. + pub(crate) fn open_tun( + request: &TunRequest, + attach_only: bool, + ) -> Result, PluginError> { + let mut config = tun::Configuration::default(); + config.tun_name(&request.name); + config.platform_config(|platform| { + // The crate's own root check is not the check we want: the + // managed path holds CAP_NET_ADMIN without being root, and the + // attach path needs no privileges at all. Whether the open + // succeeds is the honest answer either way. + platform.ensure_root_privileges(false); + }); + // Packet information stays off, so reads and writes are raw IP + // packets. `ip tuntap add ... mode tun` also defaults to no packet + // information, so the flags match when attaching to one. + + let device = tun::create_as_async(&config).map_err(|err| { + let hint = if attach_only { + format!( + "interface `{}` exists but could not be opened: {err}. \ + It must be a persistent TUN interface owned by this user.", + request.name + ) + } else { + format!( + "cannot create the TUN interface `{}`: {err}. \ + Creating one needs CAP_NET_ADMIN. Either grant it with \ + `setcap cap_net_admin+p`, or prepare the interface once as root \ + (see `tsunagi tun-setup`) and run unprivileged.", + request.name + ) + }; + PluginError::Unavailable(hint) + })?; + + let (reader, writer) = tokio::io::split(device); + Ok(Arc::new(SystemTun { + name: request.name.clone(), + mtu: request.mtu, + reader: Mutex::new(reader), + writer: Mutex::new(writer), + }) as Arc) + } + fn current_user() -> String { std::env::var("SUDO_USER") .or_else(|_| std::env::var("USER")) @@ -468,41 +569,7 @@ mod system { ))); } - let mut config = tun::Configuration::default(); - config.tun_name(&request.name); - if existed { - // Attach only. Reconfiguring an interface somebody - // prepared for us would need exactly the privileges we - // are avoiding, so no ioctl beyond TUNSETIFF is issued. - config.platform_config(|platform| { - platform.ensure_root_privileges(false); - }); - } else { - // We are creating it, so we configure it. - config.mtu(request.mtu as u16).up(); - } - // Packet information stays off, so reads and writes are raw IP - // packets. `ip tuntap add ... mode tun` also defaults to no - // packet information, so the flags match when attaching. - - let device = tun::create_as_async(&config).map_err(|err| { - let hint = if existed { - format!( - "interface `{}` exists but could not be opened: {err}. \ - It must be a persistent TUN interface owned by this user.", - request.name - ) - } else { - format!( - "cannot create the TUN interface `{}`: {err}. \ - Creating one needs CAP_NET_ADMIN. Either prepare it once as root \ - (see `tsunagi tun-setup`) and run unprivileged, or grant the \ - capability.", - request.name - ) - }; - PluginError::Unavailable(hint) - })?; + let device = open_tun(&request, existed)?; // An interface we just created has no address yet either. if let Err(reason) = check_address(&request.name, request.address) { @@ -512,13 +579,7 @@ mod system { ))); } - let (reader, writer) = tokio::io::split(device); - Ok(Arc::new(SystemTun { - name: request.name, - mtu: request.mtu, - reader: Mutex::new(reader), - writer: Mutex::new(writer), - }) as Arc) + Ok(device) }) } } diff --git a/tests/interface_provisioning.rs b/tests/interface_provisioning.rs new file mode 100644 index 0000000..915afbb --- /dev/null +++ b/tests/interface_provisioning.rs @@ -0,0 +1,244 @@ +//! The agent managing its own overlay interface. +//! +//! Everything here is real except the host: real agents, real control plane, +//! real iroh links, the real plugin lifecycle and the real reconciliation +//! rules. The host itself is a [`MockHost`], so what the agent would have +//! done to a machine's interfaces is asserted instead of done — which is how +//! this runs with no privileges and without touching the machine it is on. +//! +//! What the real provisioner adds on top of this is the netlink calls, and +//! only those. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::Arc; +use std::time::Duration; + +use common::{config_with, network, wait_until}; +use tempfile::TempDir; +use tsunagi::dataplane::IpPlugin; +use tsunagi::dataplane::wireguard::{ + Cidr, InterfaceState, LinkKind, ManagedTunFactory, MockHost, MockProvisioner, WireguardConfig, + WireguardPlugin, +}; +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::identity::NetworkId; +use tsunagi::state::DEFAULT_IPV4_RANGE; +use tsunagi::{Agent, NetworkStatus}; + +/// An agent whose overlay interface is applied to a pretend host. +struct HostedAgent { + _dir: TempDir, + agent: Agent, + plugin: Arc, + host: MockHost, +} + +impl HostedAgent { + async fn spawn(discovery: &SharedMemoryDiscovery, tag: &str, host: MockHost) -> Self { + let dir = TempDir::new().unwrap(); + let provisioner = Arc::new(MockProvisioner::new(host.clone())); + let factory = Arc::new(ManagedTunFactory::new(provisioner)); + let config = WireguardConfig::new(dir.path().join("wireguard")) + .with_interface_prefix(tag) + .with_reconcile(Duration::from_millis(20), Duration::from_millis(100)); + let plugin = WireguardPlugin::open(config, factory).await.unwrap(); + let agent = Agent::spawn( + config_with(dir.path(), discovery) + .with_overlay_ipv4_range(Some(DEFAULT_IPV4_RANGE)) + .with_plugin(plugin.clone() as Arc), + ) + .await + .unwrap(); + Self { + _dir: dir, + agent, + plugin, + host, + } + } + + /// The interface name the plugin settled on for a network. + async fn interface(&self, network: NetworkId) -> String { + wait_until("the plugin named its interface", || async { + self.plugin + .overview(network) + .map(|view| view.interface) + .filter(|name| !name.is_empty()) + }) + .await + } + + /// Waits until the pretend host shows an interface in the given state. + async fn wait_for_host( + &self, + what: &str, + name: &str, + probe: impl Fn(Option) -> Option, + ) -> T { + wait_until(what, || async { probe(self.host.get(name)) }).await + } +} + +fn v4(state: &InterfaceState) -> Vec { + state + .addresses + .iter() + .filter_map(|cidr| match cidr.addr { + IpAddr::V4(addr) => Some(addr), + IpAddr::V6(_) => None, + }) + .collect() +} + +fn has_v6(state: &InterfaceState) -> bool { + state + .addresses + .iter() + .any(|cidr| matches!(cidr.addr, IpAddr::V6(_))) +} + +#[tokio::test] +async fn an_agent_creates_and_configures_its_own_overlay_interface() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-create"); + let agent = HostedAgent::spawn(&discovery, "tsunp", MockHost::new()).await; + + let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); + let interface = agent.interface(network_id).await; + + let state = agent + .wait_for_host("the interface to be created", &interface, |state| { + state.filter(|state| !state.addresses.is_empty()) + }) + .await; + + assert_eq!(state.kind, LinkKind::Tun); + assert!(state.up, "the agent brought the link up itself"); + assert_eq!(state.mtu, 1280); + assert!(has_v6(&state), "the derived overlay address is assigned"); + + // The IPv4 address is allocated at run time, so it arrives on a later + // reconciliation than the interface itself. + let addresses = agent + .wait_for_host("the allocated IPv4 address", &interface, |state| { + state.map(|state| v4(&state)).filter(|v4| !v4.is_empty()) + }) + .await; + assert_eq!(addresses.len(), 1); + assert!( + DEFAULT_IPV4_RANGE.contains(addresses[0]), + "{:?} is outside {DEFAULT_IPV4_RANGE}", + addresses[0] + ); + + agent.agent.shutdown().await; +} + +#[tokio::test] +async fn an_interface_left_by_a_crashed_run_is_replaced_rather_than_tripped_over() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-crash"); + + // Work out the name this agent will use, then seed the host with what a + // run that died would have left there: the interface, still carrying an + // address from an allocation that no longer applies, with nothing holding + // it open. + let probe = HostedAgent::spawn(&discovery, "tsunc", MockHost::new()).await; + let network_id = probe.agent.join_network(&name, &secret).await.unwrap(); + let interface = probe.interface(network_id).await; + probe.agent.shutdown().await; + + let host = MockHost::new(); + let stale = Cidr::new(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 178)), 24).unwrap(); + host.insert_stale_tun(&interface, vec![stale]); + + let agent = HostedAgent::spawn(&discovery, "tsunc", host).await; + let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); + assert_eq!(agent.interface(network_id).await, interface); + + let state = agent + .wait_for_host("the interface to be rebuilt", &interface, |state| { + state.filter(|state| state.attached && has_v6(state)) + }) + .await; + assert!( + !state.addresses.contains(&stale), + "the stale address is gone: {:?}", + state.addresses + ); + + agent.agent.shutdown().await; +} + +#[tokio::test] +async fn an_interface_belonging_to_something_else_is_left_alone() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-foreign"); + + let probe = HostedAgent::spawn(&discovery, "tsunf", MockHost::new()).await; + let network_id = probe.agent.join_network(&name, &secret).await.unwrap(); + let interface = probe.interface(network_id).await; + probe.agent.shutdown().await; + + // Somebody else's bridge happens to hold the name. + let host = MockHost::new(); + let theirs = InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: vec![Cidr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 9, 1)), 24).unwrap()], + }; + host.insert(&interface, theirs.clone()); + + let agent = HostedAgent::spawn(&discovery, "tsunf", host).await; + agent.agent.join_network(&name, &secret).await.unwrap(); + + // Give the plugin several reconciliation rounds to do the wrong thing. + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + agent.host.get(&interface), + Some(theirs), + "the foreign interface must be untouched" + ); + + // The control plane is unaffected by the data plane refusing. + assert!(matches!( + agent.agent.network_status(network_id).await, + Ok(NetworkStatus { .. }) + )); + + agent.agent.shutdown().await; +} + +#[tokio::test] +async fn leaving_a_network_removes_the_interface_from_the_host() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-cleanup"); + let agent = HostedAgent::spawn(&discovery, "tsunx", MockHost::new()).await; + + let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); + let interface = agent.interface(network_id).await; + agent + .wait_for_host("the interface to exist", &interface, |state| state) + .await; + + agent.agent.deactivate_network(network_id).await.unwrap(); + + agent + .wait_for_host("the interface to be removed", &interface, |state| { + state.is_none().then_some(()) + }) + .await; + assert!( + agent.host.names().is_empty(), + "nothing is left behind: {:?}", + agent.host.names() + ); + + agent.agent.shutdown().await; +}