From b23e832a73c6bcf9f6cc1110d26a8534d9c28592 Mon Sep 17 00:00:00 2001 From: tsunagi Date: Mon, 21 Sep 2026 14:33:19 +0100 Subject: [PATCH] Manage the overlay interface instead of asking for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent printed a list of `ip` commands and asked a human to run them. That is fragile in the way hand-held setup always is: a persistent TUN does not survive a reboot, a changed address allocation needs another manual round, and a run that died leaves a half-configured interface the next run trips over. On Linux the agent now creates the interface, sets the MTU, brings it up and assigns both overlay addresses itself, over netlink in process. No `ip` is invoked, so nothing this path does can be influenced by PATH, a shell, or anything a remote peer said. Cleanup stops being an action. The interface is tied to an open file descriptor and is deliberately not persistent, so the kernel removes it when the agent goes — cleanly, by panic, by SIGKILL or by power loss alike. That also retires `keep_addr_on_down` and `nodad`, which existed only because an interface nobody held open lost carrier. Anything still left behind is repaired rather than tripped over: an abandoned TUN is replaced along with its stale addresses. Two cases refuse instead of guessing — a link that is not a TUN, because a name collision is no reason to destroy somebody's bridge, and a TUN another process holds open, because that is a working overlay belonging to someone else. CAP_NET_ADMIN is kept out of the effective set except around the calls that use it. Two facts shape how: capabilities are per thread, and netlink checks the credentials of whichever thread calls sendmsg, which with an async client is the connection task rather than the caller. So netlink runs on one dedicated thread with a current-thread runtime where nothing is polled outside a block_on, and opening the TUN descriptor is synchronous with no await between the guard and its release. The decision of what to change is a pure function, tested on every platform; only the execution is behind the provisioner trait. macOS and Windows get an implementation that refuses with an explanation and falls back to attaching to a prepared interface, plus a mock host the tests drive the whole plugin lifecycle against. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 65 +- Cargo.toml | 10 +- README.md | 123 ++-- docs/wireguard.md | 97 ++- src/bin/tsunagi.rs | 115 +++- src/dataplane/wireguard/mod.rs | 17 +- src/dataplane/wireguard/plugin.rs | 87 ++- src/dataplane/wireguard/provision/factory.rs | 201 ++++++ src/dataplane/wireguard/provision/linux.rs | 605 ++++++++++++++++++ src/dataplane/wireguard/provision/mock.rs | 336 ++++++++++ src/dataplane/wireguard/provision/mod.rs | 440 +++++++++++++ .../wireguard/provision/privilege.rs | 169 +++++ .../wireguard/provision/unsupported.rs | 80 +++ src/dataplane/wireguard/tun.rs | 159 +++-- tests/interface_provisioning.rs | 244 +++++++ 15 files changed, 2598 insertions(+), 150 deletions(-) create mode 100644 src/dataplane/wireguard/provision/factory.rs create mode 100644 src/dataplane/wireguard/provision/linux.rs create mode 100644 src/dataplane/wireguard/provision/mock.rs create mode 100644 src/dataplane/wireguard/provision/mod.rs create mode 100644 src/dataplane/wireguard/provision/privilege.rs create mode 100644 src/dataplane/wireguard/provision/unsupported.rs create mode 100644 tests/interface_provisioning.rs diff --git a/Cargo.lock b/Cargo.lock index 52c2407..01991bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ dependencies = [ "ip_network", "ip_network_table", "libc", - "nix", + "nix 0.31.3", "parking_lot", "portable-atomic", "rand_core 0.6.4", @@ -307,6 +307,15 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "caps" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1ddba47aba30b6a889298ad0109c3b8dcb0e8fc993b459daa7067d46f865e0" +dependencies = [ + "libc", +] + [[package]] name = "cc" version = "1.4.7" @@ -2065,6 +2074,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "netlink-proto" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e03ae6247c6cdb499e8f14e98c72b83954d85822b687056e3a21150b438f2b" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.9.0", + "netlink-sys 0.9.0", + "thiserror 2.0.20", +] + [[package]] name = "netlink-sys" version = "0.8.8" @@ -2085,8 +2109,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10" dependencies = [ "bytes", + "futures-util", "libc", "log", + "tokio", ] [[package]] @@ -2109,7 +2135,7 @@ dependencies = [ "netdev 0.46.3", "netlink-packet-core 0.8.2", "netlink-packet-route 0.31.0", - "netlink-proto", + "netlink-proto 0.12.2", "netlink-sys 0.8.8", "noq-udp", "objc2-core-foundation", @@ -2127,6 +2153,18 @@ dependencies = [ "wmi", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -2748,6 +2786,24 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "rtnetlink" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f60a3ee149005c5819e03e3d73ba4a470bc18b52b0c8422cfe1bea2cae63c6" +dependencies = [ + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.9.0", + "netlink-packet-route 0.33.0", + "netlink-proto 0.13.0", + "netlink-sys 0.9.0", + "nix 0.30.1", + "thiserror 2.0.20", + "tokio", +] + [[package]] name = "rusqlite" version = "0.40.2" @@ -3602,10 +3658,12 @@ version = "0.1.0" dependencies = [ "boringtun", "bytes", + "caps", "clap", "data-encoding", "directories", "fs4", + "futures-util", "hex", "hkdf", "hmac 0.13.0", @@ -3613,6 +3671,7 @@ dependencies = [ "netwatch", "postcard", "rand", + "rtnetlink", "rusqlite", "serde", "sha2", @@ -3639,7 +3698,7 @@ dependencies = [ "ipnet", "libc", "log", - "nix", + "nix 0.31.3", "serde_json", "thiserror 2.0.20", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 9fd1be1..e1a9871 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ cli = ["dep:clap", "dep:tracing-subscriber", "tokio/signal", "tun-device"] # A real TUN device, so the WireGuard plugin can carry actual IP traffic. # Needs CAP_NET_ADMIN at run time; without it the plugin still runs and its # in-memory device can be used for tests. -tun-device = ["dep:tun"] +tun-device = ["dep:tun", "dep:rtnetlink", "dep:caps", "dep:futures-util"] [[bin]] name = "tsunagi" @@ -49,6 +49,14 @@ bytes = "1.12.1" boringtun = { version = "0.7.1", default-features = false } tun = { version = "0.8", features = ["async"], optional = true } +# Linux-only interface provisioning. `rtnetlink` configures the interface in +# process, so no `ip` invocation is ever needed; `caps` keeps CAP_NET_ADMIN +# out of the effective set except during the moments it is used. +[target.'cfg(target_os = "linux")'.dependencies] +rtnetlink = { version = "0.23", optional = true } +caps = { version = "0.5", optional = true } +futures-util = { version = "0.3", default-features = false, optional = true } + [dev-dependencies] tokio = { version = "1.53", features = ["rt", "rt-multi-thread", "sync", "time", "macros", "process"] } tempfile = "3.24" diff --git a/README.md b/README.md index 9ef0505..aa3b8c4 100644 --- a/README.md +++ b/README.md @@ -146,19 +146,57 @@ No vote is involved — see [docs/sync-model.md](docs/sync-model.md). Because the address is allocated at run time rather than derived, it is not -known until the agent has started and agreed with its peers, so -`tsunagi tun-setup` cannot print it in advance. The agent prints the exact -`ip address add` command once it has one, and keeps saying so until the -address is actually on an interface — without it, packets leave with the -wrong source address and every peer drops them. +known until the agent has started and agreed with its peers. On Linux the +agent assigns it itself as soon as it has one; elsewhere `tsunagi tun-setup` +prints the `ip address add` line once the claim is in `state.sqlite`. -## Running unprivileged +## Privileges -The agent does not need to run as root. Creating a network interface and -giving it an address do need privileges, but they are a **one-time setup step** -that can be done separately. +On Linux the agent **manages its own overlay interface**. It creates the TUN +interface, sets the MTU, brings it up and assigns both overlay addresses, all +over netlink in process — no `ip` invocation, no shell, nothing that a +remote peer could influence. -Ask the agent what it needs, then run that once as root: +That needs `CAP_NET_ADMIN`, granted once: + +```bash +sudo setcap cap_net_admin+p /usr/local/bin/tsunagi +``` + +`+p` rather than `+ep`: the capability is then *permitted* but not +*effective*, and the agent raises it only around the handful of netlink calls +that need it — a few milliseconds at startup, and again if its address +allocation changes. Everything else, including every byte from the network, +is handled with it lowered. `+ep` works too; the agent lowers it on the way +in. + +`tsunagi doctor` says which of these applies on the host it runs on. + +### It cleans up after itself + +The interface is tied to an open file descriptor and is deliberately **not** +made persistent, so the kernel removes it when the agent exits — on a clean +shutdown, on a panic, on `SIGKILL`, on power loss alike. Keeping it is what +would take an action; removing it is the default. + +If something is left behind anyway — an interface made by an older version's +manual recipe, or one from a run killed in the instant between creating it and +recording it — the next start **replaces it**, along with any stale addresses +it carried. Two things are never touched: + +* an interface that is not a TUN, because the name colliding with somebody's + bridge is not a reason to destroy the bridge; +* a TUN that another process is holding open, because that is a working + overlay belonging to somebody else — most likely a second agent on this + host, which should be given a different `--wg-prefix`. + +Both of those refuse with an explanation rather than guessing. + +### Running without the capability + +`--interface attach` (or `auto`, which falls back on its own) opens an +interface prepared beforehand and needs **no privileges at all**. Ask the +agent what to run: ```bash tsunagi tun-setup --network lab --secret "$SECRET" @@ -174,55 +212,44 @@ sudo sysctl -qw net.ipv6.conf.tsunjwc6dcrtmo5.keep_addr_on_down=1 sudo ip -6 address add fd80:1210:f724:f620:d1bb:f982:3b6e:19bd/64 dev tsunjwc6dcrtmo5 nodad ``` -With IPv4 enabled, `tun-setup` adds an `ip address add` line for the overlay -IPv4 address too — but only once there is one to print. Unlike the IPv6 -address, the IPv4 address is not derived from the keys: it is allocated at run -time and signed (see [docs/sync-model.md](docs/sync-model.md)), so it exists -only after the agent has run once. `tun-setup` then reads it back out of -`state.sqlite`, which does not disturb a running agent, and includes it from -then on. Until then the first `tsunagi up` prints the exact command for the -address it was given. +`user ab` is the point: the interface is persistent and owned by that user, so +`tsunagi up` afterwards opens it with no privileges and no capabilities. -That IPv4 line needs no `keep_addr_on_down` and no `nodad`: Linux keeps IPv4 -addresses on an interface that loses carrier, and IPv4 has no duplicate -address detection to stall. Adding it once is enough. +The last two settings are what the managed path does not need. A persistent +TUN has **no carrier** until a process attaches to it; Linux flushes IPv6 +addresses from an interface that loses carrier unless `keep_addr_on_down` is +set, and duplicate address detection can never finish without carrier, so the +address would sit there tentative and unusable without `nodad`. An interface +the agent creates and holds open has carrier for its whole life, so neither +applies. IPv4 needs neither in either case: Linux keeps IPv4 addresses across +carrier loss and IPv4 has no duplicate address detection. + +With IPv4 enabled, `tun-setup` adds an `ip address add` line once there is an +address to print — it reads the signed claim back out of `state.sqlite`, which +does not disturb a running agent. The MTU is 1280 because that is the minimum IPv6 requires (RFC 8200). Linux disables IPv6 entirely on an interface below it — the per-device -`/proc/sys/net/ipv6` entries vanish and `ip -6 address add` fails with +`/proc/sys/net/ipv6` entries vanish and adding an address fails with `Invalid argument` — so a smaller MTU cannot work at all. The agent refuses one rather than letting it fail later. -The order and the last two lines are not decoration. A persistent TUN -interface has **no carrier** until a process attaches to it, and Linux flushes -IPv6 addresses from an interface that loses carrier unless -`keep_addr_on_down` is set — so an address added without it disappears before -the agent ever starts. `nodad` is needed for the same reason: duplicate -address detection cannot finish without a carrier, and the address would sit -there tentative and unusable. - -`user ab` is the point: the interface is persistent and owned by that user, so -`tsunagi up` afterwards opens it with **no privileges and no capabilities at -all**. The interface name and address are derived, so they are stable — the -setup survives restarts and only has to be redone if the network name, the -secret or this agent's WireGuard key changes. - -Other ways, and their trade-offs: +### Summary | approach | agent runs as | notes | |---|---|---| -| `tsunagi tun-setup` (above) | ordinary user, no capabilities | recommended | -| `sudo setcap cap_net_admin+ep ./tsunagi` | ordinary user, one capability | the agent can then create the interface itself, but **still cannot assign the IPv6 address** (see below), so the `ip -6 address add` line is needed anyway. The capability is lost on every rebuild or copy. | -| systemd service | `User=`, `AmbientCapabilities=CAP_NET_ADMIN` | same caveat about the address | -| plain `sudo tsunagi up` | root | everything works, nothing is isolated | +| `setcap cap_net_admin+p` | ordinary user, one capability | recommended on Linux: nothing to prepare, nothing left behind. Lost on every rebuild or copy of the binary. | +| systemd service | `User=`, `AmbientCapabilities=CAP_NET_ADMIN` | the same, for an installed service | +| `tsunagi tun-setup` then `--interface attach` | ordinary user, no capabilities | one privileged setup per host; needed on macOS and Windows, where no provisioner is implemented yet | +| `sudo tsunagi up` | root | everything works, nothing is isolated | +| `--no-tun` | ordinary user, no capabilities | tunnels run and handshake, traffic never reaches the OS | -**Known limitation.** The agent cannot assign the IPv6 overlay address itself: -the `tun` crate sets addresses through an IPv4-only ioctl, so an IPv6 address -has to come from `ip -6 address add` or an equivalent. Rather than start with -an interface that could never receive anything, the agent checks for the -address in `/proc/net/if_inet6` and refuses with the exact command to run. -Doing it in-process would mean talking netlink directly, which is possible but -not implemented. +**Not implemented yet.** macOS and Windows have no provisioner: both need +real platform work — `utun` and `SystemConfiguration` on one, the IP Helper +API and a Wintun adapter on the other. On those the agent says so and falls +back to attaching to a prepared interface. The decision logic that says *what* +to change is shared and tested on every platform; only the execution is +per-platform. ## Checks diff --git a/docs/wireguard.md b/docs/wireguard.md index fa99dfa..8f21026 100644 --- a/docs/wireguard.md +++ b/docs/wireguard.md @@ -267,22 +267,85 @@ async fn main() -> Result<()> { * **Userspace costs CPU.** Kernel WireGuard is faster. A kernel backend could return behind the same boundary, but it would give up transport-provided NAT traversal unless paired with a local proxy. -* **A persistent TUN interface needs `keep_addr_on_down`.** Without a process - attached it has no carrier, and Linux then flushes its IPv6 addresses. The - setup printed by `tsunagi tun-setup` sets it; the agent checks the address is - present *and usable* — not tentative, not DAD-failed — before attaching, and - reports what it actually found. This applies to IPv6 only: IPv4 addresses - survive carrier loss, so the overlay IPv4 address is added once and stays. -* **The overlay IPv4 address is not derivable, so `tun-setup` cannot print it - on a fresh state directory.** It is allocated and signed at run time, so the - first `tsunagi up` is what names it; afterwards `tun-setup` reads it back - from `state.sqlite` — a read that takes no directory lock and so does not - disturb a running agent — and includes the `ip address add` line. -* **The agent cannot assign the overlay address itself.** The `tun` crate sets - addresses through an IPv4-only ioctl, so the IPv6 overlay address must come - from `ip -6 address add` or an equivalent. The agent verifies the address is - present, via `/proc/net/if_inet6`, and refuses with the exact command rather - than running an interface that could never receive anything. Doing it - in-process would mean speaking netlink, which is not implemented. +* **The interface is managed, not prepared.** On Linux the agent creates the + TUN interface and configures it over netlink, in process. See + *Provisioning* below. +* **No provisioner exists for macOS or Windows yet.** There the agent attaches + to an interface prepared by hand and says so. * **The system interface path is not exercised by the default suite**, because it needs privileges. Everything else about the data plane is. + + +## Provisioning the interface + +Everything the printed `ip` recipe used to do happens in process now, over +netlink. The shape of it is a reconciliation rather than a sequence of +commands: the agent is handed a plan — name, MTU, the addresses the interface +should carry and no others — observes what is actually on the host, and +applies the difference. Running it twice changes nothing the second time. + +The decision of *what* to change is +`dataplane::wireguard::provision::plan_changes`: pure, platform-independent +and unit-tested on every platform. Only the execution is behind +`InterfaceProvisioner`, which has three implementations — netlink on Linux, a +`MockProvisioner` over a pretend host for the tests, and one that refuses with +an explanation everywhere else. + +### Cleanup is the default, not an action + +The interface is created by opening `/dev/net/tun` and is **not** made +persistent, so the kernel destroys it when the last descriptor closes. A clean +shutdown, a panic, `SIGKILL` and a power cut all leave the same amount behind: +nothing. There is no path by which a dead agent leaves an interface, because +keeping one alive is what requires a live process. + +This also removes the two settings the manual recipe needed. `keep_addr_on_down` +existed only because an interface nobody held open lost carrier and had its +IPv6 addresses flushed; `nodad` only because duplicate address detection +cannot finish without carrier. An interface held open for its whole life has +carrier for its whole life. + +### Repairing what an older run left + +Two things can still be sitting on the name: an interface created persistent +by the old recipe, and — narrowly — one from a run killed between `TUNSETIFF` +and the agent recording it. Both are replaced, which discards their stale +addresses with them. + +The two refusals are the interesting part: + +* **A link that is not a TUN is never touched.** The name is derived from the + network id, so colliding with a real device is unlikely rather than + impossible, and deleting somebody's bridge is not a recoverable mistake. +* **A TUN another process holds open is never deleted.** Carrier is the + signal: a TUN has it exactly while something is attached. An attached one is + a working overlay, almost certainly a second agent on this host, and it is + told to use a different `--wg-prefix` instead. + +### Privilege + +`CAP_NET_ADMIN` is required and is kept out of the *effective* set except +around the netlink calls that need it. `setcap cap_net_admin+p` leaves it +permitted but not effective at exec, which is the resting state; the agent +raises it for a few milliseconds at startup and again when its address +allocation changes. + +Two facts shape how that is done. Capabilities on Linux are **per thread**, +and netlink checks the credentials of whichever thread calls `sendmsg` — +which, with an async client, is the connection task and not the caller. So +raising a capability around an `await` would be wrong in the way that works +until the scheduler moves the task. + +Therefore: all netlink work runs on one dedicated thread with a current-thread +runtime, where nothing is polled outside a `block_on`, and the capability is +raised immediately before that call and lowered immediately after. Opening the +TUN descriptor is the other privileged act; it is a synchronous call with no +`await` between the guard and the release, so it stays on its own thread by +construction. + +### What it cannot be told to do + +Nothing here takes a name, an address or a command from the network. The +interface name is derived from the network id, the addresses come from the +local plugin and the signed allocation records, and no external program is +executed at any point — there is no `ip`, no shell and no `PATH` involved. diff --git a/src/bin/tsunagi.rs b/src/bin/tsunagi.rs index 4c2b91f..104524e 100644 --- a/src/bin/tsunagi.rs +++ b/src/bin/tsunagi.rs @@ -224,6 +224,15 @@ struct UpArgs { #[arg(long)] no_tun: bool, + /// How the overlay interface is obtained. + /// + /// `managed` has the agent create and configure it itself, which needs + /// CAP_NET_ADMIN and cleans up on exit. `attach` opens an interface that + /// was prepared beforehand (see `tsunagi tun-setup`) and needs no + /// privileges. `auto` manages it when it can and attaches when it cannot. + #[arg(long, value_enum, default_value_t = InterfaceMode::Auto)] + interface: InterfaceMode, + /// Interface name prefix for the WireGuard data plane. #[arg(long, default_value = "tsun")] wg_prefix: String, @@ -533,16 +542,33 @@ async fn doctor(paths: PathArgs) -> Result<(), Box> { #[cfg(not(feature = "tun-device"))] println!(" interfaces not built in (enable the `tun-device` feature)"); - #[cfg(unix)] + #[cfg(feature = "tun-device")] { - // Creating a network interface needs CAP_NET_ADMIN, which in practice - // means root unless capabilities were granted explicitly. - let euid = std::fs::metadata("/proc/self").ok().map(|_| ()); - let _ = euid; - println!( - " privileges creating an interface needs CAP_NET_ADMIN; \ - use --no-tun to run without it" - ); + use tsunagi::dataplane::wireguard::{Privilege, probe_net_admin}; + match probe_net_admin() { + Privilege::Available => { + println!(" privileges CAP_NET_ADMIN held"); + println!( + " interface managed by the agent: created on start, \ + removed on exit" + ); + } + Privilege::Missing(reason) => { + println!(" privileges no CAP_NET_ADMIN ({reason})"); + println!(" interface must be prepared first; run `tsunagi tun-setup`"); + println!( + " to manage it {}", + Privilege::how_to_grant(&program_path()) + ); + } + Privilege::Unsupported => { + println!( + " privileges managing interfaces is not implemented on {} yet", + std::env::consts::OS + ); + println!(" interface must be prepared first; run `tsunagi tun-setup`"); + } + } } println!("\nlocal addresses"); @@ -556,6 +582,14 @@ async fn doctor(paths: PathArgs) -> Result<(), Box> { Ok(()) } +/// This program's path, for an instruction the user can paste. +fn program_path() -> String { + std::env::current_exe() + .ok() + .and_then(|path| path.to_str().map(str::to_string)) + .unwrap_or_else(|| "tsunagi".to_string()) +} + async fn netwatch_addresses() -> Vec { // Best effort; used for diagnostics only. let state = netwatch::interfaces::State::new().await; @@ -600,7 +634,7 @@ async fn up(args: UpArgs) -> Result<(), Box> { let tun_factory: Arc = if args.no_tun { Arc::new(MemoryTunFactory::new()) } else { - system_tun_factory()? + system_tun_factory(args.interface)? }; let mut wg = WireguardConfig::new(paths.state_dir.join("wireguard")) .with_interface_prefix(args.wg_prefix.clone()); @@ -832,14 +866,69 @@ async fn stop_signal() -> &'static str { } } +/// How the overlay interface is obtained. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +enum InterfaceMode { + /// Manage it when possible, attach to a prepared one otherwise. + Auto, + /// Create and configure it in process. Needs CAP_NET_ADMIN. + Managed, + /// Open an interface prepared beforehand. Needs no privileges. + Attach, +} + +/// Builds the interface factory for the chosen mode. +/// +/// The managed path is preferred because it is the one that cleans up after +/// itself: the interface is tied to an open file descriptor, so it goes away +/// when the agent does, however the agent goes away. #[cfg(feature = "tun-device")] -fn system_tun_factory() -> Result, Box> { +fn system_tun_factory( + mode: InterfaceMode, +) -> Result, Box> { use tsunagi::dataplane::wireguard::SystemTunFactory; - Ok(Arc::new(SystemTunFactory::new())) + + if mode == InterfaceMode::Attach { + return Ok(Arc::new(SystemTunFactory::new())); + } + + match managed_tun_factory() { + Ok(factory) => Ok(factory), + Err(err) if mode == InterfaceMode::Managed => Err(err), + Err(err) => { + tracing::warn!( + "{err} Falling back to attaching to a prepared interface; \ + `tsunagi tun-setup` prints how to make one." + ); + Ok(Arc::new(SystemTunFactory::new())) + } + } +} + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +fn managed_tun_factory() -> Result, Box> { + use tsunagi::dataplane::wireguard::{ManagedTunFactory, NetlinkProvisioner}; + let provisioner = NetlinkProvisioner::new()?; + Ok(Arc::new(ManagedTunFactory::new(Arc::new(provisioner)))) +} + +/// There is no provisioner for this platform yet. +/// +/// Refused here rather than at the first packet, so `auto` falls back to +/// attaching and `--interface managed` says plainly why it cannot. +#[cfg(all(feature = "tun-device", not(target_os = "linux")))] +fn managed_tun_factory() -> Result, Box> { + Err(format!( + "managing the overlay interface is not implemented on {} yet.", + std::env::consts::OS + ) + .into()) } #[cfg(not(feature = "tun-device"))] -fn system_tun_factory() -> Result, Box> { +fn system_tun_factory( + _mode: InterfaceMode, +) -> Result, Box> { Err("this build has no interface support; rebuild with the `tun-device` feature or pass --no-tun".into()) } diff --git a/src/dataplane/wireguard/mod.rs b/src/dataplane/wireguard/mod.rs index eba2c32..56e69f0 100644 --- a/src/dataplane/wireguard/mod.rs +++ b/src/dataplane/wireguard/mod.rs @@ -34,9 +34,11 @@ //! //! The only part that needs privileges is the packet interface. With //! [`tun::MemoryTunFactory`] the whole data plane — handshake, encryption, -//! routing, address ownership — runs and is tested with no privileges at all; -//! `SystemTunFactory` swaps in a real interface when you want traffic to -//! reach the operating system. +//! routing, address ownership — runs and is tested with no privileges at all. +//! For real traffic there are two ways in: [`provision::ManagedTunFactory`], +//! where the agent creates and configures the interface itself over netlink +//! and removes it again on exit, and `SystemTunFactory`, which attaches to an +//! interface somebody else prepared and needs no privileges. //! //! See `docs/wireguard.md` for the full picture. @@ -47,6 +49,7 @@ pub mod keys; pub mod overlay; pub mod packet; pub mod plugin; +pub mod provision; pub mod store; pub mod tun; @@ -61,9 +64,17 @@ pub use plugin::{ DEFAULT_MTU, MIN_MTU, NetworkOverview, PeerOverview, WIREGUARD_OVERHEAD, WIREGUARD_PROTOCOL, WireguardConfig, WireguardPlugin, }; +pub use provision::{ + Changes, InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, ManagedTunFactory, + MockHost, MockProvisioner, Privilege, Provisioned, UnsupportedProvisioner, plan_changes, + probe_net_admin, +}; pub use store::WgKeyStore; pub use tun::{MemoryTun, MemoryTunFactory, TunDevice, TunFactory, TunRequest, address_is_local}; +#[cfg(all(feature = "tun-device", target_os = "linux"))] +pub use provision::NetlinkProvisioner; + #[cfg(feature = "tun-device")] pub use tun::{ Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6, diff --git a/src/dataplane/wireguard/plugin.rs b/src/dataplane/wireguard/plugin.rs index e52a045..1b76e66 100644 --- a/src/dataplane/wireguard/plugin.rs +++ b/src/dataplane/wireguard/plugin.rs @@ -211,6 +211,12 @@ struct NetworkState { ipv4_range: Option, /// The address last reported as missing, so it is said once, not forever. reported_missing_v4: Option, + /// What was last applied to the host interface. + /// + /// The overlay IPv4 address is allocated at run time and can change while + /// the agent runs, so the interface has to be brought back in line + /// without being recreated — recreating it would drop every tunnel. + applied: Option, } #[derive(Debug, Default)] @@ -444,6 +450,7 @@ impl Worker { allocations: HashMap::new(), ipv4_range: None, reported_missing_v4: None, + applied: None, }); } @@ -456,29 +463,34 @@ impl Worker { Ok(true) } + /// What the host interface for a network should look like. + fn desired_request(&self, state: &NetworkState, network: NetworkId) -> TunRequest { + let own_range = state.ipv4_range; + TunRequest { + name: state.interface.clone(), + address: overlay_address(network, &state.key.public()), + prefix_len: OVERLAY_PREFIX_LEN, + address_v4: state.allocations.get(&self.local_id()).copied(), + prefix_len_v4: own_range.map_or(0, |range| range.prefix_len), + mtu: self.config.mtu, + } + } + /// Creates the packet interface and starts the WireGuard device. async fn ensure_device(&self, network: NetworkId) -> Result<(), PluginError> { - let (name, key, own_v4, own_range) = { + let (request, key, own_range) = { let shared = self.lock_shared(); match shared.networks.get(&network) { Some(state) if state.device.is_none() => ( - state.interface.clone(), + self.desired_request(state, network), state.key.clone(), - state.allocations.get(&self.local_id()).copied(), state.ipv4_range, ), _ => return Ok(()), } }; - let request = TunRequest { - name: name.clone(), - address: overlay_address(network, &key.public()), - prefix_len: OVERLAY_PREFIX_LEN, - address_v4: own_v4, - prefix_len_v4: own_range.map_or(0, |range| range.prefix_len), - mtu: self.config.mtu, - }; + let applied = request.clone(); let tun = self.tun_factory.create(request).await?; let device = Arc::new(WireguardDevice::start(network, key, tun, own_range)); @@ -486,10 +498,44 @@ impl Worker { if let Some(state) = shared.networks.get_mut(&network) { state.interface = device.interface().to_string(); state.device = Some(device); + state.applied = Some(applied); } Ok(()) } + /// Brings a live interface back in line after the overlay changed its + /// mind about this agent's address. + async fn ensure_addresses(&self, network: NetworkId) { + let wanted = { + let shared = self.lock_shared(); + match shared.networks.get(&network) { + Some(state) if state.device.is_some() => { + let wanted = self.desired_request(state, network); + if state.applied.as_ref() == Some(&wanted) { + return; + } + wanted + } + _ => return, + } + }; + + match self.tun_factory.reconfigure(wanted.clone()).await { + Ok(()) => { + let mut shared = self.lock_shared(); + if let Some(state) = shared.networks.get_mut(&network) { + state.applied = Some(wanted); + } + } + Err(err) => { + // Not fatal: the tunnels keep running on the addresses that + // are there, and the next reconciliation tries again. + tracing::warn!(%err, "cannot update the overlay interface addresses"); + self.report(network, err); + } + } + } + /// Brings the running tunnels in line with what is known. /// /// A peer gets a tunnel once both halves have arrived: its announcement, @@ -601,10 +647,17 @@ impl Worker { } /// Removes a network's interface and tunnels, keeping its key. - fn teardown(&self, network: NetworkId) { + async fn teardown(&self, network: NetworkId) { // Dropping the state drops the device, which stops its tasks and - // closes the packet interface. - self.lock_shared().networks.remove(&network); + // closes the packet interface. Closing it is already enough for the + // kernel to remove an interface this agent created; the explicit + // destroy makes that immediate and definite rather than dependent on + // the last reader letting go. + let removed = self.lock_shared().networks.remove(&network); + if let Some(state) = removed { + drop(state.device); + self.tun_factory.destroy(&state.interface).await; + } } fn known_networks(&self) -> Vec { @@ -649,12 +702,12 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { } Command::Teardown(network) => { pending.remove(&network); - worker.teardown(network); + worker.teardown(network).await; continue; } Command::Stop(reply) => { for network in worker.known_networks() { - worker.teardown(network); + worker.teardown(network).await; } let _ = reply.send(()); return; @@ -670,6 +723,7 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { }, if wait_until.is_some() => { deadline = None; for network in std::mem::take(&mut pending) { + worker.ensure_addresses(network).await; worker.sync(network); } } @@ -680,6 +734,7 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { if let Err(err) = worker.ensure_device(network).await { tracing::debug!(%err, "packet interface still unavailable"); } + worker.ensure_addresses(network).await; worker.sync(network); } } diff --git a/src/dataplane/wireguard/provision/factory.rs b/src/dataplane/wireguard/provision/factory.rs new file mode 100644 index 0000000..4fd1fc9 --- /dev/null +++ b/src/dataplane/wireguard/provision/factory.rs @@ -0,0 +1,201 @@ +//! The adapter between the plugin's view of a packet interface and the +//! host-management view. +//! +//! The plugin asks a [`TunFactory`] for a device and knows nothing else. This +//! factory answers by reconciling the host — creating the interface, fixing +//! up whatever an earlier run left behind, assigning the addresses — and +//! handing back the device that came out of it. + +use std::sync::Arc; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::super::config::Cidr; +use super::super::tun::{TunDevice, TunFactory, TunRequest}; +use super::{InterfacePlan, InterfaceProvisioner}; + +/// Turns a [`TunRequest`] into the plan for a host interface. +fn plan_for(request: &TunRequest) -> Result { + let mut addresses = vec![Cidr::new(request.address.into(), request.prefix_len)?]; + if let Some(address) = request.address_v4 { + addresses.push(Cidr::new(address.into(), request.prefix_len_v4)?); + } + Ok(InterfacePlan::new( + request.name.clone(), + request.mtu, + addresses, + )) +} + +/// A [`TunFactory`] backed by an [`InterfaceProvisioner`]. +#[derive(Debug)] +pub struct ManagedTunFactory { + provisioner: Arc, +} + +impl ManagedTunFactory { + /// Wraps a provisioner. + pub fn new(provisioner: Arc) -> Self { + Self { provisioner } + } + + /// The provisioner underneath. + pub fn provisioner(&self) -> &Arc { + &self.provisioner + } +} + +impl TunFactory for ManagedTunFactory { + fn name(&self) -> &str { + self.provisioner.name() + } + + fn create<'a>( + &'a self, + request: TunRequest, + ) -> BoxFuture<'a, Result, PluginError>> { + Box::pin(async move { + let plan = plan_for(&request)?; + let provisioned = self.provisioner.reconcile(&plan).await?; + tracing::info!( + interface = %plan.name, + changes = %provisioned.changes.summary(), + "overlay interface reconciled" + ); + provisioned.device.ok_or_else(|| { + // Reaching here would mean the interface already existed and + // was held open by us, which cannot be true on the path that + // creates a device. + PluginError::Other(format!( + "interface `{}` was reconciled but no device came back", + plan.name + )) + }) + }) + } + + fn reconfigure<'a>(&'a self, request: TunRequest) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + let plan = plan_for(&request)?; + let provisioned = self.provisioner.reconcile(&plan).await?; + if !provisioned.changes.is_empty() { + tracing::info!( + interface = %plan.name, + changes = %provisioned.changes.summary(), + "overlay interface updated" + ); + } + Ok(()) + }) + } + + fn destroy<'a>(&'a self, name: &'a str) -> BoxFuture<'a, ()> { + Box::pin(async move { + match self.provisioner.remove(name).await { + Ok(()) => tracing::info!(interface = %name, "overlay interface removed"), + Err(err) => { + tracing::warn!(interface = %name, %err, "cannot remove the overlay interface") + } + } + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use std::net::{Ipv4Addr, Ipv6Addr}; + + use super::super::{LinkKind, MockHost, MockProvisioner}; + use super::*; + + fn request(v4: Option) -> TunRequest { + TunRequest { + name: "tsunfactory".into(), + address: Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + prefix_len: 64, + address_v4: v4, + prefix_len_v4: 24, + mtu: 1280, + } + } + + #[tokio::test] + async fn creating_a_device_provisions_the_host_and_returns_it() { + let provisioner = Arc::new(MockProvisioner::default()); + let factory = ManagedTunFactory::new(provisioner.clone()); + + let device = factory + .create(request(Some(Ipv4Addr::new(10, 13, 37, 69)))) + .await + .unwrap(); + assert_eq!(device.name(), "tsunfactory"); + assert_eq!(device.mtu(), 1280); + + let state = provisioner.host().get("tsunfactory").unwrap(); + assert_eq!(state.kind, LinkKind::Tun); + assert_eq!(state.mtu, 1280); + assert_eq!(state.addresses.len(), 2, "both families are assigned"); + } + + #[tokio::test] + async fn a_reallocated_address_is_applied_to_the_live_interface() { + let provisioner = Arc::new(MockProvisioner::default()); + let factory = ManagedTunFactory::new(provisioner.clone()); + factory + .create(request(Some(Ipv4Addr::new(10, 13, 37, 69)))) + .await + .unwrap(); + + factory + .reconfigure(request(Some(Ipv4Addr::new(10, 13, 37, 70)))) + .await + .unwrap(); + + let state = provisioner.host().get("tsunfactory").unwrap(); + assert!(state.attached, "the interface was not recreated"); + let addresses: Vec = state + .addresses + .iter() + .map(|entry| entry.to_string()) + .collect(); + assert!( + addresses.contains(&"10.13.37.70/24".to_string()), + "{addresses:?}" + ); + assert!( + !addresses.contains(&"10.13.37.69/24".to_string()), + "{addresses:?}" + ); + } + + #[tokio::test] + async fn destroying_takes_the_interface_off_the_host() { + let provisioner = Arc::new(MockProvisioner::default()); + let factory = ManagedTunFactory::new(provisioner.clone()); + factory.create(request(None)).await.unwrap(); + + factory.destroy("tsunfactory").await; + assert!(provisioner.host().names().is_empty()); + } + + #[tokio::test] + async fn a_host_that_cannot_be_provisioned_fails_the_create() { + let host = MockHost::new(); + host.insert( + "tsunfactory", + super::super::InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: Vec::new(), + }, + ); + let factory = ManagedTunFactory::new(Arc::new(MockProvisioner::new(host))); + let err = factory.create(request(None)).await.unwrap_err(); + assert!(err.to_string().contains("bridge"), "{err}"); + } +} diff --git a/src/dataplane/wireguard/provision/linux.rs b/src/dataplane/wireguard/provision/linux.rs new file mode 100644 index 0000000..0748446 --- /dev/null +++ b/src/dataplane/wireguard/provision/linux.rs @@ -0,0 +1,605 @@ +//! Managing the overlay interface on Linux, through netlink. +//! +//! Everything the old printed recipe did — `ip tuntap add`, `ip link set`, +//! `ip address add` — happens here instead, in this process, over a netlink +//! socket. No `ip` binary is invoked, so nothing this module does can be +//! influenced by `PATH`, by a shell, or by anything a remote peer said. +//! +//! # The interface cleans up after itself +//! +//! The TUN interface is created by opening `/dev/net/tun` and is **not** +//! made persistent, so the kernel destroys it the moment the last file +//! descriptor closes. That covers the ordinary exit, a panic, a `SIGKILL` +//! and a power loss equally: there is no path by which a dead agent leaves an +//! interface behind, because keeping it alive is what needs an action, not +//! removing it. +//! +//! It also means the interface has carrier for its whole life, which removes +//! the two settings the manual recipe needed. `keep_addr_on_down` was only +//! needed because an interface nobody held open lost carrier and had its IPv6 +//! addresses flushed; `nodad` only because duplicate address detection cannot +//! finish without carrier. +//! +//! What can still be left behind is an interface from *before* this change — +//! one created persistent by the old recipe — or one from a run killed in the +//! window between `TUNSETIFF` and this module recording it. Those are found +//! at startup and replaced; see +//! [`plan_changes`](super::plan_changes) for the rules that decide it. +//! +//! # Threads +//! +//! Capabilities on Linux are per thread, and netlink checks the credentials +//! of whichever thread calls `sendmsg` — which, with an async netlink client, +//! is the connection task rather than the caller. Raising `CAP_NET_ADMIN` +//! around an `await` would therefore be both wrong and unsound in the "works +//! until the scheduler moves the task" sense. +//! +//! So all netlink work happens on one dedicated thread running a +//! current-thread runtime. Nothing is polled outside a `block_on`, the +//! capability is raised immediately before that call and lowered immediately +//! after, and the connection task lives and dies inside it. + +use std::net::IpAddr; +use std::sync::{Arc, Mutex, mpsc}; + +use futures_util::TryStreamExt; +// Through rtnetlink's own re-export, so the packet types can never drift out +// of step with the client that sends them. +use rtnetlink::packet_route::address::{AddressAttribute, AddressMessage}; +use rtnetlink::packet_route::link::{InfoKind, LinkAttribute, LinkFlags, LinkInfo, LinkMessage}; +use rtnetlink::{LinkMessageBuilder, LinkUnspec}; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::super::config::Cidr; +use super::super::tun::{TunDevice, TunRequest}; +use super::privilege::{NetAdmin, Privilege, probe_net_admin}; +use super::{ + InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned, plan_changes, +}; + +/// A request to the netlink thread. +enum Command { + /// What does this interface look like right now? + Observe(String, Reply), + /// Remove this interface. + Delete(String, Reply<()>), + /// Apply MTU, link state and addresses. + Configure(Box, Reply<()>), + /// Stop the thread. + Stop, +} + +type Reply = mpsc::Sender>; + +/// The configuration half of a reconciliation. +struct Configure { + name: String, + mtu: Option, + bring_up: bool, + add: Vec, + remove: Vec, +} + +/// Manages the overlay interface with netlink. +#[derive(Debug)] +pub struct NetlinkProvisioner { + commands: mpsc::Sender, + worker: Mutex>>, + /// Interfaces this process created, so they are adjusted rather than + /// replaced. See [`plan_changes`]. + ours: Mutex>, + /// Devices kept alive for as long as the interface should exist. Dropping + /// one is what removes the interface from the kernel. + held: Mutex)>>, +} + +impl NetlinkProvisioner { + /// Starts the netlink thread, after checking this process can use it. + pub fn new() -> Result { + match probe_net_admin() { + Privilege::Available => {} + Privilege::Missing(reason) => { + return Err(PluginError::Unavailable(format!( + "{reason}. {}", + Privilege::how_to_grant(¤t_program()) + ))); + } + Privilege::Unsupported => { + return Err(PluginError::Unavailable( + "interface management is not compiled in".to_string(), + )); + } + } + + let (commands, requests) = mpsc::channel(); + let worker = std::thread::Builder::new() + .name("tsunagi-netlink".to_string()) + .spawn(move || netlink_thread(requests)) + .map_err(|err| { + PluginError::Unavailable(format!("cannot start the netlink thread: {err}")) + })?; + + Ok(Self { + commands, + worker: Mutex::new(Some(worker)), + ours: Mutex::new(Vec::new()), + held: Mutex::new(Vec::new()), + }) + } + + fn call( + &self, + make: impl FnOnce(Reply) -> Command, + ) -> Result { + let (reply_tx, reply_rx) = mpsc::channel(); + self.commands + .send(make(reply_tx)) + .map_err(|_| PluginError::Unavailable("the netlink thread has stopped".to_string()))?; + reply_rx.recv().map_err(|_| { + PluginError::Unavailable("the netlink thread stopped mid-request".to_string()) + })? + } + + fn is_ours(&self, name: &str) -> bool { + lock(&self.ours).iter().any(|owned| owned == name) + } + + /// Opens the TUN interface, which is what creates it. + /// + /// Synchronous on purpose: the capability guard is raised and lowered + /// without an `await` in between, so it cannot outlive this thread. + fn create_device(&self, plan: &InterfacePlan) -> Result, PluginError> { + let request = TunRequest::bare(plan.name.clone(), plan.mtu); + let _guard = NetAdmin::acquire()?; + super::super::tun::open_tun(&request, false) + } +} + +impl Drop for NetlinkProvisioner { + fn drop(&mut self) { + let _ = self.commands.send(Command::Stop); + if let Some(worker) = lock(&self.worker).take() { + let _ = worker.join(); + } + } +} + +impl InterfaceProvisioner for NetlinkProvisioner { + fn name(&self) -> &str { + "netlink" + } + + fn reconcile<'a>( + &'a self, + plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let current = self.call(|reply| Command::Observe(plan.name.clone(), reply))?; + let changes = plan_changes(¤t, plan, self.is_ours(&plan.name))?; + + if changes.delete_link { + tracing::info!( + interface = %plan.name, + "removing an abandoned interface left by an earlier run" + ); + self.call(|reply| Command::Delete(plan.name.clone(), reply))?; + } + + let device = if changes.create_link { + let device = self.create_device(plan)?; + lock(&self.ours).push(plan.name.clone()); + lock(&self.held).push((plan.name.clone(), Arc::clone(&device))); + Some(device) + } else { + None + }; + + if changes.set_mtu.is_some() + || changes.bring_up + || !changes.add.is_empty() + || !changes.remove.is_empty() + { + let configure = Box::new(Configure { + name: plan.name.clone(), + mtu: changes.set_mtu, + bring_up: changes.bring_up, + add: changes.add.clone(), + remove: changes.remove.clone(), + }); + // A failure here leaves an interface that exists but cannot + // carry traffic, which is worse than none at all, so it is + // taken back down rather than left as a trap. + if let Err(err) = self.call(move |reply| Command::Configure(configure, reply)) { + if changes.create_link { + lock(&self.held).retain(|(held, _)| held != &plan.name); + lock(&self.ours).retain(|owned| owned != &plan.name); + } + return Err(err); + } + } + + Ok(Provisioned { changes, device }) + }) + } + + fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + // Dropping the device is what removes the interface; the explicit + // delete is only so it is gone by the time this returns rather + // than whenever the last reader lets go. + lock(&self.held).retain(|(held, _)| held != name); + lock(&self.ours).retain(|owned| owned != name); + self.call(|reply| Command::Delete(name.to_string(), reply)) + }) + } +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn current_program() -> String { + std::env::current_exe() + .ok() + .and_then(|path| path.to_str().map(str::to_string)) + .unwrap_or_else(|| "tsunagi".to_string()) +} + +/// The netlink thread. +/// +/// It owns a current-thread runtime, so nothing is polled except inside the +/// `block_on` below — which is what makes the capability window exact. +fn netlink_thread(requests: mpsc::Receiver) { + // A binary granted `cap_net_admin+ep` starts with the capability + // effective. Lower it immediately so that even this thread only has it + // during the calls that need it. + NetAdmin::lower(); + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(err) => { + // Answer every request with the reason rather than hanging. + while let Ok(command) = requests.recv() { + let message = format!("the netlink thread has no runtime: {err}"); + match command { + Command::Observe(_, reply) => { + let _ = reply.send(Err(PluginError::Unavailable(message))); + } + Command::Delete(_, reply) | Command::Configure(_, reply) => { + let _ = reply.send(Err(PluginError::Unavailable(message))); + } + Command::Stop => return, + } + } + return; + } + }; + + while let Ok(command) = requests.recv() { + match command { + Command::Stop => return, + // Reading the interface list needs no privilege at all. + Command::Observe(name, reply) => { + let _ = reply.send(runtime.block_on(observe(&name))); + } + Command::Delete(name, reply) => { + let result = NetAdmin::acquire().and_then(|guard| { + let result = runtime.block_on(delete_link(&name)); + drop(guard); + result + }); + let _ = reply.send(result); + } + Command::Configure(configure, reply) => { + let result = NetAdmin::acquire().and_then(|guard| { + let result = runtime.block_on(configure_link(&configure)); + drop(guard); + result + }); + let _ = reply.send(result); + } + } + } +} + +/// Opens a netlink connection and runs one unit of work over it. +async fn with_netlink(work: impl FnOnce(rtnetlink::Handle) -> F) -> Result +where + F: std::future::Future>, +{ + let (connection, handle, _messages) = rtnetlink::new_connection() + .map_err(|err| PluginError::Unavailable(format!("cannot open netlink: {err}")))?; + let pump = tokio::spawn(connection); + let result = work(handle).await; + pump.abort(); + result +} + +/// `fe80::/10`, which the kernel assigns on its own. +fn is_link_local(addr: IpAddr) -> bool { + match addr { + IpAddr::V4(addr) => addr.is_link_local(), + IpAddr::V6(addr) => (addr.segments()[0] & 0xffc0) == 0xfe80, + } +} + +/// Whether a name belongs to a TUN interface, from sysfs. +/// +/// A fallback for the case where the kernel does not report `IFLA_LINKINFO` +/// for the link. Reading sysfs needs no privileges. +fn is_tun_in_sysfs(name: &str) -> bool { + std::path::Path::new(&format!("/sys/class/net/{name}/tun_flags")).exists() +} + +fn link_kind(message: &LinkMessage, name: &str) -> LinkKind { + for attribute in &message.attributes { + if let LinkAttribute::LinkInfo(infos) = attribute { + for info in infos { + if let LinkInfo::Kind(kind) = info { + return match kind { + InfoKind::Tun => LinkKind::Tun, + other => LinkKind::Foreign(format!("{other:?}").to_lowercase()), + }; + } + } + } + } + if is_tun_in_sysfs(name) { + LinkKind::Tun + } else { + // No `IFLA_LINKINFO` and no `tun_flags`: a plain device such as an + // ethernet port. Unknown rather than ours, so it is left alone. + LinkKind::Foreign("non-tun".to_string()) + } +} + +async fn observe(name: &str) -> Result { + with_netlink(|handle| async move { + let mut links = handle.link().get().match_name(name.to_string()).execute(); + let message = match links.try_next().await { + Ok(Some(message)) => message, + Ok(None) => return Ok(InterfaceState::absent()), + Err(err) => { + // "No such device" is the expected answer on a clean host, so + // it is not an error; anything else is. + if !std::path::Path::new(&format!("/sys/class/net/{name}")).exists() { + return Ok(InterfaceState::absent()); + } + return Err(PluginError::Unavailable(format!( + "cannot read interface `{name}`: {err}" + ))); + } + }; + + let index = message.header.index; + let flags = message.header.flags; + let mtu = message + .attributes + .iter() + .find_map(|attribute| match attribute { + LinkAttribute::Mtu(mtu) => Some(*mtu), + _ => None, + }) + .unwrap_or(0); + + let mut addresses = Vec::new(); + let mut stream = handle + .address() + .get() + .set_link_index_filter(index) + .execute(); + while let Some(message) = stream.try_next().await.map_err(|err| { + PluginError::Unavailable(format!("cannot read the addresses of `{name}`: {err}")) + })? { + if let Some(cidr) = address_of(&message) + && !is_link_local(cidr.addr) + { + addresses.push(cidr); + } + } + addresses.sort(); + + Ok(InterfaceState { + kind: link_kind(&message, name), + // `IFF_LOWER_UP` is carrier, and a TUN has carrier exactly while + // a process holds it open. + attached: flags.contains(LinkFlags::LowerUp), + up: flags.contains(LinkFlags::Up), + mtu, + addresses, + }) + }) + .await +} + +/// The address a message carries, preferring `IFA_LOCAL`. +/// +/// For a point-to-point interface `IFA_ADDRESS` is the *peer* address, so +/// taking it would compare the wrong thing. +fn address_of(message: &AddressMessage) -> Option { + let mut address = None; + for attribute in &message.attributes { + match attribute { + AddressAttribute::Local(addr) => return cidr(*addr, message.header.prefix_len), + AddressAttribute::Address(addr) => address = Some(*addr), + _ => {} + } + } + address.and_then(|addr| cidr(addr, message.header.prefix_len)) +} + +fn cidr(addr: IpAddr, prefix_len: u8) -> Option { + Cidr::new(addr, prefix_len).ok() +} + +async fn delete_link(name: &str) -> Result<(), PluginError> { + with_netlink(|handle| async move { + let mut links = handle.link().get().match_name(name.to_string()).execute(); + let index = match links.try_next().await { + Ok(Some(message)) => message.header.index, + // Already gone, which is the outcome asked for. + Ok(None) | Err(_) => return Ok(()), + }; + handle.link().del(index).execute().await.map_err(|err| { + PluginError::Unavailable(format!("cannot remove interface `{name}`: {err}")) + }) + }) + .await +} + +async fn configure_link(configure: &Configure) -> Result<(), PluginError> { + let name = configure.name.as_str(); + with_netlink(|handle| async move { + let mut links = handle.link().get().match_name(name.to_string()).execute(); + let index = links + .try_next() + .await + .ok() + .flatten() + .map(|message| message.header.index) + .ok_or_else(|| { + PluginError::Unavailable(format!( + "interface `{name}` disappeared before it could be configured" + )) + })?; + + if configure.mtu.is_some() || configure.bring_up { + let mut builder = LinkMessageBuilder::::new().index(index); + if let Some(mtu) = configure.mtu { + builder = builder.mtu(mtu); + } + if configure.bring_up { + builder = builder.up(); + } + handle + .link() + .set(builder.build()) + .execute() + .await + .map_err(|err| { + PluginError::Unavailable(format!("cannot configure interface `{name}`: {err}")) + })?; + } + + for cidr in &configure.remove { + // Delete the exact message the kernel holds rather than a + // reconstruction of it, so the family and flags always match. + let mut stream = handle + .address() + .get() + .set_link_index_filter(index) + .execute(); + let mut target: Option = None; + while let Ok(Some(message)) = stream.try_next().await { + if address_of(&message) == Some(*cidr) { + target = Some(message); + break; + } + } + drop(stream); + if let Some(message) = target { + handle + .address() + .del(message) + .execute() + .await + .map_err(|err| { + PluginError::Unavailable(format!( + "cannot remove {cidr} from interface `{name}`: {err}" + )) + })?; + } + } + + for cidr in &configure.add { + handle + .address() + .add(index, cidr.addr, cidr.prefix_len) + .execute() + .await + .map_err(|err| { + PluginError::Unavailable(format!( + "cannot add {cidr} to interface `{name}`: {err}" + )) + })?; + } + + Ok(()) + }) + .await +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[test] + fn link_local_addresses_are_not_ours_to_manage() { + // The kernel assigns these itself when the link comes up. Treating + // them as unplanned would make every reconciliation try to delete one. + assert!(is_link_local(IpAddr::V6(Ipv6Addr::new( + 0xfe80, 0, 0, 0, 0, 0, 0, 1 + )))); + assert!(is_link_local(IpAddr::V4(Ipv4Addr::new(169, 254, 1, 1)))); + assert!(!is_link_local(IpAddr::V6(Ipv6Addr::new( + 0xfd00, 0, 0, 0, 0, 0, 0, 1 + )))); + assert!(!is_link_local(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 1)))); + } + + #[tokio::test] + async fn observing_a_name_nothing_uses_reports_it_absent() { + let state = observe("tsunagi-no-such-interface").await.unwrap(); + assert_eq!(state.kind, LinkKind::Absent); + assert!(state.addresses.is_empty()); + } + + #[tokio::test] + async fn observing_the_loopback_interface_decodes_what_the_kernel_reports() { + // Reading the interface table needs no privileges, so this runs + // everywhere and checks the netlink decoding against a real kernel + // rather than against a fixture. + let state = observe("lo").await.unwrap(); + assert!( + matches!(state.kind, LinkKind::Foreign(_)), + "loopback is not a tun: {:?}", + state.kind + ); + assert!(state.up, "loopback is up"); + assert!(state.mtu >= 1280, "decoded an mtu: {}", state.mtu); + assert!( + state + .addresses + .iter() + .any(|cidr| cidr.addr == IpAddr::V4(Ipv4Addr::LOCALHOST)), + "127.0.0.1 is decoded: {:?}", + state.addresses + ); + assert!( + !state.addresses.iter().any(|cidr| is_link_local(cidr.addr)), + "link-local addresses are filtered out: {:?}", + state.addresses + ); + } + + #[test] + fn an_interface_that_is_not_a_tun_is_reported_foreign() { + let message = LinkMessage::default(); + // No `IFLA_LINKINFO` and a name with no `tun_flags` in sysfs. + assert!(matches!( + link_kind(&message, "definitely-not-an-interface"), + LinkKind::Foreign(_) + )); + } +} diff --git a/src/dataplane/wireguard/provision/mock.rs b/src/dataplane/wireguard/provision/mock.rs new file mode 100644 index 0000000..2a4b720 --- /dev/null +++ b/src/dataplane/wireguard/provision/mock.rs @@ -0,0 +1,336 @@ +//! An in-memory host, so provisioning is tested without touching this one. +//! +//! [`MockHost`] is a pretend `/sys/class/net`: a test can seed it with the +//! leftovers of a crashed run, or with somebody else's bridge, then check +//! what the provisioner did about it. It is also what the platforms that have +//! no provisioner yet are wired to in their own tests. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::super::config::Cidr; +use super::super::tun::{MemoryTun, MemoryTunFactory, TunFactory, TunRequest}; +use super::{ + Changes, InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned, + plan_changes, +}; + +/// A pretend host with interfaces on it. +#[derive(Debug, Clone, Default)] +pub struct MockHost { + links: Arc>>, +} + +impl MockHost { + /// An empty host. + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + match self.links.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + /// Puts an interface on the host. + pub fn insert(&self, name: impl Into, state: InterfaceState) { + self.lock().insert(name.into(), state); + } + + /// Seeds the leftovers of a run that died: a TUN nobody holds open, still + /// carrying whatever addresses it had. + pub fn insert_stale_tun(&self, name: impl Into, addresses: Vec) { + self.insert( + name, + InterfaceState { + kind: LinkKind::Tun, + attached: false, + up: true, + mtu: 1280, + addresses, + }, + ); + } + + /// The state of an interface, if it exists. + pub fn get(&self, name: &str) -> Option { + self.lock().get(name).cloned() + } + + /// The names currently on the host. + pub fn names(&self) -> Vec { + let mut names: Vec = self.lock().keys().cloned().collect(); + names.sort(); + names + } +} + +/// Applies plans to a [`MockHost`]. +#[derive(Debug, Clone)] +pub struct MockProvisioner { + host: MockHost, + ours: Arc>>, + /// The devices handed out, so a test can drive packets through them. + devices: MemoryTunFactory, + /// Set to fail every call, to exercise the error path. + failure: Option, +} + +impl Default for MockProvisioner { + fn default() -> Self { + Self::new(MockHost::new()) + } +} + +impl MockProvisioner { + /// A provisioner over a host. + pub fn new(host: MockHost) -> Self { + Self { + host, + ours: Arc::new(Mutex::new(Vec::new())), + devices: MemoryTunFactory::new(), + failure: None, + } + } + + /// A provisioner that refuses everything, with this reason. + pub fn failing(reason: impl Into) -> Self { + Self { + host: MockHost::new(), + ours: Arc::new(Mutex::new(Vec::new())), + devices: MemoryTunFactory::new(), + failure: Some(reason.into()), + } + } + + /// The host it applies to. + pub fn host(&self) -> &MockHost { + &self.host + } + + /// The device created for an interface name, if any. + pub fn device(&self, name: &str) -> Option> { + self.devices.device(name) + } + + fn owned(&self) -> std::sync::MutexGuard<'_, Vec> { + match self.ours.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + fn apply(&self, plan: &InterfacePlan) -> Result { + if let Some(reason) = &self.failure { + return Err(PluginError::Unavailable(reason.clone())); + } + + let ours = self.owned().iter().any(|name| name == &plan.name); + let current = self + .host + .get(&plan.name) + .unwrap_or_else(InterfaceState::absent); + let changes = plan_changes(¤t, plan, ours)?; + + let mut state = current; + if changes.delete_link { + self.host.lock().remove(&plan.name); + state = InterfaceState::absent(); + } + if changes.create_link { + state = InterfaceState { + kind: LinkKind::Tun, + // Creating it means holding it open, so it has carrier. + attached: true, + up: false, + mtu: 1500, + addresses: Vec::new(), + }; + self.owned().push(plan.name.clone()); + } + if let Some(mtu) = changes.set_mtu { + state.mtu = mtu; + } + if changes.bring_up { + state.up = true; + } + state + .addresses + .retain(|addr| !changes.remove.contains(addr)); + state.addresses.extend(changes.add.iter().copied()); + state.addresses.sort(); + state.addresses.dedup(); + self.host.insert(plan.name.clone(), state); + + Ok(changes) + } +} + +impl InterfaceProvisioner for MockProvisioner { + fn name(&self) -> &str { + "mock" + } + + fn reconcile<'a>( + &'a self, + plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let changes = self.apply(plan)?; + let device = if changes.create_link { + Some( + self.devices + .create(TunRequest::bare(plan.name.clone(), plan.mtu)) + .await?, + ) + } else { + None + }; + Ok(Provisioned { changes, device }) + }) + } + + fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + self.host.lock().remove(name); + self.owned().retain(|owned| owned != name); + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + fn v6() -> Cidr { + Cidr { + addr: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1)), + prefix_len: 64, + } + } + + fn v4(last: u8) -> Cidr { + Cidr { + addr: IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)), + prefix_len: 24, + } + } + + fn plan(addresses: Vec) -> InterfacePlan { + InterfacePlan::new("tsunmock", 1280, addresses) + } + + #[tokio::test] + async fn reconciling_an_empty_host_creates_a_configured_interface() { + let provisioner = MockProvisioner::default(); + let plan = plan(vec![v6(), v4(69)]); + let changes = provisioner.reconcile(&plan).await.unwrap().changes; + assert!(changes.create_link); + + let state = provisioner.host().get("tsunmock").unwrap(); + assert_eq!(state.kind, LinkKind::Tun); + assert!(state.up); + assert_eq!(state.mtu, 1280); + assert_eq!(state.addresses, vec![v4(69), v6()]); + } + + #[tokio::test] + async fn reconciling_is_idempotent() { + let provisioner = MockProvisioner::default(); + let plan = plan(vec![v6(), v4(69)]); + provisioner.reconcile(&plan).await.unwrap(); + let second = provisioner.reconcile(&plan).await.unwrap().changes; + assert!(second.is_empty(), "{second:?}"); + } + + #[tokio::test] + async fn a_crashed_run_is_repaired_on_the_next_start() { + let host = MockHost::new(); + // What the previous run left: the interface, with the address it had + // been allocated back then. + host.insert_stale_tun("tsunmock", vec![v4(178)]); + let provisioner = MockProvisioner::new(host); + + let changes = provisioner + .reconcile(&plan(vec![v6(), v4(69)])) + .await + .unwrap() + .changes; + assert!(changes.delete_link && changes.create_link); + + let state = provisioner.host().get("tsunmock").unwrap(); + assert_eq!( + state.addresses, + vec![v4(69), v6()], + "the stale address is gone and the current one is there" + ); + assert!(state.attached, "the new interface is held open by us"); + } + + #[tokio::test] + async fn a_changed_allocation_is_applied_without_recreating_the_interface() { + let provisioner = MockProvisioner::default(); + provisioner + .reconcile(&plan(vec![v6(), v4(69)])) + .await + .unwrap(); + + // The overlay agreed on a different address for us while running. + let changes = provisioner + .reconcile(&plan(vec![v6(), v4(70)])) + .await + .unwrap() + .changes; + assert!( + !changes.delete_link && !changes.create_link, + "recreating would drop every tunnel" + ); + assert_eq!(changes.add, vec![v4(70)]); + assert_eq!(changes.remove, vec![v4(69)]); + assert_eq!( + provisioner.host().get("tsunmock").unwrap().addresses, + vec![v4(70), v6()] + ); + } + + #[tokio::test] + async fn removing_takes_the_interface_off_the_host_and_is_idempotent() { + let provisioner = MockProvisioner::default(); + provisioner.reconcile(&plan(vec![v6()])).await.unwrap(); + assert_eq!(provisioner.host().names(), vec!["tsunmock".to_string()]); + + provisioner.remove("tsunmock").await.unwrap(); + assert!(provisioner.host().names().is_empty()); + provisioner.remove("tsunmock").await.unwrap(); + } + + #[tokio::test] + async fn a_foreign_interface_makes_reconciling_fail_and_changes_nothing() { + let host = MockHost::new(); + host.insert( + "tsunmock", + InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: vec![v4(1)], + }, + ); + let provisioner = MockProvisioner::new(host); + assert!(provisioner.reconcile(&plan(vec![v6()])).await.is_err()); + + let state = provisioner.host().get("tsunmock").unwrap(); + assert_eq!(state.kind, LinkKind::Foreign("bridge".into())); + assert_eq!(state.addresses, vec![v4(1)], "left exactly as it was"); + } +} diff --git a/src/dataplane/wireguard/provision/mod.rs b/src/dataplane/wireguard/provision/mod.rs new file mode 100644 index 0000000..f5d93b9 --- /dev/null +++ b/src/dataplane/wireguard/provision/mod.rs @@ -0,0 +1,440 @@ +//! Bringing a host interface into the state the overlay needs. +//! +//! The agent used to print a list of `ip` commands and ask a human to run +//! them. That is fragile in exactly the way hand-held setup always is: the +//! interface does not survive a reboot, a changed address allocation needs +//! another manual round, and a crashed run leaves a half-configured interface +//! that the next run then trips over. +//! +//! So the agent does it itself, and the shape of that is a *reconciliation*: +//! it is handed an [`InterfacePlan`] describing what the interface should look +//! like, it observes what is actually there, and it applies the difference. +//! Running it twice changes nothing the second time, and running it after a +//! crash repairs whatever was left behind. +//! +//! # Why this is a trait +//! +//! Every platform does this differently — netlink on Linux, `SystemConfiguration` +//! on macOS, the Windows IP Helper API — while the *decision* of what to change +//! is the same everywhere. So the decision lives in [`plan_changes`], which is +//! pure and tested on every platform, and only the execution is behind +//! [`InterfaceProvisioner`]. +//! +//! Three implementations: +//! +//! * `NetlinkProvisioner` on Linux, which needs `CAP_NET_ADMIN`. +//! * [`MockProvisioner`], an in-memory host used by the tests. +//! * [`UnsupportedProvisioner`] elsewhere, which fails with an explanation +//! and a pointer at the manual route rather than pretending to work. +//! +//! # What it is not allowed to do +//! +//! Nothing here takes a name, an address or a command from the network. The +//! interface name is derived from the network id, the addresses come from the +//! local plugin, and an interface this agent did not create is never deleted +//! or reconfigured — see [`plan_changes`]. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::config::Cidr; +use super::tun::TunDevice; + +mod mock; +pub use mock::{MockHost, MockProvisioner}; + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +mod linux; +#[cfg(all(feature = "tun-device", target_os = "linux"))] +pub use linux::NetlinkProvisioner; + +mod privilege; +pub use privilege::{Privilege, probe_net_admin}; + +mod unsupported; +pub use unsupported::UnsupportedProvisioner; + +mod factory; +pub use factory::ManagedTunFactory; + +/// What an interface should look like. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InterfacePlan { + /// Interface name. Derived from the network id, never from the network. + pub name: String, + /// Interface MTU. + pub mtu: u32, + /// Every address the interface should carry, and no others. + pub addresses: Vec, +} + +impl InterfacePlan { + /// Builds a plan, normalising the address list. + pub fn new(name: impl Into, mtu: u32, addresses: Vec) -> Self { + let unique: BTreeSet = addresses.into_iter().collect(); + Self { + name: name.into(), + mtu, + addresses: unique.into_iter().collect(), + } + } +} + +/// What kind of link is sitting on a name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LinkKind { + /// Nothing is there. + Absent, + /// A TUN interface. + Tun, + /// Something else entirely — a bridge, a physical device, a VPN from + /// another program. Never ours to touch. + Foreign(String), +} + +/// What an interface currently looks like. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InterfaceState { + /// What kind of link holds the name. + pub kind: LinkKind, + /// Whether a process is attached to the TUN, which the kernel reports as + /// carrier. A leftover interface from a crashed run has none. + pub attached: bool, + /// Whether the link is administratively up. + pub up: bool, + /// The current MTU. + pub mtu: u32, + /// The addresses currently assigned. + pub addresses: Vec, +} + +impl InterfaceState { + /// The state of a name nothing is using. + pub fn absent() -> Self { + Self { + kind: LinkKind::Absent, + attached: false, + up: false, + mtu: 0, + addresses: Vec::new(), + } + } +} + +/// The steps that turn an [`InterfaceState`] into an [`InterfacePlan`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Changes { + /// Remove a stale interface left behind by an earlier run. + pub delete_link: bool, + /// Create the interface. + pub create_link: bool, + /// Set the MTU, when it is not already right. + pub set_mtu: Option, + /// Bring the link up. + pub bring_up: bool, + /// Addresses to add. + pub add: Vec, + /// Addresses to remove, because the plan no longer contains them. + pub remove: Vec, +} + +impl Changes { + /// Whether anything at all needs doing. + pub fn is_empty(&self) -> bool { + !self.delete_link + && !self.create_link + && self.set_mtu.is_none() + && !self.bring_up + && self.add.is_empty() + && self.remove.is_empty() + } + + /// A one-line summary for a log line. + pub fn summary(&self) -> String { + if self.is_empty() { + return "already as planned".to_string(); + } + let mut parts = Vec::new(); + if self.delete_link { + parts.push("remove a stale interface".to_string()); + } + if self.create_link { + parts.push("create the interface".to_string()); + } + if let Some(mtu) = self.set_mtu { + parts.push(format!("set mtu {mtu}")); + } + if self.bring_up { + parts.push("bring it up".to_string()); + } + for address in &self.add { + parts.push(format!("add {address}")); + } + for address in &self.remove { + parts.push(format!("remove {address}")); + } + parts.join(", ") + } +} + +/// Decides what to change, or refuses. +/// +/// `ours` says whether this process created the interface in its current run. +/// It is the whole reason this can be safe: an interface we made is adjusted +/// in place, and an interface we did not make is only ever *replaced* when it +/// is plainly abandoned. +/// +/// The refusals matter more than the changes: +/// +/// * A link that is not a TUN is never touched. Deriving the name from the +/// network id makes a collision with a real device unlikely, not impossible, +/// and destroying somebody's bridge because it happened to share a name is +/// not a recoverable mistake. +/// * A TUN with a process attached to it is never deleted. It is somebody +/// else's working interface — most likely another agent on this host — and +/// yanking it out from under them would break a running overlay. +/// +/// What is left is a TUN with nothing attached, which is precisely the +/// footprint of a run that died: those are removed and rebuilt. +pub fn plan_changes( + current: &InterfaceState, + plan: &InterfacePlan, + ours: bool, +) -> Result { + let mut changes = Changes::default(); + + match ¤t.kind { + LinkKind::Foreign(kind) => { + return Err(PluginError::Unavailable(format!( + "`{}` already exists and is a {kind} interface, not one of ours. \ + Refusing to touch it. Run with a different interface prefix.", + plan.name + ))); + } + LinkKind::Tun if !ours && current.attached => { + return Err(PluginError::Unavailable(format!( + "`{}` already exists and another process is attached to it. \ + That is most likely a second agent on this host in the same \ + network; give one of them a different interface prefix.", + plan.name + ))); + } + LinkKind::Tun if !ours => { + // Abandoned: a TUN with no carrier is one nothing holds open. It + // is either a leftover from a run that died or an interface made + // by the old manual recipe. Either way it is replaced, which also + // discards whatever stale addresses it carried. + changes.delete_link = true; + changes.create_link = true; + } + LinkKind::Tun => {} + LinkKind::Absent => changes.create_link = true, + } + + if changes.create_link { + // A fresh interface starts down, with the kernel default MTU and no + // addresses, so everything in the plan has to be applied. + changes.set_mtu = Some(plan.mtu); + changes.bring_up = true; + changes.add = plan.addresses.clone(); + return Ok(changes); + } + + if current.mtu != plan.mtu { + changes.set_mtu = Some(plan.mtu); + } + if !current.up { + changes.bring_up = true; + } + + let wanted: BTreeSet = plan.addresses.iter().copied().collect(); + let present: BTreeSet = current.addresses.iter().copied().collect(); + changes.add = wanted.difference(&present).copied().collect(); + changes.remove = present.difference(&wanted).copied().collect(); + + Ok(changes) +} + +/// The result of a reconciliation. +pub struct Provisioned { + /// What was changed to get here. + pub changes: Changes, + /// The packet interface, when this call is what created it. + /// + /// Creating the interface and configuring it are the same privileged act + /// and belong together, so the provisioner owns both. A reconciliation + /// that only adjusted an interface already in place returns `None`. + pub device: Option>, +} + +impl std::fmt::Debug for Provisioned { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Provisioned") + .field("changes", &self.changes) + .field("device", &self.device.as_ref().map(|device| device.name())) + .finish() + } +} + +/// Applies an [`InterfacePlan`] to the host. +/// +/// Implementations are expected to be idempotent: calling [`reconcile`] twice +/// with the same plan changes nothing the second time. +/// +/// [`reconcile`]: InterfaceProvisioner::reconcile +pub trait InterfaceProvisioner: Send + Sync + std::fmt::Debug + 'static { + /// A short name used in diagnostics. + fn name(&self) -> &str; + + /// Brings the interface in line with the plan, and reports what it did. + fn reconcile<'a>( + &'a self, + plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result>; + + /// Removes an interface this provisioner created. + /// + /// Removing one that is already gone succeeds: this runs on the shutdown + /// path, where the interface having vanished is the desired outcome. + fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), PluginError>>; +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + fn v6(last: u16) -> Cidr { + Cidr { + addr: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, last)), + prefix_len: 64, + } + } + + fn v4(last: u8) -> Cidr { + Cidr { + addr: IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)), + prefix_len: 24, + } + } + + fn plan() -> InterfacePlan { + InterfacePlan::new("tsuntest", 1280, vec![v6(1), v4(69)]) + } + + #[test] + fn an_absent_interface_is_created_and_fully_configured() { + let changes = plan_changes(&InterfaceState::absent(), &plan(), false).unwrap(); + assert!(changes.create_link); + assert!(!changes.delete_link); + assert_eq!(changes.set_mtu, Some(1280)); + assert!(changes.bring_up); + assert_eq!(changes.add, vec![v4(69), v6(1)]); + assert!(changes.remove.is_empty()); + } + + #[test] + fn an_abandoned_tun_from_a_crashed_run_is_replaced() { + // The footprint of a run that died: the interface is still there, it + // still has its old addresses, and nothing holds it open. + let current = InterfaceState { + kind: LinkKind::Tun, + attached: false, + up: true, + mtu: 1280, + addresses: vec![v4(178)], + }; + let changes = plan_changes(¤t, &plan(), false).unwrap(); + assert!(changes.delete_link, "the stale interface goes"); + assert!(changes.create_link); + // Replacing it discards the stale address, so it need not be removed + // one by one. + assert_eq!(changes.add, vec![v4(69), v6(1)]); + assert!(changes.remove.is_empty()); + } + + #[test] + fn a_foreign_interface_is_never_touched() { + let current = InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: vec![v4(1)], + }; + let err = plan_changes(¤t, &plan(), false).unwrap_err(); + let message = err.to_string(); + assert!(message.contains("bridge"), "{message}"); + assert!(message.contains("Refusing to touch it"), "{message}"); + } + + #[test] + fn a_tun_another_process_is_using_is_never_deleted() { + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: true, + mtu: 1280, + addresses: vec![v6(1)], + }; + let err = plan_changes(¤t, &plan(), false).unwrap_err(); + assert!(err.to_string().contains("another process"), "{err}"); + } + + #[test] + fn our_own_interface_is_adjusted_in_place() { + // The live case: our address allocation changed while running. The + // interface must not be recreated, or every tunnel on it would drop. + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: true, + mtu: 1280, + addresses: vec![v6(1), v4(178)], + }; + let changes = plan_changes(¤t, &plan(), true).unwrap(); + assert!(!changes.delete_link && !changes.create_link); + assert_eq!(changes.add, vec![v4(69)]); + assert_eq!(changes.remove, vec![v4(178)]); + } + + #[test] + fn reconciling_an_interface_that_already_matches_changes_nothing() { + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: true, + mtu: 1280, + addresses: vec![v4(69), v6(1)], + }; + let changes = plan_changes(¤t, &plan(), true).unwrap(); + assert!(changes.is_empty(), "{changes:?}"); + assert_eq!(changes.summary(), "already as planned"); + } + + #[test] + fn a_wrong_mtu_or_a_down_link_is_corrected() { + let current = InterfaceState { + kind: LinkKind::Tun, + attached: true, + up: false, + mtu: 1500, + addresses: vec![v4(69), v6(1)], + }; + let changes = plan_changes(¤t, &plan(), true).unwrap(); + assert_eq!(changes.set_mtu, Some(1280)); + assert!(changes.bring_up); + assert!(changes.add.is_empty() && changes.remove.is_empty()); + } + + #[test] + fn a_plan_deduplicates_and_orders_its_addresses() { + let plan = InterfacePlan::new("x", 1280, vec![v6(1), v4(69), v6(1)]); + assert_eq!(plan.addresses, vec![v4(69), v6(1)]); + } +} diff --git a/src/dataplane/wireguard/provision/privilege.rs b/src/dataplane/wireguard/provision/privilege.rs new file mode 100644 index 0000000..882ada2 --- /dev/null +++ b/src/dataplane/wireguard/provision/privilege.rs @@ -0,0 +1,169 @@ +//! Holding `CAP_NET_ADMIN` for as short a time as possible. +//! +//! Creating a TUN interface and assigning addresses to it needs +//! `CAP_NET_ADMIN`, and there is no way around that on Linux. What *is* in +//! our control is how long the process can actually use it. +//! +//! Linux splits capabilities into sets. The *permitted* set is what a process +//! may use; the *effective* set is what it may use **right now**. A process +//! can lower a capability out of effective and raise it back later, but it +//! can never add to permitted. So the agent keeps `CAP_NET_ADMIN` out of the +//! effective set and raises it only around the handful of netlink calls that +//! need it, which is a few milliseconds at startup and again whenever the +//! address allocation changes. +//! +//! Grant it with: +//! +//! ```text +//! sudo setcap cap_net_admin+p /usr/local/bin/tsunagi +//! ``` +//! +//! `+p` rather than `+ep`: with `+p` the capability is permitted but not +//! effective at exec, which is exactly the resting state this module wants. +//! `+ep` also works — [`NetAdmin::acquire`] lowers it on the way in. +//! +//! # Capabilities are per thread +//! +//! `capset` affects the calling thread only, so raising one inside an async +//! block would be a bug the moment the task migrated to another worker. Every +//! caller here runs on a single dedicated thread; see +//! [`linux`](super::linux). + +/// Whether this process can configure interfaces. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Privilege { + /// `CAP_NET_ADMIN` is available. The agent manages the interface itself. + Available, + /// It is not, with a description of what was found. + Missing(String), + /// This platform has no provisioner yet. + Unsupported, +} + +impl Privilege { + /// Whether interfaces can be managed. + pub fn is_available(&self) -> bool { + matches!(self, Privilege::Available) + } + + /// How to obtain it, for a diagnostic. + pub fn how_to_grant(program: &str) -> String { + format!( + "Grant it once with `sudo setcap cap_net_admin+p {program}` \ + and the agent manages its own interface. Without it, prepare the \ + interface by hand with `tsunagi tun-setup`." + ) + } +} + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +pub use linux_impl::{NetAdmin, probe_net_admin}; + +#[cfg(not(all(feature = "tun-device", target_os = "linux")))] +pub use other_impl::probe_net_admin; + +#[cfg(not(all(feature = "tun-device", target_os = "linux")))] +mod other_impl { + use super::Privilege; + + /// Whether this process can configure interfaces. + pub fn probe_net_admin() -> Privilege { + Privilege::Unsupported + } +} + +#[cfg(all(feature = "tun-device", target_os = "linux"))] +mod linux_impl { + use caps::{CapSet, Capability}; + + use super::Privilege; + use crate::dataplane::PluginError; + + /// Whether this thread holds `CAP_NET_ADMIN` in its permitted set. + pub fn probe_net_admin() -> Privilege { + match caps::has_cap(None, CapSet::Permitted, Capability::CAP_NET_ADMIN) { + Ok(true) => Privilege::Available, + Ok(false) => Privilege::Missing( + "this process does not hold CAP_NET_ADMIN, so it cannot create \ + or configure a network interface" + .to_string(), + ), + Err(err) => { + Privilege::Missing(format!("cannot read this process's capabilities: {err}")) + } + } + } + + /// `CAP_NET_ADMIN`, raised for as long as this value is alive. + /// + /// Dropping it lowers the capability again, including on the error paths, + /// which is the point of it being a guard rather than a pair of calls. + #[derive(Debug)] + pub struct NetAdmin { + /// Whether this guard is the one that raised it, and so the one that + /// must lower it. Nested acquisition leaves the inner guard inert. + raised: bool, + } + + impl NetAdmin { + /// Raises `CAP_NET_ADMIN` into the effective set. + pub fn acquire() -> Result { + let already = caps::has_cap(None, CapSet::Effective, Capability::CAP_NET_ADMIN) + .map_err(|err| { + PluginError::Unavailable(format!("cannot read capabilities: {err}")) + })?; + if already { + return Ok(Self { raised: false }); + } + caps::raise(None, CapSet::Effective, Capability::CAP_NET_ADMIN).map_err(|err| { + PluginError::Unavailable(format!( + "cannot raise CAP_NET_ADMIN: {err}. {}", + Privilege::how_to_grant("tsunagi") + )) + })?; + Ok(Self { raised: true }) + } + + /// Lowers `CAP_NET_ADMIN` out of the effective set of this thread. + /// + /// Called on the way in as well as on the way out, so that a binary + /// granted `cap_net_admin+ep` — which starts with it effective — still + /// spends almost all of its life unable to use it. + pub fn lower() { + let _ = caps::drop(None, CapSet::Effective, Capability::CAP_NET_ADMIN); + } + } + + impl Drop for NetAdmin { + fn drop(&mut self) { + if self.raised { + Self::lower(); + } + } + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + #[test] + fn the_grant_instructions_name_the_program() { + let text = Privilege::how_to_grant("/usr/local/bin/tsunagi"); + assert!(text.contains("setcap cap_net_admin+p /usr/local/bin/tsunagi")); + assert!(text.contains("tun-setup"), "the fallback is offered too"); + } + + #[test] + fn probing_says_something_definite_about_this_host() { + // Whatever the answer is, it must be one of the three, and a missing + // capability must come with a reason rather than a bare `false`. + match probe_net_admin() { + Privilege::Available => {} + Privilege::Missing(reason) => assert!(!reason.is_empty()), + Privilege::Unsupported => {} + } + } +} diff --git a/src/dataplane/wireguard/provision/unsupported.rs b/src/dataplane/wireguard/provision/unsupported.rs new file mode 100644 index 0000000..e630f96 --- /dev/null +++ b/src/dataplane/wireguard/provision/unsupported.rs @@ -0,0 +1,80 @@ +//! The provisioner for platforms that do not have one yet. +//! +//! macOS and Windows both need real work here — `utun` plus the +//! `SystemConfiguration` framework on one, the IP Helper API and a Wintun +//! adapter on the other — and neither is written. Rather than let the agent +//! come up and fail obscurely at the first packet, this refuses at the point +//! of provisioning and says what to do instead. + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +use super::{InterfacePlan, InterfaceProvisioner, Provisioned}; + +/// Refuses to provision, with an explanation. +#[derive(Debug, Clone)] +pub struct UnsupportedProvisioner { + platform: &'static str, +} + +impl Default for UnsupportedProvisioner { + fn default() -> Self { + Self::new() + } +} + +impl UnsupportedProvisioner { + /// A provisioner naming the platform it is standing in for. + pub fn new() -> Self { + Self { + platform: std::env::consts::OS, + } + } + + fn refusal(&self) -> PluginError { + PluginError::Unavailable(format!( + "managing the overlay interface is not implemented on {} yet. \ + Prepare the interface by hand — `tsunagi tun-setup` prints what to run — \ + and the agent will attach to it.", + self.platform + )) + } +} + +impl InterfaceProvisioner for UnsupportedProvisioner { + fn name(&self) -> &str { + "unsupported" + } + + fn reconcile<'a>( + &'a self, + _plan: &'a InterfacePlan, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { Err(self.refusal()) }) + } + + fn remove<'a>(&'a self, _name: &'a str) -> BoxFuture<'a, Result<(), PluginError>> { + // Nothing was ever created, so there is nothing to clean up and no + // reason to fail a shutdown path. + Box::pin(async move { Ok(()) }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + #[tokio::test] + async fn it_refuses_with_a_usable_pointer_and_still_cleans_up_quietly() { + let provisioner = UnsupportedProvisioner::new(); + let plan = InterfacePlan::new("tsuntest", 1280, Vec::new()); + let err = provisioner.reconcile(&plan).await.unwrap_err(); + let message = err.to_string(); + assert!(message.contains("tun-setup"), "{message}"); + assert!(message.contains(std::env::consts::OS), "{message}"); + + provisioner.remove("tsuntest").await.unwrap(); + } +} diff --git a/src/dataplane/wireguard/tun.rs b/src/dataplane/wireguard/tun.rs index 3318972..69e33f4 100644 --- a/src/dataplane/wireguard/tun.rs +++ b/src/dataplane/wireguard/tun.rs @@ -10,7 +10,8 @@ //! is what the test suite uses, so the entire data plane — handshake, //! encryption, routing — is exercised without touching the host. //! * `SystemTun`, behind the `tun-device` feature, is a real TUN interface. -//! Creating one needs `CAP_NET_ADMIN` on Linux or the equivalent elsewhere. +//! Creating one needs `CAP_NET_ADMIN`; attaching to one somebody else +//! prepared needs nothing. use std::net::Ipv6Addr; use std::sync::Arc; @@ -21,7 +22,7 @@ use crate::BoxFuture; use crate::dataplane::PluginError; /// What a device should look like once created. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct TunRequest { /// Interface name to ask for. pub name: String, @@ -37,6 +38,23 @@ pub struct TunRequest { pub mtu: u32, } +impl TunRequest { + /// A request carrying nothing but a name and an MTU. + /// + /// Used where the addresses have already been applied to the host, so the + /// device itself only needs opening. + pub fn bare(name: impl Into, mtu: u32) -> Self { + Self { + name: name.into(), + address: Ipv6Addr::UNSPECIFIED, + prefix_len: 0, + address_v4: None, + prefix_len_v4: 0, + mtu, + } + } +} + /// A packet interface. /// /// `recv` yields packets the operating system wants sent; `send` delivers @@ -66,6 +84,28 @@ pub trait TunFactory: Send + Sync + std::fmt::Debug + 'static { &'a self, request: TunRequest, ) -> BoxFuture<'a, Result, PluginError>>; + + /// Applies a changed request to an interface that already exists. + /// + /// The overlay IPv4 address is allocated at run time, so it can change + /// while the agent runs. A factory that manages the host applies that to + /// the live interface, without recreating it: recreating would drop every + /// tunnel riding on it. + /// + /// The default does nothing, which is right for a factory that only + /// attaches to an interface somebody else prepared. + fn reconfigure<'a>(&'a self, _request: TunRequest) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { Ok(()) }) + } + + /// Removes an interface this factory created. + /// + /// Runs on the teardown path, so it reports rather than fails: there is + /// nothing useful to do about a failure at that point, and an interface + /// that is already gone is the desired outcome anyway. + fn destroy<'a>(&'a self, _name: &'a str) -> BoxFuture<'a, ()> { + Box::pin(async move {}) + } } /// An in-memory packet interface. @@ -196,6 +236,8 @@ impl TunFactory for MemoryTunFactory { } } +#[cfg(feature = "tun-device")] +pub(crate) use system::open_tun; #[cfg(feature = "tun-device")] pub use system::{ Assigned, SystemTunFactory, interface_addresses, interface_exists, parse_if_inet6, @@ -223,11 +265,16 @@ mod system { /// this user. This is the recommended way to run the agent unprivileged. /// * **Create** it here, which needs `CAP_NET_ADMIN`. /// - /// Either way the overlay address has to be assigned by something - /// privileged: assigning an IPv6 address to an interface is not something - /// this crate's dependencies can do, so the agent checks that it is there - /// and says exactly what to run if it is not, rather than coming up in a - /// state where no traffic could ever arrive. + /// `SystemTunFactory` is the **attach** path, for a host where the agent + /// has no privileges at all: the interface and its addresses were put + /// there by something else, so it checks they are present and says + /// exactly what to run if they are not, rather than coming up in a state + /// where no traffic could ever arrive. + /// + /// The other path is + /// [`ManagedTunFactory`](super::super::provision::ManagedTunFactory), + /// where the agent creates and configures the interface itself. That is + /// the default on Linux and needs no preparation at all. pub struct SystemTun { name: String, mtu: u32, @@ -426,6 +473,60 @@ mod system { commands } + /// Opens the TUN interface, creating it if it is not already there. + /// + /// Synchronous, and deliberately so: on the managed path the caller holds + /// a capability guard across this call, and a guard must not span an + /// `await` because Linux capabilities are per thread. + /// + /// `attach_only` says the interface already exists and was prepared by + /// something else, so nothing beyond `TUNSETIFF` is issued — reconfiguring + /// it would need exactly the privileges that path is avoiding. + pub(crate) fn open_tun( + request: &TunRequest, + attach_only: bool, + ) -> Result, PluginError> { + let mut config = tun::Configuration::default(); + config.tun_name(&request.name); + config.platform_config(|platform| { + // The crate's own root check is not the check we want: the + // managed path holds CAP_NET_ADMIN without being root, and the + // attach path needs no privileges at all. Whether the open + // succeeds is the honest answer either way. + platform.ensure_root_privileges(false); + }); + // Packet information stays off, so reads and writes are raw IP + // packets. `ip tuntap add ... mode tun` also defaults to no packet + // information, so the flags match when attaching to one. + + let device = tun::create_as_async(&config).map_err(|err| { + let hint = if attach_only { + format!( + "interface `{}` exists but could not be opened: {err}. \ + It must be a persistent TUN interface owned by this user.", + request.name + ) + } else { + format!( + "cannot create the TUN interface `{}`: {err}. \ + Creating one needs CAP_NET_ADMIN. Either grant it with \ + `setcap cap_net_admin+p`, or prepare the interface once as root \ + (see `tsunagi tun-setup`) and run unprivileged.", + request.name + ) + }; + PluginError::Unavailable(hint) + })?; + + let (reader, writer) = tokio::io::split(device); + Ok(Arc::new(SystemTun { + name: request.name.clone(), + mtu: request.mtu, + reader: Mutex::new(reader), + writer: Mutex::new(writer), + }) as Arc) + } + fn current_user() -> String { std::env::var("SUDO_USER") .or_else(|_| std::env::var("USER")) @@ -468,41 +569,7 @@ mod system { ))); } - let mut config = tun::Configuration::default(); - config.tun_name(&request.name); - if existed { - // Attach only. Reconfiguring an interface somebody - // prepared for us would need exactly the privileges we - // are avoiding, so no ioctl beyond TUNSETIFF is issued. - config.platform_config(|platform| { - platform.ensure_root_privileges(false); - }); - } else { - // We are creating it, so we configure it. - config.mtu(request.mtu as u16).up(); - } - // Packet information stays off, so reads and writes are raw IP - // packets. `ip tuntap add ... mode tun` also defaults to no - // packet information, so the flags match when attaching. - - let device = tun::create_as_async(&config).map_err(|err| { - let hint = if existed { - format!( - "interface `{}` exists but could not be opened: {err}. \ - It must be a persistent TUN interface owned by this user.", - request.name - ) - } else { - format!( - "cannot create the TUN interface `{}`: {err}. \ - Creating one needs CAP_NET_ADMIN. Either prepare it once as root \ - (see `tsunagi tun-setup`) and run unprivileged, or grant the \ - capability.", - request.name - ) - }; - PluginError::Unavailable(hint) - })?; + let device = open_tun(&request, existed)?; // An interface we just created has no address yet either. if let Err(reason) = check_address(&request.name, request.address) { @@ -512,13 +579,7 @@ mod system { ))); } - let (reader, writer) = tokio::io::split(device); - Ok(Arc::new(SystemTun { - name: request.name, - mtu: request.mtu, - reader: Mutex::new(reader), - writer: Mutex::new(writer), - }) as Arc) + Ok(device) }) } } diff --git a/tests/interface_provisioning.rs b/tests/interface_provisioning.rs new file mode 100644 index 0000000..915afbb --- /dev/null +++ b/tests/interface_provisioning.rs @@ -0,0 +1,244 @@ +//! The agent managing its own overlay interface. +//! +//! Everything here is real except the host: real agents, real control plane, +//! real iroh links, the real plugin lifecycle and the real reconciliation +//! rules. The host itself is a [`MockHost`], so what the agent would have +//! done to a machine's interfaces is asserted instead of done — which is how +//! this runs with no privileges and without touching the machine it is on. +//! +//! What the real provisioner adds on top of this is the netlink calls, and +//! only those. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::Arc; +use std::time::Duration; + +use common::{config_with, network, wait_until}; +use tempfile::TempDir; +use tsunagi::dataplane::IpPlugin; +use tsunagi::dataplane::wireguard::{ + Cidr, InterfaceState, LinkKind, ManagedTunFactory, MockHost, MockProvisioner, WireguardConfig, + WireguardPlugin, +}; +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::identity::NetworkId; +use tsunagi::state::DEFAULT_IPV4_RANGE; +use tsunagi::{Agent, NetworkStatus}; + +/// An agent whose overlay interface is applied to a pretend host. +struct HostedAgent { + _dir: TempDir, + agent: Agent, + plugin: Arc, + host: MockHost, +} + +impl HostedAgent { + async fn spawn(discovery: &SharedMemoryDiscovery, tag: &str, host: MockHost) -> Self { + let dir = TempDir::new().unwrap(); + let provisioner = Arc::new(MockProvisioner::new(host.clone())); + let factory = Arc::new(ManagedTunFactory::new(provisioner)); + let config = WireguardConfig::new(dir.path().join("wireguard")) + .with_interface_prefix(tag) + .with_reconcile(Duration::from_millis(20), Duration::from_millis(100)); + let plugin = WireguardPlugin::open(config, factory).await.unwrap(); + let agent = Agent::spawn( + config_with(dir.path(), discovery) + .with_overlay_ipv4_range(Some(DEFAULT_IPV4_RANGE)) + .with_plugin(plugin.clone() as Arc), + ) + .await + .unwrap(); + Self { + _dir: dir, + agent, + plugin, + host, + } + } + + /// The interface name the plugin settled on for a network. + async fn interface(&self, network: NetworkId) -> String { + wait_until("the plugin named its interface", || async { + self.plugin + .overview(network) + .map(|view| view.interface) + .filter(|name| !name.is_empty()) + }) + .await + } + + /// Waits until the pretend host shows an interface in the given state. + async fn wait_for_host( + &self, + what: &str, + name: &str, + probe: impl Fn(Option) -> Option, + ) -> T { + wait_until(what, || async { probe(self.host.get(name)) }).await + } +} + +fn v4(state: &InterfaceState) -> Vec { + state + .addresses + .iter() + .filter_map(|cidr| match cidr.addr { + IpAddr::V4(addr) => Some(addr), + IpAddr::V6(_) => None, + }) + .collect() +} + +fn has_v6(state: &InterfaceState) -> bool { + state + .addresses + .iter() + .any(|cidr| matches!(cidr.addr, IpAddr::V6(_))) +} + +#[tokio::test] +async fn an_agent_creates_and_configures_its_own_overlay_interface() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-create"); + let agent = HostedAgent::spawn(&discovery, "tsunp", MockHost::new()).await; + + let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); + let interface = agent.interface(network_id).await; + + let state = agent + .wait_for_host("the interface to be created", &interface, |state| { + state.filter(|state| !state.addresses.is_empty()) + }) + .await; + + assert_eq!(state.kind, LinkKind::Tun); + assert!(state.up, "the agent brought the link up itself"); + assert_eq!(state.mtu, 1280); + assert!(has_v6(&state), "the derived overlay address is assigned"); + + // The IPv4 address is allocated at run time, so it arrives on a later + // reconciliation than the interface itself. + let addresses = agent + .wait_for_host("the allocated IPv4 address", &interface, |state| { + state.map(|state| v4(&state)).filter(|v4| !v4.is_empty()) + }) + .await; + assert_eq!(addresses.len(), 1); + assert!( + DEFAULT_IPV4_RANGE.contains(addresses[0]), + "{:?} is outside {DEFAULT_IPV4_RANGE}", + addresses[0] + ); + + agent.agent.shutdown().await; +} + +#[tokio::test] +async fn an_interface_left_by_a_crashed_run_is_replaced_rather_than_tripped_over() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-crash"); + + // Work out the name this agent will use, then seed the host with what a + // run that died would have left there: the interface, still carrying an + // address from an allocation that no longer applies, with nothing holding + // it open. + let probe = HostedAgent::spawn(&discovery, "tsunc", MockHost::new()).await; + let network_id = probe.agent.join_network(&name, &secret).await.unwrap(); + let interface = probe.interface(network_id).await; + probe.agent.shutdown().await; + + let host = MockHost::new(); + let stale = Cidr::new(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 178)), 24).unwrap(); + host.insert_stale_tun(&interface, vec![stale]); + + let agent = HostedAgent::spawn(&discovery, "tsunc", host).await; + let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); + assert_eq!(agent.interface(network_id).await, interface); + + let state = agent + .wait_for_host("the interface to be rebuilt", &interface, |state| { + state.filter(|state| state.attached && has_v6(state)) + }) + .await; + assert!( + !state.addresses.contains(&stale), + "the stale address is gone: {:?}", + state.addresses + ); + + agent.agent.shutdown().await; +} + +#[tokio::test] +async fn an_interface_belonging_to_something_else_is_left_alone() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-foreign"); + + let probe = HostedAgent::spawn(&discovery, "tsunf", MockHost::new()).await; + let network_id = probe.agent.join_network(&name, &secret).await.unwrap(); + let interface = probe.interface(network_id).await; + probe.agent.shutdown().await; + + // Somebody else's bridge happens to hold the name. + let host = MockHost::new(); + let theirs = InterfaceState { + kind: LinkKind::Foreign("bridge".into()), + attached: true, + up: true, + mtu: 1500, + addresses: vec![Cidr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 9, 1)), 24).unwrap()], + }; + host.insert(&interface, theirs.clone()); + + let agent = HostedAgent::spawn(&discovery, "tsunf", host).await; + agent.agent.join_network(&name, &secret).await.unwrap(); + + // Give the plugin several reconciliation rounds to do the wrong thing. + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + agent.host.get(&interface), + Some(theirs), + "the foreign interface must be untouched" + ); + + // The control plane is unaffected by the data plane refusing. + assert!(matches!( + agent.agent.network_status(network_id).await, + Ok(NetworkStatus { .. }) + )); + + agent.agent.shutdown().await; +} + +#[tokio::test] +async fn leaving_a_network_removes_the_interface_from_the_host() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("provision-cleanup"); + let agent = HostedAgent::spawn(&discovery, "tsunx", MockHost::new()).await; + + let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); + let interface = agent.interface(network_id).await; + agent + .wait_for_host("the interface to exist", &interface, |state| state) + .await; + + agent.agent.deactivate_network(network_id).await.unwrap(); + + agent + .wait_for_host("the interface to be removed", &interface, |state| { + state.is_none().then_some(()) + }) + .await; + assert!( + agent.host.names().is_empty(), + "nothing is left behind: {:?}", + agent.host.names() + ); + + agent.agent.shutdown().await; +}