From 21be7e9b447cfeb0f633eebc38a3af7dd4f3d90c Mon Sep 17 00:00:00 2001 From: tsunagi Date: Mon, 21 Sep 2026 11:55:20 +0100 Subject: [PATCH] Separate control and data logically, move WireGuard into userspace, add a CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrects the architecture on two points raised in review, while the project is still small enough to change cheaply. 1. Control and data are separated *logically*, not physically. The old reading — "nothing but control may ride on iroh" — threw away iroh's whole value and would have forced the data plane to reimplement STUN, ICE and a relay. Now both planes ride on iroh with different ALPNs and different connections, so the data plane inherits hole punching and relay fallback, while proto/ still knows nothing about packets and dataplane/ knows nothing about the control protocol. New boundary: PacketTransport / PacketLink, an authenticated unreliable datagram channel per (network, peer, protocol). tsunagi/data/1 runs the same membership handshake, then DataOpen/DataOpenAck, then QUIC datagrams. Only the smaller endpoint id dials, so exactly one link exists per pair. A plugin is handed links and never learns reachability, so the WireGuard announcement shrank to a public key: there is no address left to lie about. 2. WireGuard now runs in userspace, on boringtun's protocol state machine. No kernel module, no wg tool, no ip shell-out, no loopback proxy: the wgtool, backend and bridge modules are gone. Only creating a TUN device needs privileges, and that sits behind TunFactory, so the entire data plane — handshake, encryption, routing, address ownership — is tested with none. Address ownership is enforced rather than believed: outbound packets go to the owner of the destination address, inbound packets are dropped unless their source is the address derived for the peer that sent them. 3. A `tsunagi` binary: secret, doctor, id, up. It owns the runtime, the logging subscriber and Ctrl-C, which the library still refuses to. Also fixes a reference cycle where IrohTransport held Arc, which kept the databases open and the directory lock held after shutdown; two storage tests caught it once the cycle existed. 81 tests pass offline with no privileges, including real IPv6 packets crossing a real WireGuard tunnel over real iroh connections. Verified by hand: two CLI processes forming a mesh both on loopback and via n0 discovery using only an endpoint id. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 13 +- AGENTS.md | 38 +- Cargo.lock | 523 ++++++++++++++- Cargo.toml | 20 +- README.md | 81 ++- docs/architecture.md | 42 +- docs/protocol.md | 28 + docs/sync-model.md | 4 +- docs/testing.md | 35 +- docs/threat-model.md | 31 +- docs/wireguard.md | 238 ++++--- examples/wireguard_mesh.rs | 192 +++--- src/agent/events.rs | 29 + src/agent/mod.rs | 108 +++- src/agent/network.rs | 197 +++++- src/agent/status.rs | 4 + src/bin/tsunagi.rs | 554 ++++++++++++++++ src/dataplane/mod.rs | 13 + src/dataplane/transport/iroh_link.rs | 317 +++++++++ src/dataplane/transport/mod.rs | 146 +++++ src/dataplane/wireguard/announcement.rs | 175 +---- src/dataplane/wireguard/backend.rs | 214 ------ src/dataplane/wireguard/config.rs | 461 +------------ src/dataplane/wireguard/device.rs | 554 ++++++++++++++++ src/dataplane/wireguard/keys.rs | 15 +- src/dataplane/wireguard/mod.rs | 50 +- src/dataplane/wireguard/packet.rs | 125 ++++ src/dataplane/wireguard/plugin.rs | 475 +++++++------- src/dataplane/wireguard/tun.rs | 307 +++++++++ src/dataplane/wireguard/wgtool.rs | 667 ------------------- src/net.rs | 8 +- src/proto/message.rs | 31 + src/proto/mod.rs | 10 +- tests/wireguard.rs | 826 +++++++++++------------- tests/wireguard_system.rs | 120 ---- 35 files changed, 3987 insertions(+), 2664 deletions(-) create mode 100644 src/bin/tsunagi.rs create mode 100644 src/dataplane/transport/iroh_link.rs create mode 100644 src/dataplane/transport/mod.rs delete mode 100644 src/dataplane/wireguard/backend.rs create mode 100644 src/dataplane/wireguard/device.rs create mode 100644 src/dataplane/wireguard/packet.rs create mode 100644 src/dataplane/wireguard/tun.rs delete mode 100644 src/dataplane/wireguard/wgtool.rs delete mode 100644 tests/wireguard_system.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a4855f..52d8f09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,19 +43,16 @@ jobs: cargo run --locked --example two_agents cargo run --locked --example wireguard_mesh - # The real wg/ip backend changes the host's network and needs - # CAP_NET_ADMIN, so it is never part of the default job above. It is run on - # demand only. - wireguard-system: - name: wireguard system backend (opt-in) - if: github.event_name == 'workflow_dispatch' + # The library must keep building without the command line binary and + # without interface support, for embedders that want neither. + no-default-features: + name: library only runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: sudo apt-get update && sudo apt-get install -y wireguard-tools - - run: sudo -E $(which cargo) test --locked --test wireguard_system -- --ignored --test-threads=1 + - run: cargo check --locked --no-default-features --lib docs: name: rustdoc diff --git a/AGENTS.md b/AGENTS.md index c183dc1..694fb69 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,18 +17,27 @@ Design accordingly: a majority is not a root of trust. Keep these separate. Crossing them is the main thing to review for. -- **Control plane vs data plane.** iroh carries control messages only. User IP - traffic is never tunnelled through it. The core must never parse a plugin's - payload — see `src/dataplane/mod.rs`. Only +- **Control plane vs data plane.** The separation is **logical, not physical**. + The control protocol in `src/proto/` knows nothing about packets, and + `src/dataplane/` knows nothing about the control protocol; either can be + replaced on its own. Both may ride on iroh — refusing to would throw away + iroh's NAT traversal and force the data plane to reimplement it. They use + different ALPNs and different connections, so a busy or broken data plane + cannot disturb control traffic. +- **Plugins never learn reachability.** An `IpPlugin` is handed a `PacketLink` + per peer and moves datagrams over it. Addresses, hole punching and relays + belong to `src/dataplane/transport/`. A plugin announcement says *who*, never + *where*. +- **The core never parses a plugin payload.** See `src/dataplane/mod.rs`. Only `src/dataplane/wireguard/announcement.rs` interprets WireGuard payloads, and only after bounding every field. A data plane failure must never stop the control plane. -- **Derived, not claimed.** A WireGuard peer's `AllowedIPs` are always derived - locally from its public key. Never take them from what the peer announces, or - a member can route another member's traffic to itself. +- **Derived, not claimed.** A peer's overlay address is derived from its public + key. Outbound packets are routed to the owner of the destination address; + inbound packets are dropped unless their source is the address derived for + the peer that sent them. Never trust an address a peer announces. - **Plugins own their system objects.** A plugin creates and removes its own - interface and nothing else. An interface that already exists and is not ours - is refused, never adopted. Never touch routing, DNS or firewall settings. + interface and nothing else. Never touch routing, DNS or firewall settings. - **Device identity vs network identity.** The iroh endpoint id is the device's public key. `NetworkId` is derived from name + secret only. Never conflate them, and never let one change the other. @@ -88,7 +97,8 @@ Keep these separate. Crossing them is the main thing to review for. | `src/proto/` | framing, message formats, membership handshake | | `src/net.rs` | iroh endpoint adapter and observability snapshots | | `src/agent/` | agent lifecycle, per-network runtimes, sessions, events, status | -| `src/dataplane/` | the contract IP plugins implement, and the WireGuard plugin | +| `src/dataplane/` | the plugin contract, the packet transport, and the WireGuard plugin | +| `src/bin/tsunagi.rs`| the command line agent; the only place that owns a runtime, a logger and signals | | `tests/` | integration tests; `tests/common/` is the shared harness | Add abstractions only at real substitution or testing boundaries. Do not add a @@ -106,11 +116,11 @@ trait per struct. Prefer one crate with clear modules over many small crates. happen. - The default suite must pass with no internet, no DHT, no public relay, no administrator rights and no changes to OS network settings. Anything needing - the internet, or root, stays out of the default set — the real WireGuard - backend's tests live in `tests/wireguard_system.rs` behind `--ignored`. -- The WireGuard backend may be substituted (`RecordingBackend`). Its key - handling, announcements, derived addressing, configuration builder and - reconciliation may not. + the internet or root stays out of the default set. +- The WireGuard packet interface may be substituted (`MemoryTunFactory`). Its + key handling, announcements, derived addressing, the WireGuard protocol + itself and address-ownership enforcement may not — the default suite runs + real handshakes and real encryption. - Running several library instances in one process is not a test of several system processes; do not describe it as one. diff --git a/Cargo.lock b/Cargo.lock index e206f1f..52c2407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -76,6 +126,24 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async_io_stream" version = "0.3.3" @@ -140,6 +208,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "blake3" version = "1.8.7" @@ -153,6 +230,15 @@ dependencies = [ "cpufeatures 0.3.1", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -171,6 +257,44 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "boringtun" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15dd6a8a89cbe8997f37ca0cf035e6ea4d64cd2ecea4aed83ffb9f99f7126939" +dependencies = [ + "aead", + "base64 0.22.1", + "blake2", + "chacha20poly1305", + "hex", + "hmac 0.12.1", + "ip_network", + "ip_network_table", + "libc", + "nix", + "parking_lot", + "portable-atomic", + "rand_core 0.6.4", + "ring", + "tracing", + "untrusted", + "x25519-dalek", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -211,6 +335,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "chacha20" version = "0.10.2" @@ -219,7 +354,20 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20 0.9.1", + "cipher", + "poly1305", + "zeroize", ] [[package]] @@ -242,8 +390,49 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", "inout", + "zeroize", ] +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + [[package]] name = "cmov" version = "0.5.4" @@ -259,6 +448,12 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.8" @@ -269,6 +464,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -334,6 +538,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.1.7" @@ -341,6 +551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -351,7 +562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "hybrid-array", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -372,6 +583,21 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + [[package]] name = "curve25519-dalek" version = "5.0.0" @@ -381,9 +607,9 @@ dependencies = [ "cfg-if", "cpufeatures 0.3.1", "curve25519-dalek-derive", - "digest", - "fiat-crypto", - "rand_core", + "digest 0.11.3", + "fiat-crypto 0.3.0", + "rand_core 0.10.1", "rustc_version", "serde", "subtle", @@ -473,13 +699,24 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", "ctutils", @@ -557,9 +794,9 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 5.0.0", "ed25519", - "rand_core", + "rand_core 0.10.1", "serde", "sha2", "signature", @@ -606,6 +843,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -624,6 +881,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "fiat-crypto" version = "0.3.0" @@ -823,7 +1086,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "rand_core", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -896,7 +1159,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac", + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -905,7 +1177,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -1174,6 +1446,28 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ip_network" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2f047c0a98b2f299aa5d6d7088443570faae494e9ae1305e48be000c9e0eb1" + +[[package]] +name = "ip_network_table" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4099b7cfc5c5e2fe8c5edf3f6f7adf7a714c9cc697534f63a5a5da30397cb2c0" +dependencies = [ + "ip_network", + "ip_network_table-deps-treebitmap", +] + +[[package]] +name = "ip_network_table-deps-treebitmap" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e537132deb99c0eb4b752f0346b6a836200eaaa3516dd7e5514b63930a09e5d" + [[package]] name = "ipconfig" version = "0.3.4" @@ -1248,7 +1542,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffd1efd1aaecd68d4d1b0727a30c5811f43fb822843e48f65f6e5db3b7256cc9" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 5.0.0", "data-encoding", "data-encoding-macro", "derive_more", @@ -1355,6 +1649,12 @@ dependencies = [ "ws_stream_wasm", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -1458,6 +1758,16 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libredox" version = "0.1.24" @@ -1490,6 +1800,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -1808,6 +2127,18 @@ dependencies = [ "wmi", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "noq" version = "1.3.0" @@ -2012,6 +2343,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -2046,6 +2383,29 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" @@ -2103,6 +2463,17 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs8" version = "0.11.0" @@ -2132,6 +2503,17 @@ dependencies = [ "time", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "polyval" version = "0.6.2" @@ -2240,9 +2622,18 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ - "chacha20", + "chacha20 0.10.2", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -2257,7 +2648,16 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", ] [[package]] @@ -2502,6 +2902,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -2583,6 +2989,19 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "serdect" version = "0.4.3" @@ -2607,7 +3026,7 @@ checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "digest", + "digest 0.11.3", ] [[package]] @@ -2641,7 +3060,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -2742,6 +3161,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.28.0" @@ -3175,12 +3600,15 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" name = "tsunagi" version = "0.1.0" dependencies = [ + "boringtun", + "bytes", + "clap", "data-encoding", "directories", "fs4", "hex", "hkdf", - "hmac", + "hmac 0.13.0", "iroh", "netwatch", "postcard", @@ -3194,10 +3622,32 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", - "x25519-dalek", + "tun", "zeroize", ] +[[package]] +name = "tun" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb55e468c585e02ef89ba7a1a9848871ac942dfc64547ad4d1166b0f61a98be" +dependencies = [ + "bytes", + "cfg-if", + "futures", + "futures-core", + "ipnet", + "libc", + "log", + "nix", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "windows-sys 0.61.2", + "wintun-bindings", +] + [[package]] name = "typenum" version = "1.20.1" @@ -3257,6 +3707,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.1" @@ -3721,6 +4177,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "wintun-bindings" +version = "0.7.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4316764300a7eb4aecf4770c81ae629ff1161b54905d48721a6832fd296ce5f" +dependencies = [ + "blocking", + "futures", + "libloading", + "log", + "thiserror 2.0.20", + "windows-sys 0.61.2", +] + [[package]] name = "wmi" version = "0.18.4" @@ -3763,12 +4233,13 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "3.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ - "curve25519-dalek", - "rand_core", + "curve25519-dalek 4.1.3", + "rand_core 0.6.4", + "serde", "zeroize", ] @@ -3888,3 +4359,9 @@ dependencies = [ "quote", "syn 3.0.6", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index c76465a..9fd1be1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,23 @@ readme = "README.md" keywords = ["mesh", "p2p", "iroh", "networking"] categories = ["network-programming"] +[features] +default = ["cli"] +# The `tsunagi` command line binary. Library users can opt out. +cli = ["dep:clap", "dep:tracing-subscriber", "tokio/signal", "tun-device"] +# A real TUN device, so the WireGuard plugin can carry actual IP traffic. +# Needs CAP_NET_ADMIN at run time; without it the plugin still runs and its +# in-memory device can be used for tests. +tun-device = ["dep:tun"] + +[[bin]] +name = "tsunagi" +path = "src/bin/tsunagi.rs" +required-features = ["cli"] + [dependencies] +clap = { version = "4.5", features = ["derive", "env"], optional = true } +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } iroh = { version = "1.2", default-features = false, features = ["tls-ring"] } tokio = { version = "1.53", features = ["rt", "rt-multi-thread", "sync", "time", "macros"] } rusqlite = { version = "0.40", features = ["bundled"] } @@ -28,8 +44,10 @@ thiserror = "2.0" tracing = "0.1" fs4 = { version = "1.1", features = ["sync"] } directories = "6.0" -x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } netwatch = "0.19.3" +bytes = "1.12.1" +boringtun = { version = "0.7.1", default-features = false } +tun = { version = "0.8", features = ["async"], optional = true } [dev-dependencies] tokio = { version = "1.53", features = ["rt", "rt-multi-thread", "sync", "time", "macros", "process"] } diff --git a/README.md b/README.md index 8891c08..1d9fbad 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,10 @@ A working library with **real iroh connections** and integration tests: - status snapshots, an event stream and honest diagnostics; - configuration restored after a restart; - correct behaviour when the disposable cache is missing or corrupt; -- a **WireGuard data plane plugin**: its own key per network, deterministic - IPv6 overlay addressing, a full-mesh configuration built locally, and - reconciliation that repairs drift. +- a **WireGuard data plane**, in userspace: its own key per network, + deterministic IPv6 overlay addressing, real tunnels carried over iroh, and + address ownership enforced rather than believed; +- a **command line agent**, `tsunagi`. ### What it deliberately does **not** do @@ -45,15 +46,69 @@ and signed revocations are designed for but not implemented — see including that its system backend is Linux-only, are in [docs/wireguard.md](docs/wireguard.md#limits-and-future-work). -**Only control messages travel over iroh. User IP traffic is not tunnelled -through it.** Filtering user traffic is the operating system's and the user's -responsibility, not this library's. +**Control and data are separated logically, not physically.** Both ride on +iroh, on different ALPNs and different connections, so the data plane inherits +iroh's hole punching and relay fallback instead of reimplementing them — while +the control protocol still knows nothing about packets and can keep a different +transport underneath it later. Filtering user traffic remains the operating +system's and the user's responsibility, not this library's. ## Requirements -- Rust 1.91 or newer (iroh 1.2 requires it) (edition 2024). Pinned dependencies in `Cargo.lock`. +- Rust 1.91 or newer (iroh 1.2 requires it), edition 2024. Pinned dependencies + in `Cargo.lock`. - No internet, no DHT, no public relay, no administrator rights and no changes to OS network settings are needed to build or test. +- WireGuard runs in userspace (boringtun): **no kernel module and no `wg` + tool**. Only creating a real network interface needs `CAP_NET_ADMIN`, and + `--no-tun` skips even that. + +## Trying it on two machines + +On the first machine: + +```bash +cargo build --release +./target/release/tsunagi secret # prints tsn1...; share it privately +./target/release/tsunagi doctor # what this host can and cannot do + +./target/release/tsunagi up --network lab --secret "$SECRET" --wireguard +``` + +It prints its endpoint id and then waits. On the second machine, pass that id: + +```bash +./target/release/tsunagi up --network lab --secret "$SECRET" --wireguard \ + --peer +``` + +Within a few seconds both print something like: + +```text + + peer b47c958462 connected over Direct rtt=Some(4.5ms) + + data link to b47c958462 for wireguard: Direct via Ip(…), datagram 1382 + +--- status --- +control: 1 peer(s), 0 dial failure(s), 0 handshake failure(s) +wireguard: tsunkkcp43lmdje on fd15:1d9e:fa21:f201:…/64 mtu 1100, 1/1 tunnel(s) established + 4jO4kx9Z fd15:1d9e:fa21:f201:… handshake 3s ago tx=0 rx=0 dropped=0 path=Direct via Ip(…) +``` + +`1/1 tunnel(s) established` means a real WireGuard handshake completed. Then +`ping6` the peer's overlay address. + +Notes: + +- Only one side needs `--peer`; the link is bidirectional. Peer discovery + beyond this manual bootstrap is future work. +- The default `--transport n0` uses iroh's public address lookup and relays, so + two machines behind NAT find each other. `--transport local` keeps everything + on the local network. +- Without `CAP_NET_ADMIN`, add `--no-tun`: the mesh, the data links and the + WireGuard handshakes all still run and are visible in the status output, only + traffic does not reach the operating system. That is the quickest way to + confirm the network forms. +- Run as root (or grant `CAP_NET_ADMIN`) to get a real interface. ## Checks @@ -71,18 +126,11 @@ tests: ```bash cargo run --example two_agents # control plane only -cargo run --example wireguard_mesh # two agents forming a WireGuard overlay +cargo run --example wireguard_mesh # a WireGuard overlay carrying a real packet ``` Both run with no privileges and change nothing on the host. -The one part that does change the host's network — the real `wg`/`ip` backend — -is behind `--ignored` and needs Linux, wireguard-tools and `CAP_NET_ADMIN`: - -```bash -sudo -E cargo test --test wireguard_system -- --ignored --test-threads=1 -``` - ## Usage ```rust,no_run @@ -146,6 +194,9 @@ The WireGuard plugin keeps its own keys in its own `wireguard.sqlite`, wherever its configuration points, because plugin keys are neither the iroh identity nor the network secret. +The command line agent puts everything under the platform's per-user +directories by default; `--state-dir` and `--cache-dir` override them. + One state directory belongs to one live agent, enforced with a real OS file lock rather than an existence check. diff --git a/docs/architecture.md b/docs/architecture.md index ca5f85f..7a4422c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,12 +12,21 @@ authentication, participant announcements, capability exchange and — later — state synchronisation and delivery of IP-plugin data. **Data plane.** Separate plugins create IP connectivity. WireGuard is the first -one and is implemented — see [wireguard.md](wireguard.md). Plugin keys, -configuration and lifecycle are separate from iroh identity and from the +one and is implemented in userspace — see [wireguard.md](wireguard.md). Plugin +keys, configuration and lifecycle are separate from iroh identity and from the network secret. The core moves an opaque, bounded payload and never parses it. -Only control messages travel over iroh. User IP traffic is not tunnelled -through it; WireGuard packets travel over WireGuard's own UDP sockets. +**The transport in between.** Plugins do not open connections. They are handed +a `PacketLink` — an authenticated, unreliable datagram channel to one peer for +one protocol — and never learn how it is carried. + +The separation between the two planes is **logical, not physical**. Both ride +on iroh, on different ALPNs and different connections. That is deliberate: +iroh's whole value is hole punching a direct path between peers behind NAT, +with a relay as fallback, and a data plane that refused to use it would have to +reimplement all of it. What the separation buys is that `proto` knows nothing +about packets and `dataplane` knows nothing about the control protocol, so +either can be replaced on its own. A plugin talks to the core through three narrow hooks — `on_network_activated`, a `PluginContext` for re-announcements and error reports, and a bounded @@ -36,27 +45,36 @@ and the agent stays manageable. | `proto` | message format, handshake, membership proof, protocol limits | | `agent` | agent and per-network lifecycle, reconnect, in-process message routing | | `storage` | mandatory state and the separately recoverable cache | +| `dataplane::transport` | authenticated datagram links to peers; where reachability lives | | `dataplane` | the contract IP plugins implement, plus the WireGuard plugin | Abstractions exist only where something is really substituted or really needs -isolating for tests: `NetworkDiscovery`, `IpPlugin`, and `WireguardBackend` -(which is what lets the plugin be tested in full without root). Everything else -is a concrete type. +isolating for tests: `NetworkDiscovery`, `IpPlugin`, `PacketTransport` / +`PacketLink` (so the data plane's carrier can change), and `TunFactory` (which +is what lets the whole data plane be tested without privileges). Everything +else is a concrete type. ## Runtime shape ```text Agent one persistent identity, one iroh endpoint, - ├── EndpointAdapter one state directory, N networks + ├── EndpointAdapter (two ALPNs) one state directory, N networks ├── Storage (state.sqlite + cache.sqlite + ownership lock) - ├── accept loop task ── weak ref, exits when the agent is dropped + ├── IrohTransport ── data plane links, weak ref back to the agent + ├── accept loop task ── routes by ALPN; weak ref, exits when the agent drops + ├── plugin request loop ── re-announcements and plugin error reports └── NetworkRuntime per NetworkId ├── discovery + dial loop (bounded concurrency, backoff with jitter) - └── Session per peer - ├── reader task ── frames in -> SessionEvent - └── writer task ── encoded frames out + ├── Session per peer (control) + │ ├── reader task ── frames in -> SessionEvent + │ └── writer task ── encoded frames out + └── PacketLink per (peer, plugin protocol), handed to the plugin ``` +Every strong reference from a background task back to the agent is a `Weak`. +A cycle there would keep the databases open and the directory lock held +forever after shutdown. + The library starts no runtime, installs no logging subscriber, handles no signals, never forks and never calls `process::exit`. Startup (`Agent::spawn`) and shutdown (`Agent::shutdown`) are explicit, background diff --git a/docs/protocol.md b/docs/protocol.md index 9664bbe..dacd64f 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -132,6 +132,34 @@ A `Hello` naming a network this agent does not have active is rejected with "unknown network". Because the claim is unverified at that point, the rejection event does not report a network id. +## The data plane protocol + +IP plugin packets never travel on a control connection. They use their own +ALPN, `tsunagi/data/1`, on their own iroh connection: + +```text +initiator -> responder : (the same membership handshake as above) +initiator -> responder : DataOpen { protocol } +initiator <- responder : DataOpenAck { accepted, max_datagram } +thereafter : QUIC datagrams carrying that plugin's packets +``` + +The membership handshake is identical and bound to the same network, so a data +channel cannot be opened by somebody who does not know the secret. `protocol` +is bounded and must name a plugin the responder actually runs; otherwise the +channel is declined, which is an ordinary outcome rather than an error. + +Only one side dials — the one with the smaller endpoint id — so two agents +never open two channels for the same thing. + +Packets ride as QUIC **datagrams**: unreliable and unordered, which is what a +tunnelled protocol wants, and free of the head-of-line blocking a stream would +add. The datagram limit is what caps a plugin's MTU. + +Separate connections mean separate congestion control, so a saturated data +plane cannot delay control messages, and a data plane failure cannot take the +control plane down with it. + ## Control messages After authentication, every frame is an `Envelope { network_id, message }` and diff --git a/docs/sync-model.md b/docs/sync-model.md index 4c11f11..5e26c6e 100644 --- a/docs/sync-model.md +++ b/docs/sync-model.md @@ -3,8 +3,8 @@ **Nothing in this document is implemented.** The proof of concept exchanges hostname and capability announcements over live sessions and keeps no replicated history. That is also why WireGuard peer membership is -session-scoped today: a peer disappears from the overlay configuration when its -control session ends, because there is no agreed durable state to keep it. This file records the intended direction so the module +session-scoped today: a peer leaves the overlay when its control session ends, +because there is no agreed durable state to keep it. This file records the intended direction so the module boundaries in [architecture.md](architecture.md) stay compatible with it, and so nobody mistakes the current announcements for synchronisation. diff --git a/docs/testing.md b/docs/testing.md index 7e0a9ed..20de3da 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -38,18 +38,17 @@ of several system processes, and is not presented as one. | 9 | wrong version, a message before authentication, a proof replayed on another connection, an oversized frame and a `Hello` for an inactive network are all rejected without taking the agent down | `tests/authentication.rs` | | 10 | a second agent on the same state directory gets a clear error; after a clean stop the directory reopens; shutdown ends background tasks and refuses further work; independent agents coexist in one process | `tests/resilience.rs` | -`tests/wireguard.rs` drives the WireGuard plugin over real iroh connections -with the in-memory backend: a pair and a three-agent mesh converge to `N - 1` -peers with locally derived `AllowedIPs`, a departing peer is removed, two -networks get separate interfaces, keys and overlays, reconciliation repairs a -configuration edited by hand, a backend failure leaves the control plane -untouched, a restart keeps the WireGuard identity, shutdown removes every -interface, a member claiming another member's overlay address is rejected, and -the core carries the payload without interpreting it. - -`tests/wireguard_system.rs` exercises the real `wg`/`ip` backend. It is -**ignored by default** because it changes the host's network and needs Linux, -wireguard-tools and `CAP_NET_ADMIN`. +`tests/wireguard.rs` drives the WireGuard data plane over real iroh +connections. Everything is real except the packet interface: real agents, real +control plane, real data links, real WireGuard handshakes and encryption from +boringtun, with an in-memory TUN device so none of it needs privileges. It +covers real IPv6 packets travelling both ways through a tunnel, a three-agent +mesh, a peer that sends from an address it does not own being dropped, packets +for unowned addresses being counted rather than broadcast, a departing peer +losing its tunnel, two networks keeping separate interfaces and keys, restart +keeping the WireGuard identity, shutdown removing every interface, a forged +overlay claim being rejected, and the core carrying the payload without +interpreting it. `tests/discovery.rs` covers the discovery contract itself: a static bootstrap candidate is enough to join, several backends compose, entries are withdrawn @@ -59,15 +58,17 @@ Unit tests in `src/proto/handshake.rs` cover the transcript construction itself: role separation, channel binding, identity and network binding, unambiguous encoding, and rejection under the wrong key. -Unit tests in `src/dataplane/wireguard/` cover the parts that would otherwise -need root: key clamping against the RFC 7748 vector, overlay derivation, -announcement validation including the address-hijack attempt, configuration -building and rendering, the exact command plan the real backend would run, and -parsing `wg showconf` and `ip address show` output. +Unit tests in `src/dataplane/wireguard/` cover key clamping against the RFC +7748 vector, overlay derivation, announcement validation including the +address-hijack attempt, interface naming, and IP header parsing against +truncated and nonsense input. `tests/end_to_end.rs` is the vertical slice: persistent identity → network space → discovery → iroh → authentication → message exchange. +What the default suite does **not** cover is the real TUN interface, because +that needs `CAP_NET_ADMIN`. Everything above it does run. + ## Not covered, and not claimed to be Listed in [sync-model.md](sync-model.md#future-tests): snapshots, revocations, diff --git a/docs/threat-model.md b/docs/threat-model.md index 1e3e807..b6ba1d3 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -19,10 +19,15 @@ Read this before relying on anything here. The protocol is in messages for network B, even over a shared physical connection. - **Confidentiality and integrity in transit.** Provided by QUIC/TLS. This crate adds no encryption of its own. -- **Overlay address ownership.** A WireGuard peer's `AllowedIPs` are derived - from its public key, not taken from its announcement, so a member cannot - claim another member's overlay address and receive its traffic. See - [wireguard.md](wireguard.md#deterministic-overlay-addressing). +- **Overlay address ownership.** A peer's overlay address is derived from its + public key, not taken from its announcement. Outbound packets go to the owner + of the destination address; inbound packets are dropped unless their source + is the address derived for the peer that sent them. A member can therefore + neither receive nor forge another member's traffic. See + [wireguard.md](wireguard.md#address-ownership-is-enforced-not-announced). +- **Tunnelled traffic is end-to-end encrypted by WireGuard**, independently of + this crate. The transport underneath is also encrypted by iroh, but the + tunnel's confidentiality does not depend on that. - **Resource bounds.** Frame lengths are validated before allocation; strings, lists, queues, concurrent dials and in-flight handshakes are all bounded; handshakes, dials and writes have timeouts. @@ -49,10 +54,14 @@ Read this before relying on anything here. The protocol is in permissions are owner-only where the platform supports it, and the state directory takes an ownership lock, but neither defends against a user who can read the file or against malware running as that user. -- **User IP traffic.** Carried by the WireGuard plugin, not by iroh, and - encrypted by WireGuard itself. *Filtering* it is still the operating system's - and the user's job: the plugin creates connectivity between members and does - not police what flows over it. +- **User IP traffic.** Carried by the WireGuard plugin over an iroh data + connection, and encrypted by WireGuard end to end. *Filtering* it is still + the operating system's and the user's job: the plugin creates connectivity + between members and does not police what flows over it. +- **Traffic metadata reaches the relay when one is used.** If iroh cannot hole + punch, the data connection goes through a relay, which then sees the volume + and timing of tunnelled traffic — though not its contents, which WireGuard + encrypted, nor the iroh layer's contents. - **Overlay address squatting.** A member can mint many WireGuard keys and therefore occupy many overlay addresses. It cannot pick which ones, but it can consume them and appear as many participants. @@ -60,9 +69,9 @@ Read this before relying on anything here. The protocol is in `wireguard.sqlite`, owner-only where the platform supports it. Copying that file copies this agent's overlay identity, exactly as copying `state.sqlite` copies its control plane identity. -- **What the data plane does not police.** The plugin sets `AllowedIPs` per - peer, which stops a member impersonating another member's overlay address. - It does not stop a member sending whatever it likes *from its own* address. +- **What the data plane does not police.** Address ownership stops a member + impersonating another member. It does not stop a member sending whatever it + likes *from its own* address. - **Denial of service.** Bounds and timeouts stop trivial resource exhaustion from a single peer. They do not make the agent resistant to a determined attacker who knows the secret, and no rate limiting per identity exists yet. diff --git a/docs/wireguard.md b/docs/wireguard.md index 6f455dd..8cb9eef 100644 --- a/docs/wireguard.md +++ b/docs/wireguard.md @@ -1,22 +1,54 @@ -# The WireGuard plugin +# The WireGuard data plane -WireGuard is the first IP plugin. It creates real IP connectivity between +WireGuard is the first IP plugin. It carries user traffic between participants while the control plane keeps doing its own job: deciding who is in the network and carrying each participant's opaque announcement. Module boundaries are in [architecture.md](architecture.md), the control -protocol in [protocol.md](protocol.md), and the security consequences in +protocol in [protocol.md](protocol.md), the security consequences in [threat-model.md](threat-model.md). -## What stays separate +## Userspace, not the kernel -| | | -|---|---| -| **User IP traffic** | Never goes through iroh. iroh carries announcements; packets travel over WireGuard's own UDP sockets. | -| **Addresses** | An iroh address is an address for iroh. The plugin gathers and advertises its own reachability. | -| **Payloads** | The core moves a bounded opaque blob. Only `dataplane::wireguard::announcement` interprets it. | -| **Keys** | One WireGuard key per network, in the plugin's own store. Unrelated to the iroh device key and to the network secret. | -| **Failures** | A data plane error is reported and retried. The control plane keeps running and the agent stays manageable. | +WireGuard here is [boringtun]'s protocol state machine running in this +process. There is **no kernel WireGuard module** and **no `wg` tool**: the same +code runs everywhere, and the protocol can be exercised in tests without any +privileges at all. + +The only privileged step left is creating a packet interface so the operating +system can hand us IP packets, and even that is behind a trait +([`TunFactory`]) with an in-memory implementation. + +| | needs privileges | what it proves | +|---|---|---| +| `MemoryTunFactory` | no | handshake, encryption, routing, address ownership | +| `SystemTunFactory` | `CAP_NET_ADMIN` | traffic actually reaches the OS | + +[boringtun]: https://docs.rs/boringtun +[`TunFactory`]: https://docs.rs/tsunagi + +## Where the packets go + +The plugin does not know and does not care. It is handed a `PacketLink` per +peer by the agent and runs a WireGuard tunnel over it: + +```text + TUN device (IP packets) PacketLink per peer + | | + v v + destination address -> peer --Tunn.encapsulate--> ciphertext -> transport + source address checked <--Tunn.decapsulate-- ciphertext <- transport +``` + +Reachability — hole punching, relay fallback — belongs to the transport, which +today is iroh. That is the whole reason the plugin's announcement says *who* it +is and never *where* it is: there is no address for a peer to advertise, get +wrong, or lie about. + +**Two peers behind NAT work exactly as well as iroh does.** iroh hole punches a +direct path when it can and falls back to a relay when it cannot; the tunnel +rides on whichever it got. There is no separate STUN, no separate hole punching +and no second set of NAT problems to solve for WireGuard. ## Deterministic overlay addressing @@ -36,110 +68,73 @@ Two consequences matter: * every member of a network derives the **same `/64`**, so the overlay is one subnet that nobody had to allocate; -* a member's address is bound to its WireGuard public key, so - **`AllowedIPs` are derived locally and never taken from what a peer claims**. +* a member's address is bound to its WireGuard public key, so address + ownership can be checked locally rather than believed. -That second point is the plugin's central security property. A participant who -knows the network secret can mint as many WireGuard keys — and therefore as -many overlay addresses — as it likes, but it cannot choose to collide with an -existing member's address without finding a hash preimage. An announcement -whose claimed address does not match the derivation is rejected outright. +## Address ownership is enforced, not announced -## The announcement +Kernel WireGuard enforces `AllowedIPs`. In userspace that is our job, and +[`device`] does it on both sides: -Carried as the opaque `PluginCapability { protocol: "wireguard", .. }` payload, -encoded with postcard: +* **outbound**, a packet is routed to the peer that *owns* its destination + address; a destination nobody owns is counted as unroutable and dropped; +* **inbound**, a decrypted packet is dropped unless its *source* is exactly the + address derived for the peer whose tunnel decrypted it. -| field | meaning | -|---|---| -| `version` | announcement format version, currently 1 | -| `public_key` | the peer's X25519 WireGuard key | -| `listen_port` | the UDP port its interface listens on | -| `endpoints` | reachability the plugin gathered for itself, at most 8 | -| `overlay_address` | what the peer believes its address is — cross-checked, never used | +So a participant cannot receive traffic addressed to somebody else and cannot +forge traffic that appears to come from somebody else. A participant who knows +the network secret can mint many keys and therefore occupy many addresses, but +it cannot choose to collide with an existing member without finding a hash +preimage. -Validation, all before anything reaches a configuration: the version must -match, the key must not be zero and must not be our own, the port must not be -zero, the endpoint list must be within bounds, unusable endpoints -(unspecified, multicast, broadcast, documentation, port zero) are dropped, and -the claimed overlay address must equal the derived one. +The announcement also carries the address the peer believes it has. It is never +used — only cross-checked — so a version skew produces a clear rejection rather +than silent non-connectivity. -## Building the configuration +[`device`]: https://docs.rs/tsunagi -Each agent builds its **own** configuration from the agreed set of -participants: for a full mesh of `N` members that is `N - 1` peers locally. -Nobody hands a configuration to anybody else and no participant is -authoritative. +## MTU -* **Interface name** — `prefix + base32(network_id)`, truncated to the - platform's 15 characters. Stable across restarts. Two agents on one host in - the same network need different prefixes. -* **Port** — `PortPolicy::Derived` picks a stable port from the network id - inside a range, so a peer's cached endpoint keeps working across restarts and - two networks on one host do not collide. `PortPolicy::Fixed` pins it. -* **Addresses** — the agent's own `/128` plus the shared `/64`. -* **Peer entries** — public key, derived `AllowedIPs`, the peer's first usable - advertised endpoint, and a keepalive. +Every packet rides in one transport datagram, and WireGuard adds 32 bytes. A +QUIC datagram on a relayed path can be as small as roughly 1160 bytes, so the +default interface MTU is **1100**, which leaves headroom rather than relying on +the best case. Packets that do not fit are dropped and counted +(`dropped_oversize`), never truncated. The observed datagram limit of each link +is reported in the status output. -Nothing free-form from the network reaches a command argument or a -configuration directive: peer keys, endpoints, prefixes and keepalives are -typed values that the plugin re-serialises itself. +## Lifecycle -## Backends +* A network is activated → the plugin loads or creates its key for that + network, derives the interface name, and creates the packet interface. If + that fails — no privileges, for instance — the key and the announcement still + work and the interface is retried on the next reconcile. +* A peer announces its key → recorded. +* A data link to that peer arrives → recorded. +* Reconciliation starts a tunnel for every peer that has **both**, and removes + tunnels for peers that lost either. +* A network is deactivated, or the agent shuts down → the interface and every + tunnel go away. The key stays, so coming back keeps the same overlay address. -The plugin computes *what* the interface should look like; a backend makes it -so. Splitting them is what keeps every interesting decision testable without -root. - -* **`RecordingBackend`** — applies configurations in memory. The default test - suite and the example use it, so neither needs privileges nor touches the - host. It can also be told to fail, or to report drift. -* **`WgToolBackend`** — drives the real `wg` and `ip` tools. Linux only, - requires `CAP_NET_ADMIN`. It is split into a **pure planner** and pure - parsers, which are unit tested on every platform, plus a thin executor. The - WireGuard configuration is piped to `wg setconf` / `wg syncconf` on standard - input, so the private key never reaches the filesystem. - -Ownership is explicit: the plugin creates the interface and the plugin removes -it. An interface that already exists and is not a WireGuard device is -**refused, not adopted**, so the agent never takes over something it did not -create. It changes no routing, DNS or firewall settings. - -## Reconciliation - -The plugin reconciles on every change — a peer announcement, a peer leaving — -coalesced over a short debounce, and again on a timer. Each pass reads the -interface back, compares it with the desired state, and applies only if they -differ. A configuration edited by hand is therefore put back the way it should -be, which is exactly what `reconciliation_repairs_a_configuration_edited_by_hand` -in `tests/wireguard.rs` checks. - -Deactivating a network removes its interface but **keeps its key**, so coming -back later keeps the same overlay address. Agent shutdown removes every -interface the plugin created. - -## What the plugin needs from the core - -Three small additions to the `IpPlugin` contract, all generic rather than -WireGuard-specific: - -* `on_network_activated` — prepare per-network state before any peer appears; -* `attach(PluginContext)` — a handle to ask for a re-announcement when the - plugin's own capability changes, and to report an error from its own tasks; -* `shutdown` — remove system objects during a bounded agent shutdown. - -Errors reported through the context are counted by the owning network's -runtime, so `NetworkMetrics::plugin_errors` and `Event::PluginError` always -agree. +There is no external configuration file and no command line tool, so unlike a +kernel-WireGuard setup there is nothing outside this process for anybody to +edit. Reconciliation is purely "do the running tunnels match what is known". ## Using it +```bash +# On both machines +tsunagi up --network lab --secret "$SECRET" --wireguard +``` + +See the two-machine walkthrough in [../README.md](../README.md#trying-it-on-two-machines). + +From the library: + ```rust,no_run use std::sync::Arc; -use std::time::Duration; use tsunagi::config::{AgentConfig, StoragePaths, TransportPolicy}; use tsunagi::dataplane::IpPlugin; -use tsunagi::dataplane::wireguard::{WgToolBackend, WireguardConfig, WireguardPlugin}; +use tsunagi::dataplane::wireguard::{MemoryTunFactory, WireguardConfig, WireguardPlugin}; use tsunagi::identity::{NetworkName, NetworkSecret}; use tsunagi::{Agent, Result}; @@ -147,12 +142,14 @@ use tsunagi::{Agent, Result}; async fn main() -> Result<()> { let paths = StoragePaths::user_default()?; - // The plugin's own state, separate from the agent's. - let wireguard = WireguardConfig::new(paths.state_dir.join("wireguard")); - let backend = WgToolBackend::new().expect("Linux with wg and CAP_NET_ADMIN"); - let plugin = WireguardPlugin::open(wireguard, Arc::new(backend)) - .await - .expect("wireguard plugin"); + // MemoryTunFactory needs no privileges; swap in SystemTunFactory for a + // real interface. + let plugin = WireguardPlugin::open( + WireguardConfig::new(paths.state_dir.join("wireguard")), + Arc::new(MemoryTunFactory::new()), + ) + .await + .expect("wireguard plugin"); let agent = Agent::spawn( AgentConfig::new(paths) @@ -162,35 +159,30 @@ async fn main() -> Result<()> { .await?; let network = agent - .join_network(&NetworkName::new("kitchen-table")?, &NetworkSecret::generate()) + .join_network(&NetworkName::new("lab")?, &NetworkSecret::generate()) .await?; if let Some(view) = plugin.overview(network) { - println!("{} on {} at {}", view.interface, view.overlay_prefix, view.overlay_address); + println!("{} on {}", view.interface, view.overlay_address); } - - tokio::time::sleep(Duration::from_secs(60)).await; - agent.shutdown().await; // removes the interface + agent.shutdown().await; Ok(()) } ``` -There is a runnable version in `examples/wireguard_mesh.rs`, which uses the -in-memory backend by default and the real one with `--real`. - ## Limits and future work -* **Full mesh only.** Every member configures every other member. Routing - through an intermediate participant is not implemented. -* **No IPv4 overlay.** Addressing is IPv6 ULA, because it can be derived +* **Full mesh only.** Every member runs a tunnel to every other member. + Routing through an intermediate participant is not implemented. +* **IPv6 overlay only.** Addressing is IPv6 ULA because it can be derived collision-free. An IPv4 overlay would need an allocator, which needs the agreed state described in [sync-model.md](sync-model.md). -* **Peer membership is session-scoped.** A peer disappears from the - configuration when its control session ends. Persisting membership across a - long absence is part of the same future work. -* **`WgToolBackend` is Linux only.** A netlink backend, and backends for macOS - and Windows, are not implemented. `WgToolBackend::new()` fails with a clear - message elsewhere. -* **No MTU or path discovery.** The MTU is a configured constant. -* **The real backend is not exercised by the default suite.** It needs root, - so its tests live in `tests/wireguard_system.rs` behind `--ignored`. +* **No routes, DNS or firewall rules.** The plugin creates its interface and + nothing else. Anything beyond the overlay `/64` is the operator's business. +* **Membership is session-scoped.** A peer leaves the overlay when its control + session ends; surviving a long absence is the same future work. +* **Userspace costs CPU.** Kernel WireGuard is faster. A kernel backend could + return behind the same boundary, but it would give up transport-provided NAT + traversal unless paired with a local proxy. +* **The system interface path is barely exercised by the default suite**, + because it needs privileges. Everything else about the data plane is. diff --git a/examples/wireguard_mesh.rs b/examples/wireguard_mesh.rs index 98545ea..470d4e4 100644 --- a/examples/wireguard_mesh.rs +++ b/examples/wireguard_mesh.rs @@ -1,23 +1,22 @@ -//! Two agents forming a WireGuard overlay, printed step by step. +//! Two agents forming a WireGuard overlay and exchanging a real IP packet. //! //! ```text //! cargo run --example wireguard_mesh //! ``` //! -//! By default it uses the in-memory backend, so it needs no privileges and -//! changes nothing on the host: it shows the configuration each agent *would* -//! apply. Pass `--real` to drive the actual `wg` and `ip` tools instead, which -//! needs Linux and `CAP_NET_ADMIN`. +//! It uses an in-memory packet interface, so it needs no privileges and +//! changes nothing on the host: the WireGuard handshake, the encryption and +//! the transport over iroh are all real, only the TUN device is simulated. -use std::net::IpAddr; +use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; +use bytes::Bytes; use tsunagi::config::{AgentConfig, StoragePaths, TransportPolicy}; use tsunagi::dataplane::IpPlugin; use tsunagi::dataplane::wireguard::{ - AdvertisePolicy, PortPolicy, RecordingBackend, WireguardBackend, WireguardConfig, - WireguardPlugin, + MemoryTun, MemoryTunFactory, WireguardConfig, WireguardPlugin, }; use tsunagi::discovery::SharedMemoryDiscovery; use tsunagi::identity::{NetworkName, NetworkSecret}; @@ -26,32 +25,21 @@ use tsunagi::{Agent, NetworkId, Result}; struct Node { agent: Agent, plugin: Arc, - backend: Option, + tuns: MemoryTunFactory, } async fn start( root: &std::path::Path, discovery: &SharedMemoryDiscovery, prefix: &str, - advertise: IpAddr, - real: bool, ) -> Result { - let recording = (!real).then(RecordingBackend::new); - let backend: Arc = match &recording { - Some(backend) => Arc::new(backend.clone()), - None => Arc::new( - tsunagi::dataplane::wireguard::WgToolBackend::new() - .map_err(|err| tsunagi::Error::Discovery(err.to_string()))?, - ), - }; - - let wireguard = WireguardConfig::new(root.join("wireguard")) - .with_interface_prefix(prefix) - .with_advertise(AdvertisePolicy::Explicit(vec![advertise])) - .with_ports(PortPolicy::Fixed(if real { 51820 } else { 51821 })); - let plugin = WireguardPlugin::open(wireguard, backend) - .await - .map_err(|err| tsunagi::Error::Discovery(err.to_string()))?; + let tuns = MemoryTunFactory::new(); + let plugin = WireguardPlugin::open( + WireguardConfig::new(root.join("wireguard")).with_interface_prefix(prefix), + Arc::new(tuns.clone()), + ) + .await + .map_err(|err| tsunagi::Error::Discovery(err.to_string()))?; let agent = Agent::spawn( AgentConfig::new(StoragePaths::under(root)) @@ -66,49 +54,67 @@ async fn start( Ok(Node { agent, plugin, - backend: recording, + tuns, }) } fn report(label: &str, node: &Node, network: NetworkId) { let Some(view) = node.plugin.overview(network) else { - println!("{label}: the plugin has not prepared this network yet"); + println!("{label}: not prepared yet"); return; }; println!("\n{label}"); - println!(" interface {}", view.interface); - println!(" public key {}", view.public_key); + println!(" interface {} (mtu {})", view.interface, view.mtu); + println!(" public key {}", view.public_key); println!( - " overlay {} in {}", - view.overlay_address, view.overlay_prefix + " overlay {} in {}/{}", + view.overlay_address, view.overlay_prefix, view.overlay_prefix_len ); - println!(" listening on :{}", view.listen_port); - println!(" advertising {:?}", view.advertised); for peer in &view.peers { - println!( - " peer {} -> {} via {:?}", - peer.public_key.fmt_short(), - peer.overlay_address, - peer.endpoint - ); - } - if let Some(backend) = &node.backend - && let Some(state) = backend.state(&view.interface) - { - println!(" applied {} peer(s)", state.peers.len()); - for peer in &state.peers { - println!( - " AllowedIPs for {} = {:?}", + match &peer.tunnel { + Some(tunnel) => println!( + " peer {} at {} — handshake {:?}, tx {} rx {}, path {}", peer.public_key.fmt_short(), - peer.allowed_ips - .iter() - .map(ToString::to_string) - .collect::>() - ); + peer.overlay_address, + tunnel.health.since_handshake, + tunnel.stats.tx_packets, + tunnel.stats.rx_packets, + tunnel.path + ), + None => println!( + " peer {} at {} — no data link yet", + peer.public_key.fmt_short(), + peer.overlay_address + ), } } } +fn ipv6_packet(source: Ipv6Addr, destination: Ipv6Addr, payload: &[u8]) -> Bytes { + let mut packet = Vec::with_capacity(40 + payload.len()); + packet.push(6 << 4); + packet.extend_from_slice(&[0, 0, 0]); + packet.extend_from_slice(&(payload.len() as u16).to_be_bytes()); + packet.push(59); + packet.push(64); + packet.extend_from_slice(&source.octets()); + packet.extend_from_slice(&destination.octets()); + packet.extend_from_slice(payload); + Bytes::from(packet) +} + +fn overlay_of(node: &Node, network: NetworkId) -> Option { + match node.plugin.overview(network)?.overlay_address { + IpAddr::V6(addr) => Some(addr), + IpAddr::V4(_) => None, + } +} + +fn tun_of(node: &Node, network: NetworkId) -> Option> { + let view = node.plugin.overview(network)?; + node.tuns.device(&view.interface) +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -118,12 +124,7 @@ async fn main() -> Result<()> { ) .init(); - let real = std::env::args().any(|arg| arg == "--real"); - if real { - println!("driving the real wg/ip tools; this needs Linux and CAP_NET_ADMIN\n"); - } else { - println!("using the in-memory backend; nothing on this host is changed\n"); - } + println!("in-memory packet interface: nothing on this host is changed\n"); let root = match tempfile::TempDir::new() { Ok(root) => root, @@ -134,22 +135,8 @@ async fn main() -> Result<()> { }; let discovery = SharedMemoryDiscovery::new(); - let alice = start( - &root.path().join("alice"), - &discovery, - "wga", - "10.88.0.1".parse().unwrap_or(IpAddr::from([10, 88, 0, 1])), - real, - ) - .await?; - let bob = start( - &root.path().join("bob"), - &discovery, - "wgb", - "10.88.0.2".parse().unwrap_or(IpAddr::from([10, 88, 0, 2])), - real, - ) - .await?; + let alice = start(&root.path().join("alice"), &discovery, "wga").await?; + let bob = start(&root.path().join("bob"), &discovery, "wgb").await?; let name = NetworkName::new("wireguard-demo")?; let secret = NetworkSecret::generate(); @@ -162,34 +149,43 @@ async fn main() -> Result<()> { bob.agent.join_network(&name, &secret).await?; println!("network id: {network}"); - // Wait until both sides configured one peer, under a bounded deadline. - let deadline = std::time::Instant::now() + Duration::from_secs(20); + let deadline = Instant::now() + Duration::from_secs(20); loop { let ready = [&alice, &bob].iter().all(|node| { - let Some(view) = node.plugin.overview(network) else { - return false; - }; - if view.peers.len() != 1 { - return false; - } - // With the in-memory backend we can also wait for the - // configuration to actually be applied. - match &node.backend { - Some(backend) => backend - .state(&view.interface) - .map(|state| state.peers.len() == 1) - .unwrap_or(false), - None => true, - } + node.plugin + .overview(network) + .map(|view| view.established_peers() == 1) + .unwrap_or(false) }); if ready { break; } - if std::time::Instant::now() > deadline { - println!("\nthe overlay did not converge in time"); - break; + if Instant::now() > deadline { + println!("\nthe overlay did not come up in time"); + report("alice", &alice, network); + report("bob", &bob, network); + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // A real IP packet, encrypted by WireGuard and carried over iroh. + if let (Some(from), Some(to), Some(tun_a), Some(tun_b)) = ( + overlay_of(&alice, network), + overlay_of(&bob, network), + tun_of(&alice, network), + tun_of(&bob, network), + ) { + tun_a.push_from_os(ipv6_packet(from, to, b"hello over the overlay")); + match tokio::time::timeout(Duration::from_secs(5), tun_b.pop_to_os()).await { + Ok(Some(packet)) => println!( + "\nbob received {} bytes from {}: {:?}", + packet.len(), + from, + String::from_utf8_lossy(&packet[40..]) + ), + _ => println!("\nthe packet did not arrive"), } - tokio::time::sleep(Duration::from_millis(100)).await; } report("alice", &alice, network); diff --git a/src/agent/events.rs b/src/agent/events.rs index 9137ec9..a61cf6d 100644 --- a/src/agent/events.rs +++ b/src/agent/events.rs @@ -100,6 +100,35 @@ pub enum Event { /// Why it was discarded. Free of secrets. reason: String, }, + /// A data plane link to a peer is up. + /// + /// The data plane is a separate connection from the control plane; this + /// says nothing about the control session, and vice versa. + DataLinkUp { + /// The network. + network: NetworkId, + /// The peer. + peer: EndpointId, + /// Plugin protocol the link carries. + protocol: String, + /// What the transport reports about the path in use. + path: String, + /// Largest datagram the link can carry. + max_datagram: usize, + }, + /// A data plane link went away or could not be opened. + /// + /// Never fatal: the control plane keeps running and the link is retried. + DataLinkDown { + /// The network. + network: NetworkId, + /// The peer. + peer: EndpointId, + /// Plugin protocol the link would have carried. + protocol: String, + /// Why it is not up. + reason: String, + }, /// An IP plugin reported an error. Never fatal. PluginError { /// The network the call was scoped to. diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 123940e..2082e69 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -37,6 +37,8 @@ use tokio::sync::{RwLock, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use crate::config::{AgentConfig, Limits}; +use crate::dataplane::transport::PacketTransport; +use crate::dataplane::transport::iroh_link::{IrohTransport, TransportContext}; use crate::dataplane::{PluginContext, PluginRequest}; use crate::error::{Error, Result}; use crate::identity::{DeviceIdentity, NetworkId, NetworkKeys, NetworkName, NetworkSecret}; @@ -82,20 +84,47 @@ struct Inner { shutdown: Shutdown, accept_task: std::sync::Mutex>>, plugin_task: std::sync::Mutex>>, + transport: std::sync::OnceLock>, } -impl Drop for Inner { - fn drop(&mut self) { - // Nothing here awaits; this is only a safety net for a handle that was - // dropped without an explicit shutdown. - self.shutdown.trigger(); - for guard in [&self.accept_task, &self.plugin_task] { - if let Ok(mut guard) = guard.lock() - && let Some(task) = guard.take() - { - task.abort(); +/// Answers the data plane transport's questions about the agent. +/// +/// Holds a weak reference on purpose: the transport lives inside the agent, so +/// a strong one would be a cycle and the agent — with its open databases and +/// its directory lock — would never be released. +#[derive(Debug)] +struct TransportCtx(Weak); + +impl TransportContext for TransportCtx { + fn snapshot(&self) -> crate::BoxFuture<'_, HashMap> { + Box::pin(async move { + let Some(inner) = self.0.upgrade() else { + return HashMap::new(); + }; + inner + .networks + .read() + .await + .iter() + .map(|(id, handle)| (*id, handle.keys.clone())) + .collect() + }) + } + + fn serves<'a>(&'a self, network: NetworkId, protocol: &'a str) -> crate::BoxFuture<'a, bool> { + Box::pin(async move { + let Some(inner) = self.0.upgrade() else { + return false; + }; + if !inner.networks.read().await.contains_key(&network) { + return false; } - } + inner + .config + .plugins + .iter() + .any(|plugin| plugin.protocol_id() == protocol) + }) } } @@ -127,9 +156,20 @@ impl Agent { shutdown: Shutdown::new(), accept_task: std::sync::Mutex::new(None), plugin_task: std::sync::Mutex::new(None), + transport: std::sync::OnceLock::new(), config, }); + // The data plane rides on iroh too, which is where it gets hole + // punching and relay fallback from. It is a separate ALPN and a + // separate connection, so the two planes stay independent. + let transport: Arc = Arc::new(IrohTransport::new( + inner.adapter.clone(), + Arc::clone(&inner.limits), + Arc::new(TransportCtx(Arc::downgrade(&inner))) as Arc, + )); + let _ = inner.transport.set(transport); + if let CacheOutcome::Reset(reason) = inner.storage.cache_outcome().clone() { let _ = inner.events.send(Event::CacheReset { reason }); } @@ -257,6 +297,7 @@ impl Agent { discovery_interval: self.inner.config.discovery_interval, plugins: self.inner.config.plugins.clone(), hostname: self.inner.hostname.clone(), + transport: self.inner.transport.get().cloned(), }); networks.insert(network_id, handle); drop(networks); @@ -625,6 +666,12 @@ async fn handle_incoming(inner: Arc, incoming: iroh::endpoint::Incoming) }; let peer = conn.remote_id(); + // Two protocols share the endpoint; they are told apart here and never mix. + if conn.alpn() == crate::proto::message::DATA_ALPN { + handle_inbound_data(inner, conn).await; + return; + } + let (mut send, mut recv) = match conn.accept_bi().await { Ok(streams) => streams, Err(err) => { @@ -696,6 +743,45 @@ async fn handle_incoming(inner: Arc, incoming: iroh::endpoint::Incoming) } } +/// Completes an inbound data plane connection and routes it to its network. +async fn handle_inbound_data(inner: Arc, conn: iroh::endpoint::Connection) { + let Some(transport) = inner.transport.get().cloned() else { + conn.close(5u32.into(), b"data plane not ready"); + return; + }; + // Downcasting is avoided by keeping the accept side on the concrete type. + let Some(iroh_transport) = transport.as_ref().as_any().downcast_ref::() else { + conn.close(5u32.into(), b"unsupported data transport"); + return; + }; + + let inbound = match iroh_transport.accept(conn).await { + Ok(inbound) => inbound, + Err(err) => { + tracing::debug!(%err, "inbound data channel rejected"); + return; + } + }; + + let sender = { + let networks = inner.networks.read().await; + networks + .get(&inbound.network) + .map(|handle| handle.commands.clone()) + }; + let Some(sender) = sender else { + // The network went away while the channel was being set up. + return; + }; + if sender + .send(NetCommand::InboundLink(Box::new(inbound))) + .await + .is_err() + { + tracing::debug!("network runtime stopped before the data link was installed"); + } +} + /// Picks the hostname to announce. /// /// Order: explicit configuration, then what the state store already holds, then diff --git a/src/agent/network.rs b/src/agent/network.rs index efb5c68..35be599 100644 --- a/src/agent/network.rs +++ b/src/agent/network.rs @@ -5,7 +5,7 @@ //! explicit [`NetworkId`], so deactivating or breaking one network cannot //! disturb another and cannot stop the agent. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -15,6 +15,7 @@ use tokio::sync::{broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use crate::config::{Limits, ReconnectPolicy}; +use crate::dataplane::transport::{InboundLink, PacketTransport, SharedLink}; use crate::dataplane::{PluginCapability, SharedPlugin}; use crate::discovery::{Candidate, CandidateSource, NetworkDiscovery}; use crate::error::{Error, Result}; @@ -50,6 +51,8 @@ pub(crate) enum NetCommand { message: ControlMessage, reply: oneshot::Sender, }, + /// A peer opened a data plane link towards us. + InboundLink(Box), Status { reply: oneshot::Sender>, }, @@ -69,6 +72,7 @@ impl std::fmt::Debug for NetCommand { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { NetCommand::Inbound(_) => f.write_str("Inbound"), + NetCommand::InboundLink(link) => write!(f, "InboundLink({})", link.protocol), NetCommand::Send { peer, message, .. } => { write!(f, "Send({}, {})", peer.fmt_short(), kind(message)) } @@ -110,6 +114,15 @@ pub(crate) struct RuntimeParams { pub(crate) discovery_interval: Duration, pub(crate) plugins: Vec, pub(crate) hostname: String, + /// How data plane links are opened. `None` disables the data plane. + pub(crate) transport: Option>, +} + +/// Outcome of one attempt to open a data plane link. +struct LinkOutcome { + peer: EndpointId, + protocol: String, + result: Result, } /// Outcome of one outbound dial. @@ -175,6 +188,12 @@ struct Runtime { session_events_rx: mpsc::Receiver, dial_results_tx: mpsc::Sender, dial_results_rx: mpsc::Receiver, + /// Live data plane links, keyed by peer and plugin protocol. + links: HashMap<(EndpointId, String), SharedLink>, + /// Links currently being opened, so we do not start two. + opening: HashSet<(EndpointId, String)>, + link_results_tx: mpsc::Sender, + link_results_rx: mpsc::Receiver, } impl Runtime { @@ -183,6 +202,7 @@ impl Runtime { let local_id = params.adapter.endpoint_id(); let (session_events_tx, session_events_rx) = mpsc::channel(256); let (dial_results_tx, dial_results_rx) = mpsc::channel(64); + let (link_results_tx, link_results_rx) = mpsc::channel(64); Self { params, network_id, @@ -196,6 +216,10 @@ impl Runtime { session_events_rx, dial_results_tx, dial_results_rx, + links: HashMap::new(), + opening: HashSet::new(), + link_results_tx, + link_results_rx, } } @@ -226,7 +250,15 @@ impl Runtime { self.handle_dial_result(result).await; } } - _ = ticker.tick() => self.discovery_round().await, + result = self.link_results_rx.recv() => { + if let Some(result) = result { + self.handle_link_result(result); + } + } + _ = ticker.tick() => { + self.discovery_round().await; + self.ensure_links(); + } } } @@ -242,6 +274,7 @@ impl Runtime { .unpublish(self.params.keys.discovery_key(), self.local_id) .await; } + self.links.clear(); let peers: Vec = self.sessions.keys().copied().collect(); for peer in peers { if let Some(session) = self.sessions.remove(&peer) { @@ -260,6 +293,7 @@ impl Runtime { NetCommand::Inbound(inbound) => { self.install_session(*inbound).await; } + NetCommand::InboundLink(inbound) => self.install_link(*inbound), NetCommand::Send { peer, message, @@ -516,6 +550,163 @@ impl Runtime { } } + // ------------------------------------------------------------ data plane + + /// Protocol ids this agent has a plugin for. + fn served_protocols(&self) -> Vec { + self.params + .plugins + .iter() + .map(|plugin| plugin.protocol_id().to_string()) + .collect() + } + + /// Opens whatever data plane links are missing, and forgets dead ones. + /// + /// Only one side dials, chosen by a rule both sides compute the same way, + /// so two agents never open two links for the same thing. + fn ensure_links(&mut self) { + let Some(transport) = self.params.transport.clone() else { + return; + }; + let served = self.served_protocols(); + if served.is_empty() { + return; + } + + let mut dead: Vec<(EndpointId, String)> = Vec::new(); + for (key, link) in &self.links { + if link.is_closed() { + dead.push(key.clone()); + } + } + for (peer, protocol) in dead { + self.links.remove(&(peer, protocol.clone())); + self.emit(Event::DataLinkDown { + network: self.network_id, + peer, + protocol, + reason: "link closed".into(), + }); + } + + let wanted: Vec<(EndpointId, String)> = self + .sessions + .values() + .flat_map(|session| { + let peer = session.peer; + session + .capabilities + .iter() + .filter(|capability| capability.enabled) + .map(move |capability| (peer, capability.protocol.clone())) + }) + .filter(|(_, protocol)| served.contains(protocol)) + .collect(); + + for (peer, protocol) in wanted { + let key = (peer, protocol.clone()); + if self.links.contains_key(&key) || self.opening.contains(&key) { + continue; + } + // The smaller endpoint id dials; the other side accepts. Both + // compute this identically, so exactly one link is created. + if self.local_id.as_bytes() >= peer.as_bytes() { + continue; + } + self.opening.insert(key); + + let results = self.link_results_tx.clone(); + let transport = Arc::clone(&transport); + let network = self.network_id; + let shutdown = self.shutdown.clone(); + tokio::spawn(async move { + let result = tokio::select! { + biased; + _ = shutdown.wait() => Err("network deactivated".to_string()), + result = transport.open(network, peer, &protocol) => { + result.map_err(|err| err.to_string()) + } + }; + let _ = results + .send(LinkOutcome { + peer, + protocol, + result, + }) + .await; + }); + } + } + + fn handle_link_result(&mut self, outcome: LinkOutcome) { + let key = (outcome.peer, outcome.protocol.clone()); + self.opening.remove(&key); + match outcome.result { + Ok(link) => self.adopt_link(outcome.peer, outcome.protocol, link), + Err(reason) => { + // A data plane that cannot be set up is reported, never fatal. + self.metrics.data_link_failures += 1; + self.emit(Event::DataLinkDown { + network: self.network_id, + peer: outcome.peer, + protocol: outcome.protocol, + reason, + }); + } + } + } + + fn install_link(&mut self, inbound: InboundLink) { + self.adopt_link(inbound.peer, inbound.protocol, inbound.link); + } + + /// Hands a link to the plugin that owns its protocol. + fn adopt_link(&mut self, peer: EndpointId, protocol: String, link: SharedLink) { + let Some(plugin) = self + .params + .plugins + .iter() + .find(|plugin| plugin.protocol_id() == protocol) + .cloned() + else { + return; + }; + + let path = link.path_description(); + let max_datagram = link.max_datagram_size(); + self.links + .insert((peer, protocol.clone()), Arc::clone(&link)); + plugin.on_peer_link(self.network_id, peer, link); + self.metrics.data_links_established += 1; + self.emit(Event::DataLinkUp { + network: self.network_id, + peer, + protocol, + path, + max_datagram, + }); + } + + /// Drops every link to a peer. + fn drop_links_for(&mut self, peer: EndpointId) { + let keys: Vec<(EndpointId, String)> = self + .links + .keys() + .filter(|(id, _)| *id == peer) + .cloned() + .collect(); + for key in keys { + self.links.remove(&key); + self.emit(Event::DataLinkDown { + network: self.network_id, + peer, + protocol: key.1, + reason: "peer session ended".into(), + }); + } + } + // --------------------------------------------------------------- sessions async fn install_session(&mut self, inbound: InboundSession) { @@ -682,6 +873,7 @@ impl Runtime { session.conn.close(0u32.into(), b"session ended"); } self.metrics.disconnects += 1; + self.drop_links_for(peer); for plugin in &self.params.plugins { plugin.on_peer_gone(self.network_id, peer); } @@ -711,6 +903,7 @@ impl Runtime { session.capabilities = capabilities.clone(); } self.dispatch_capabilities(peer, &capabilities); + self.ensure_links(); } ControlMessage::Ping { seq, payload } => { let pong = ControlMessage::Pong { diff --git a/src/agent/status.rs b/src/agent/status.rs index 3dd45fc..116be5d 100644 --- a/src/agent/status.rs +++ b/src/agent/status.rs @@ -94,6 +94,10 @@ pub struct NetworkMetrics { pub protocol_violations: u64, /// Errors reported by IP plugins. Never fatal. pub plugin_errors: u64, + /// Data plane links that were established. + pub data_links_established: u64, + /// Attempts to open a data plane link that failed. + pub data_link_failures: u64, } /// Status of one network. diff --git a/src/bin/tsunagi.rs b/src/bin/tsunagi.rs new file mode 100644 index 0000000..4487fd3 --- /dev/null +++ b/src/bin/tsunagi.rs @@ -0,0 +1,554 @@ +//! The `tsunagi` command line agent. +//! +//! This binary owns everything the library deliberately refuses to do: it +//! starts the tokio runtime, installs a logging subscriber and handles +//! Ctrl-C. The library itself does none of that. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use clap::{Args, Parser, Subcommand, ValueEnum}; +use tsunagi::agent::Event; +use tsunagi::config::{AgentConfig, StoragePaths, TransportPolicy}; +use tsunagi::dataplane::IpPlugin; +use tsunagi::dataplane::wireguard::{ + MemoryTunFactory, TunFactory, WireguardConfig, WireguardPlugin, +}; +use tsunagi::discovery::{CompositeDiscovery, NetworkDiscovery, StaticBootstrap}; +use tsunagi::identity::{NetworkName, NetworkSecret}; +use tsunagi::iroh_types::EndpointAddr; +use tsunagi::{Agent, NetworkId}; + +/// A small agent for private mesh networks. +#[derive(Debug, Parser)] +#[command(name = "tsunagi", version, about, long_about = None)] +struct Cli { + /// Log filter, for example `info` or `tsunagi=debug`. + #[arg(long, global = true, env = "TSUNAGI_LOG", default_value = "warn")] + log: String, + + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Generates a fresh network secret and prints it. + Secret, + /// Reports what this machine can and cannot do. + Doctor(PathArgs), + /// Shows this device's identity without joining anything. + Id(PathArgs), + /// Joins a network and runs until interrupted. + Up(UpArgs), +} + +#[derive(Debug, Args, Clone)] +struct PathArgs { + /// Directory for the mandatory state. Defaults to the platform location. + #[arg(long, env = "TSUNAGI_STATE_DIR")] + state_dir: Option, + /// Directory for the disposable cache. Defaults to the platform location. + #[arg(long, env = "TSUNAGI_CACHE_DIR")] + cache_dir: Option, +} + +impl PathArgs { + fn resolve(&self) -> Result { + let mut paths = StoragePaths::user_default()?; + if let Some(dir) = &self.state_dir { + paths.state_dir = dir.clone(); + } + if let Some(dir) = &self.cache_dir { + paths.cache_dir = dir.clone(); + } + Ok(paths) + } +} + +/// How much external connectivity machinery the endpoint may use. +#[derive(Debug, Clone, Copy, ValueEnum)] +enum Transport { + /// Loopback and the local network only. No relays, no address lookup. + Local, + /// Public address lookup, but no relays. + Direct, + /// iroh's defaults: address lookup plus the public n0 relays. + N0, +} + +impl From for TransportPolicy { + fn from(value: Transport) -> Self { + match value { + Transport::Local => TransportPolicy::LocalOnly, + Transport::Direct => TransportPolicy::DirectOnly, + Transport::N0 => TransportPolicy::N0Defaults, + } + } +} + +#[derive(Debug, Args)] +struct UpArgs { + #[command(flatten)] + paths: PathArgs, + + /// Network name. Must be identical on every participant. + #[arg(long, short = 'n')] + network: String, + + /// The shared secret, as printed by `tsunagi secret`. + #[arg( + long, + short = 's', + env = "TSUNAGI_SECRET", + conflicts_with = "secret_file" + )] + secret: Option, + + /// Read the shared secret from a file instead of the command line. + #[arg(long)] + secret_file: Option, + + /// Hostname to announce. Defaults to the machine's. + #[arg(long)] + hostname: Option, + + /// How much external connectivity to use. + #[arg(long, value_enum, default_value_t = Transport::N0)] + transport: Transport, + + /// A peer to contact, as `` or `@,...`. + /// + /// One agent needs to know another to begin with. Repeat for several. + #[arg(long = "peer", value_name = "PEER")] + peers: Vec, + + /// Local address to bind. Repeat for several; defaults to iroh's choice. + #[arg(long = "bind", value_name = "ADDR")] + binds: Vec, + + /// Run the WireGuard data plane. + #[arg(long)] + wireguard: bool, + + /// Do not create a real network interface. + /// + /// The WireGuard tunnels still run and handshake, so the mesh can be + /// verified with no privileges; traffic just does not reach the + /// operating system. + #[arg(long)] + no_tun: bool, + + /// Interface name prefix for the WireGuard data plane. + #[arg(long, default_value = "tsun")] + wg_prefix: String, + + /// Interface MTU for the WireGuard data plane. + #[arg(long)] + wg_mtu: Option, + + /// How often to print a status summary, in seconds. Zero disables it. + #[arg(long, default_value_t = 15)] + status_interval: u64, +} + +impl UpArgs { + fn load_secret(&self) -> Result> { + let text = match (&self.secret, &self.secret_file) { + (Some(secret), _) => secret.clone(), + (None, Some(path)) => std::fs::read_to_string(path)?, + (None, None) => { + return Err("provide --secret, --secret-file or TSUNAGI_SECRET".into()); + } + }; + let text = text.trim(); + // The canonical form is preferred, but a raw high-entropy value is + // accepted so an existing secret can be reused. + match NetworkSecret::decode(text) { + Ok(secret) => Ok(secret), + Err(_) => Ok(NetworkSecret::from_bytes(text.as_bytes().to_vec())?), + } + } +} + +/// Parses `` or `@,`. +fn parse_peer(text: &str) -> Result { + let (id_text, addr_text) = match text.split_once('@') { + Some((id, addrs)) => (id, Some(addrs)), + None => (text, None), + }; + let id: tsunagi::iroh_types::EndpointId = id_text + .parse() + .map_err(|err| format!("`{id_text}` is not an endpoint id: {err}"))?; + let mut addr = EndpointAddr::new(id); + if let Some(addrs) = addr_text { + for entry in addrs.split(',') { + let socket: SocketAddr = entry + .trim() + .parse() + .map_err(|err| format!("`{entry}` is not an address: {err}"))?; + addr = addr.with_ip_addr(socket); + } + } + Ok(addr) +} + +fn main() -> std::process::ExitCode { + let cli = Cli::parse(); + + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::new(&cli.log)) + .with_writer(std::io::stderr) + .init(); + + // The library never starts a runtime; this binary owns it. + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(err) => { + eprintln!("cannot start the async runtime: {err}"); + return std::process::ExitCode::FAILURE; + } + }; + + match runtime.block_on(run(cli.command)) { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(err) => { + eprintln!("error: {err}"); + std::process::ExitCode::FAILURE + } + } +} + +async fn run(command: Command) -> Result<(), Box> { + match command { + Command::Secret => { + let secret = NetworkSecret::generate(); + println!("{}", secret.encode().as_str()); + eprintln!( + "\nShare this with every participant, over a channel you trust.\n\ + Anyone who has it can join the network." + ); + Ok(()) + } + Command::Doctor(paths) => doctor(paths).await, + Command::Id(paths) => show_id(paths).await, + Command::Up(args) => up(args).await, + } +} + +async fn show_id(paths: PathArgs) -> Result<(), Box> { + let paths = paths.resolve()?; + println!("state directory {}", paths.state_dir.display()); + println!("cache directory {}", paths.cache_dir.display()); + + let agent = + Agent::spawn(AgentConfig::new(paths).with_transport(TransportPolicy::LocalOnly)).await?; + println!("endpoint id {}", agent.endpoint_id()); + println!("hostname {}", agent.hostname()); + for network in agent.list_networks().await? { + println!( + "network {} ({}) auto-start={}", + network.name, network.network_id, network.auto_start + ); + } + agent.shutdown().await; + Ok(()) +} + +async fn doctor(paths: PathArgs) -> Result<(), Box> { + let paths = paths.resolve()?; + println!("tsunagi doctor\n"); + + println!("state directory {}", paths.state_dir.display()); + println!("cache directory {}", paths.cache_dir.display()); + match std::fs::create_dir_all(&paths.state_dir) { + Ok(()) => println!(" writable yes"), + Err(err) => println!(" writable NO ({err})"), + } + + println!("\ncontrol plane"); + println!(" needs outbound UDP; no privileges"); + println!(" status always available"); + + println!("\ndata plane (WireGuard)"); + println!(" implementation userspace (boringtun); no kernel module needed"); + #[cfg(feature = "tun-device")] + { + let tun_path = std::path::Path::new("/dev/net/tun"); + if cfg!(target_os = "linux") { + if tun_path.exists() { + match std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(tun_path) + { + Ok(_) => println!(" /dev/net/tun openable"), + Err(err) => println!(" /dev/net/tun present but not openable ({err})"), + } + } else { + println!(" /dev/net/tun missing (load the `tun` module)"); + } + } + println!(" interfaces supported on this build"); + } + #[cfg(not(feature = "tun-device"))] + println!(" interfaces not built in (enable the `tun-device` feature)"); + + #[cfg(unix)] + { + // Creating a network interface needs CAP_NET_ADMIN, which in practice + // means root unless capabilities were granted explicitly. + let euid = std::fs::metadata("/proc/self").ok().map(|_| ()); + let _ = euid; + println!( + " privileges creating an interface needs CAP_NET_ADMIN; \ + use --no-tun to run without it" + ); + } + + println!("\nlocal addresses"); + let state = netwatch_addresses().await; + if state.is_empty() { + println!(" none found"); + } + for addr in state { + println!(" {addr}"); + } + Ok(()) +} + +async fn netwatch_addresses() -> Vec { + // Best effort; used for diagnostics only. + let state = netwatch::interfaces::State::new().await; + let mut addresses = state.local_addresses.regular; + addresses.sort(); + addresses.dedup(); + addresses +} + +async fn up(args: UpArgs) -> Result<(), Box> { + let name = NetworkName::new(args.network.clone())?; + let secret = args.load_secret()?; + let paths = args.paths.resolve()?; + + let mut bootstrap: Vec = Vec::new(); + for peer in &args.peers { + bootstrap.push(parse_peer(peer)?); + } + let discovery: Arc = + Arc::new(CompositeDiscovery::new([ + Arc::new(StaticBootstrap::new(bootstrap)) as Arc, + ])); + + let mut config = AgentConfig::new(paths.clone()) + .with_transport(args.transport.into()) + .with_discovery(discovery) + .with_discovery_interval(Duration::from_secs(5)); + if let Some(hostname) = &args.hostname { + config = config.with_hostname(hostname.clone()); + } + if !args.binds.is_empty() { + config = config.with_bind_addrs(args.binds.clone()); + } + + // The data plane is optional and never required for the control plane. + let wireguard = if args.wireguard { + let tun_factory: Arc = if args.no_tun { + Arc::new(MemoryTunFactory::new()) + } else { + system_tun_factory()? + }; + let mut wg = WireguardConfig::new(paths.state_dir.join("wireguard")) + .with_interface_prefix(args.wg_prefix.clone()); + if let Some(mtu) = args.wg_mtu { + wg = wg.with_mtu(mtu); + } + let plugin = WireguardPlugin::open(wg, tun_factory).await?; + config = config.with_plugin(plugin.clone() as Arc); + Some(plugin) + } else { + None + }; + + let agent = Agent::spawn(config).await?; + let mut events = agent.subscribe(); + let network = agent.join_network(&name, &secret).await?; + + println!("tsunagi is up"); + println!(" endpoint id {}", agent.endpoint_id()); + println!(" hostname {}", agent.hostname()); + println!(" network {name} ({network})"); + println!(" state {}", paths.state_dir.display()); + if args.peers.is_empty() { + println!( + "\nNo --peer was given, so this agent waits to be contacted.\n\ + On the other machine run:\n\n tsunagi up --network {name} --secret \\\n --peer {}\n", + agent.endpoint_id() + ); + } + println!("Press Ctrl-C to stop.\n"); + + let status_every = + (args.status_interval > 0).then(|| Duration::from_secs(args.status_interval)); + let mut ticker = status_every.map(tokio::time::interval); + + loop { + tokio::select! { + signal = tokio::signal::ctrl_c() => { + if let Err(err) = signal { + eprintln!("cannot listen for Ctrl-C: {err}"); + } + println!("\nstopping..."); + break; + } + event = events.recv() => match event { + Ok(event) => print_event(&event), + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + println!(" (missed {skipped} events)"); + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + }, + _ = async { + match ticker.as_mut() { + Some(ticker) => { ticker.tick().await; } + None => std::future::pending::<()>().await, + } + }, if ticker.is_some() => { + print_status(&agent, network, wireguard.as_deref()).await; + } + } + } + + agent.shutdown().await; + println!("stopped."); + Ok(()) +} + +#[cfg(feature = "tun-device")] +fn system_tun_factory() -> Result, Box> { + use tsunagi::dataplane::wireguard::SystemTunFactory; + Ok(Arc::new(SystemTunFactory::new())) +} + +#[cfg(not(feature = "tun-device"))] +fn system_tun_factory() -> Result, Box> { + Err("this build has no interface support; rebuild with the `tun-device` feature or pass --no-tun".into()) +} + +fn print_event(event: &Event) { + match event { + Event::PeerConnected { + peer, + transport, + rtt, + .. + } => println!( + " + peer {} connected over {transport:?} rtt={rtt:?}", + peer.fmt_short() + ), + Event::PeerDisconnected { peer, reason, .. } => { + println!(" - peer {} gone: {reason}", peer.fmt_short()) + } + Event::DataLinkUp { + peer, + protocol, + path, + max_datagram, + .. + } => println!( + " + data link to {} for {protocol}: {path}, datagram {max_datagram}", + peer.fmt_short() + ), + Event::DataLinkDown { + peer, + protocol, + reason, + .. + } => println!( + " - data link to {} for {protocol}: {reason}", + peer.fmt_short() + ), + Event::HandshakeRejected { peer, reason, .. } => println!( + " ! rejected {}: {reason}", + peer.map(|peer| peer.fmt_short().to_string()) + .unwrap_or_else(|| "a caller".into()) + ), + Event::PluginError { + protocol, reason, .. + } => println!(" ! {protocol}: {reason}"), + Event::CacheReset { reason } => println!(" ! cache was reset: {reason}"), + _ => {} + } +} + +async fn print_status(agent: &Agent, network: NetworkId, wireguard: Option<&WireguardPlugin>) { + let Ok(status) = agent.network_status(network).await else { + return; + }; + println!("\n--- status ---"); + println!( + "control: {} peer(s), {} dial failure(s), {} handshake failure(s)", + status.peers.len(), + status.metrics.dial_failures, + status.metrics.handshake_failures + ); + for peer in &status.peers { + println!( + " {} {} {:?} rtt={:?}", + peer.endpoint_id.fmt_short(), + peer.hostname.as_deref().unwrap_or("?"), + peer.transport, + peer.rtt + ); + } + + if let Some(plugin) = wireguard + && let Some(view) = plugin.overview(network) + { + println!( + "wireguard: {} on {}/{} mtu {}, {}/{} tunnel(s) established", + view.interface, + view.overlay_address, + view.overlay_prefix_len, + view.mtu, + view.established_peers(), + view.peers.len() + ); + for peer in &view.peers { + match &peer.tunnel { + Some(tunnel) => println!( + " {} {} {} tx={} rx={} dropped={} path={}", + peer.public_key.fmt_short(), + peer.overlay_address, + match tunnel.health.since_handshake { + Some(since) => format!("handshake {}s ago", since.as_secs()), + None => "NOT HANDSHAKEN".to_string(), + }, + tunnel.stats.tx_packets, + tunnel.stats.rx_packets, + tunnel.stats.dropped_wrong_source + tunnel.stats.dropped_oversize, + tunnel.path + ), + None => println!( + " {} {} waiting for a data link", + peer.public_key.fmt_short(), + peer.overlay_address + ), + } + } + if view.unroutable_packets > 0 { + println!( + " {} packet(s) for unknown addresses", + view.unroutable_packets + ); + } + } + println!(); +} diff --git a/src/dataplane/mod.rs b/src/dataplane/mod.rs index d9a6fcf..a9b5998 100644 --- a/src/dataplane/mod.rs +++ b/src/dataplane/mod.rs @@ -19,6 +19,7 @@ //! A data plane failure never stops the daemon: errors returned here are //! recorded and surfaced, the control plane keeps running. +pub mod transport; pub mod wireguard; use std::sync::Arc; @@ -29,6 +30,8 @@ use tokio::sync::mpsc; use crate::BoxFuture; use crate::identity::NetworkId; +pub use transport::{PacketLink, PacketTransport, SharedLink, TransportError}; + /// Maximum length of a plugin protocol identifier. pub const MAX_PROTOCOL_ID_LEN: usize = 32; @@ -194,7 +197,17 @@ pub trait IpPlugin: Send + Sync + std::fmt::Debug + 'static { capability: &PluginCapability, ) -> std::result::Result<(), PluginError>; + /// A data plane link to a peer is available for this plugin's protocol. + /// + /// The plugin moves its packets over this link and never learns how the + /// link is carried. A new link for a peer replaces any previous one. + fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) { + let _ = (network, peer, link); + } + /// Called when a peer's session in a network goes away. + /// + /// Any link handed to the plugin for that peer must be dropped here. fn on_peer_gone(&self, network: NetworkId, peer: EndpointId); /// Called when a network is deactivated locally. diff --git a/src/dataplane/transport/iroh_link.rs b/src/dataplane/transport/iroh_link.rs new file mode 100644 index 0000000..9657a84 --- /dev/null +++ b/src/dataplane/transport/iroh_link.rs @@ -0,0 +1,317 @@ +//! A data plane transport built on iroh. +//! +//! This is where the data plane gets NAT traversal from. iroh hole punches a +//! direct path between two peers when it can and falls back to a relay when it +//! cannot, so every plugin inherits that without implementing STUN, ICE or a +//! relay of its own. +//! +//! Data connections are separate from control connections in every way that +//! matters: their own ALPN ([`DATA_ALPN`]), their own QUIC connection, their +//! own congestion control. They carry one plugin protocol for one network. +//! A data connection that breaks or floods cannot disturb the control plane. +//! +//! Packets travel as QUIC datagrams: unreliable and unordered, which is what a +//! tunnelled UDP protocol wants, and free of the head-of-line blocking a +//! stream would add. +//! +//! The channel is authenticated exactly like a control connection — the same +//! membership handshake, bound to the same network — so a data link cannot be +//! opened by someone who does not know the network secret. + +use std::collections::HashMap; +use std::sync::Arc; + +use bytes::Bytes; +use iroh::EndpointId; +use iroh::endpoint::{Connection, RecvStream, SendStream}; + +use crate::BoxFuture; +use crate::config::Limits; +use crate::error::ProtocolError; +use crate::identity::{NetworkId, NetworkKeys}; +use crate::net::EndpointAdapter; +use crate::proto::handshake; +use crate::proto::message::{ + DATA_ALPN, DataOpen, DataOpenAck, MAX_DATA_PROTOCOL_LEN, decode, encode, +}; +use crate::proto::{read_frame, write_frame}; + +use super::{InboundLink, PacketLink, PacketTransport, SharedLink, TransportError}; + +/// What the iroh transport needs from the agent. +/// +/// Implemented by the agent, which is the only thing that knows which networks +/// are active and which plugin protocols are served. +pub trait TransportContext: Send + Sync + std::fmt::Debug + 'static { + /// Key material of every network that is active right now. + /// + /// Taken as one snapshot because the membership handshake resolves the + /// requested network synchronously, exactly as the control plane's accept + /// path does. + fn snapshot<'a>(&'a self) -> BoxFuture<'a, HashMap>; + + /// Whether a plugin protocol is served in a network. + fn serves<'a>(&'a self, network: NetworkId, protocol: &'a str) -> BoxFuture<'a, bool>; +} + +/// One authenticated datagram channel over an iroh connection. +#[derive(Debug)] +pub struct IrohLink { + network: NetworkId, + peer: EndpointId, + conn: Connection, + max_datagram: usize, + // Kept alive so the peer sees the channel as open; the connection closes + // when the link is dropped. + _send: tokio::sync::Mutex, + _recv: tokio::sync::Mutex, +} + +impl IrohLink { + fn new( + network: NetworkId, + peer: EndpointId, + conn: Connection, + peer_limit: usize, + send: SendStream, + recv: RecvStream, + ) -> Self { + let local_limit = conn.max_datagram_size().unwrap_or(0); + // Both ends must agree, so the smaller limit wins. + let max_datagram = local_limit.min(peer_limit); + Self { + network, + peer, + conn, + max_datagram, + _send: tokio::sync::Mutex::new(send), + _recv: tokio::sync::Mutex::new(recv), + } + } +} + +impl PacketLink for IrohLink { + fn network(&self) -> NetworkId { + self.network + } + + fn peer(&self) -> EndpointId { + self.peer + } + + fn max_datagram_size(&self) -> usize { + self.max_datagram + } + + fn send(&self, payload: Bytes) -> Result<(), TransportError> { + if payload.len() > self.max_datagram { + return Err(TransportError::TooLarge { + size: payload.len(), + limit: self.max_datagram, + }); + } + self.conn.send_datagram(payload).map_err(|err| { + use iroh::endpoint::SendDatagramError; + match err { + SendDatagramError::ConnectionLost(_) => TransportError::Closed, + other => TransportError::Other(other.to_string()), + } + }) + } + + fn recv(&self) -> BoxFuture<'_, Option> { + Box::pin(async move { self.conn.read_datagram().await.ok() }) + } + + fn closed(&self) -> BoxFuture<'_, ()> { + Box::pin(async move { + let _ = self.conn.closed().await; + }) + } + + fn is_closed(&self) -> bool { + self.conn.close_reason().is_some() + } + + fn path_description(&self) -> String { + // Report what iroh actually knows, never a guess. + let snapshot = crate::net::snapshot_connection(&self.conn); + match snapshot.paths.iter().find(|path| path.is_selected) { + Some(path) => format!("{:?} via {:?}", snapshot.transport, path.remote), + None => format!("{:?}, no selected path yet", snapshot.transport), + } + } +} + +/// Opens and accepts data plane links over iroh. +#[derive(Debug, Clone)] +pub struct IrohTransport { + adapter: EndpointAdapter, + limits: Arc, + lookup: Arc, +} + +impl IrohTransport { + /// Creates a transport on an existing endpoint. + pub fn new( + adapter: EndpointAdapter, + limits: Arc, + lookup: Arc, + ) -> Self { + Self { + adapter, + limits, + lookup, + } + } + + /// Completes an inbound data connection that the accept loop routed here. + /// + /// The membership handshake runs first, exactly as on a control + /// connection, so an unauthenticated caller never reaches a plugin. + pub async fn accept(&self, conn: Connection) -> Result { + let peer = conn.remote_id(); + let (mut send, mut recv) = conn + .accept_bi() + .await + .map_err(|err| TransportError::Other(format!("no data channel stream: {err}")))?; + + let local_id = self.adapter.endpoint_id(); + let known = self.lookup.snapshot().await; + let outcome = handshake::respond( + &conn, + &mut send, + &mut recv, + local_id, + &self.limits, + |network| known.get(&network).cloned(), + ) + .await + .map_err(|err| match err { + ProtocolError::UnknownNetwork => { + TransportError::Other("network is not active for the data plane".into()) + } + other => TransportError::Other(other.to_string()), + })?; + + let open: DataOpen = decode( + &read_frame(&mut recv, self.limits.max_frame_len) + .await + .map_err(|err| TransportError::Other(err.to_string()))?, + ) + .map_err(|err| TransportError::Other(err.to_string()))?; + + if open.protocol.is_empty() || open.protocol.len() > MAX_DATA_PROTOCOL_LEN { + return Err(TransportError::Other( + "data channel protocol id is out of bounds".into(), + )); + } + + let serves = self.lookup.serves(outcome.network_id, &open.protocol).await; + let max_datagram = conn.max_datagram_size().unwrap_or(0); + let ack = DataOpenAck { + accepted: serves, + max_datagram: max_datagram as u32, + }; + write_frame( + &mut send, + &encode(&ack).map_err(|err| TransportError::Other(err.to_string()))?, + self.limits.max_frame_len, + ) + .await + .map_err(|err| TransportError::Other(err.to_string()))?; + + if !serves { + conn.close(4u32.into(), b"no plugin for this protocol"); + return Err(TransportError::Declined(open.protocol)); + } + + let link = IrohLink::new(outcome.network_id, peer, conn, usize::MAX, send, recv); + Ok(InboundLink { + network: outcome.network_id, + peer, + protocol: open.protocol, + link: Arc::new(link), + }) + } +} + +impl PacketTransport for IrohTransport { + fn name(&self) -> &str { + "iroh" + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn open<'a>( + &'a self, + network: NetworkId, + peer: EndpointId, + protocol: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if protocol.is_empty() || protocol.len() > MAX_DATA_PROTOCOL_LEN { + return Err(TransportError::Other( + "data channel protocol id is out of bounds".into(), + )); + } + let keys = self + .lookup + .snapshot() + .await + .remove(&network) + .ok_or_else(|| TransportError::Other("network is not active".into()))?; + + let addr = iroh::EndpointAddr::new(peer); + let conn = self + .adapter + .endpoint() + .connect(addr, DATA_ALPN) + .await + .map_err(|err| TransportError::Unreachable(err.to_string()))?; + let (mut send, mut recv) = conn + .open_bi() + .await + .map_err(|err| TransportError::Unreachable(err.to_string()))?; + + handshake::initiate( + &conn, + &mut send, + &mut recv, + self.adapter.endpoint_id(), + &keys, + &self.limits, + ) + .await + .map_err(|err| TransportError::Other(err.to_string()))?; + + let open = DataOpen { + protocol: protocol.to_string(), + }; + write_frame( + &mut send, + &encode(&open).map_err(|err| TransportError::Other(err.to_string()))?, + self.limits.max_frame_len, + ) + .await + .map_err(|err| TransportError::Other(err.to_string()))?; + + let ack: DataOpenAck = decode( + &read_frame(&mut recv, self.limits.max_frame_len) + .await + .map_err(|err| TransportError::Other(err.to_string()))?, + ) + .map_err(|err| TransportError::Other(err.to_string()))?; + + if !ack.accepted { + conn.close(4u32.into(), b"declined"); + return Err(TransportError::Declined(protocol.to_string())); + } + + let link = IrohLink::new(network, peer, conn, ack.max_datagram as usize, send, recv); + Ok(Arc::new(link) as SharedLink) + }) + } +} diff --git a/src/dataplane/transport/mod.rs b/src/dataplane/transport/mod.rs new file mode 100644 index 0000000..7d764ba --- /dev/null +++ b/src/dataplane/transport/mod.rs @@ -0,0 +1,146 @@ +//! The data plane transport boundary. +//! +//! This is the seam that keeps the control protocol and the data plane +//! independent. An [`IpPlugin`] never learns how its packets are moved: it is +//! handed a [`PacketLink`] to a peer and writes datagrams into it. Whether +//! that link runs over iroh today, a raw UDP socket, or something else +//! entirely tomorrow is the transport's business alone. +//! +//! [`IpPlugin`]: crate::dataplane::IpPlugin +//! +//! # Why the transport may use iroh +//! +//! The separation between control and data is **logical**, not a ban on +//! sharing technology. Refusing to use iroh for data would throw away exactly +//! what iroh is good at — hole punching a direct path between two peers behind +//! NAT, with a relay as fallback — and force the data plane to reimplement it. +//! So the default transport is [`iroh_link::IrohTransport`], which gives every +//! plugin that connectivity for free. +//! +//! What the separation does buy is that the control protocol in +//! [`crate::proto`] knows nothing about packets, and this module knows nothing +//! about WireGuard. Either side can be replaced on its own. +//! +//! # Semantics +//! +//! A link is an **unreliable, unordered datagram** channel, because that is +//! what a tunnelled UDP protocol needs: no retransmission, no head-of-line +//! blocking, loss is normal rather than an error. It is authenticated and +//! encrypted by the transport, and scoped to exactly one network, one peer and +//! one plugin protocol. + +pub mod iroh_link; + +use bytes::Bytes; +use iroh::EndpointId; + +use crate::BoxFuture; +use crate::identity::NetworkId; + +/// Why a data plane link failed. +/// +/// None of these ever stop the control plane. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum TransportError { + /// The peer is not reachable for the data plane right now. + #[error("peer is unreachable: {0}")] + Unreachable(String), + /// The peer declined to open a channel for this protocol. + #[error("peer declined a data channel for protocol `{0}`")] + Declined(String), + /// The link is closed. + #[error("data link is closed")] + Closed, + /// A datagram was larger than the link can carry. + #[error("datagram of {size} bytes exceeds the {limit} byte link limit")] + TooLarge { + /// Size that was attempted. + size: usize, + /// Largest datagram this link accepts. + limit: usize, + }, + /// Anything else. + #[error("data transport error: {0}")] + Other(String), +} + +/// An authenticated datagram channel to one peer, for one plugin protocol. +/// +/// Dropping the link closes it. +pub trait PacketLink: Send + Sync + std::fmt::Debug + 'static { + /// The network this link belongs to. + fn network(&self) -> NetworkId; + + /// The authenticated peer on the other end. + fn peer(&self) -> EndpointId; + + /// The largest datagram this link can carry, in bytes. + /// + /// A plugin must size its own packets to fit, because there is no + /// fragmentation here. + fn max_datagram_size(&self) -> usize; + + /// Sends one datagram. + /// + /// Delivery is not guaranteed. Returning `Ok` means the datagram was + /// handed to the transport, nothing more. + fn send(&self, payload: Bytes) -> Result<(), TransportError>; + + /// Receives the next datagram, or `None` once the link is finished. + fn recv(&self) -> BoxFuture<'_, Option>; + + /// Resolves once the link is closed, for whatever reason. + fn closed(&self) -> BoxFuture<'_, ()>; + + /// Whether the link is already closed. + /// + /// Lets the owner notice a dead link and ask for a new one without + /// keeping a task parked on [`PacketLink::closed`]. + fn is_closed(&self) -> bool; + + /// A short description of the path in use, for diagnostics. + /// + /// Reports what the transport actually knows. It must not invent a value. + fn path_description(&self) -> String; +} + +/// A shared handle to a link. +pub type SharedLink = std::sync::Arc; + +/// An inbound link a peer opened towards us. +#[derive(Debug)] +pub struct InboundLink { + /// The network it belongs to. + pub network: NetworkId, + /// The peer that opened it. + pub peer: EndpointId, + /// The plugin protocol it carries. + pub protocol: String, + /// The link itself. + pub link: SharedLink, +} + +/// Opens and accepts data plane links. +/// +/// The agent owns one of these and hands links to plugins; plugins never call +/// it directly. +pub trait PacketTransport: Send + Sync + std::fmt::Debug + 'static { + /// A short name used in diagnostics. + fn name(&self) -> &str; + + /// Lets the agent recover the concrete transport to drive its accept side. + /// + /// Accepting is inherently transport-specific — it starts from whatever + /// the transport's own listener produced — so it is not part of this + /// trait's uniform interface. + fn as_any(&self) -> &dyn std::any::Any; + + /// Opens a link to `peer` in `network` for `protocol`. + fn open<'a>( + &'a self, + network: NetworkId, + peer: EndpointId, + protocol: &'a str, + ) -> BoxFuture<'a, Result>; +} diff --git a/src/dataplane/wireguard/announcement.rs b/src/dataplane/wireguard/announcement.rs index e350b8c..c8a5a27 100644 --- a/src/dataplane/wireguard/announcement.rs +++ b/src/dataplane/wireguard/announcement.rs @@ -4,11 +4,13 @@ //! [`crate::dataplane::PluginCapability`]. The agent core never parses it — //! only this module does, and only after bounding every field. //! -//! An iroh address is an address for iroh. It is **not** reused here: the -//! plugin advertises its own reachability, gathered by itself, for its own -//! listening port. +//! The announcement is deliberately tiny: a participant says **who it is**, +//! not **where it is**. Reachability is the data plane transport's job, and +//! the transport already solves it — see +//! [`crate::dataplane::transport`]. A plugin that also tried to advertise +//! addresses would be reimplementing NAT traversal badly. -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::net::Ipv6Addr; use serde::{Deserialize, Serialize}; @@ -21,9 +23,6 @@ use super::overlay::overlay_address; /// Version of the announcement format. pub const ANNOUNCEMENT_VERSION: u16 = 1; -/// Largest number of advertised endpoints accepted from a peer. -pub const MAX_ENDPOINTS: usize = 8; - /// What one participant advertises for the WireGuard data plane. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct WgAnnouncement { @@ -31,10 +30,6 @@ pub struct WgAnnouncement { pub version: u16, /// The peer's WireGuard public key. Its overlay address is derived from it. pub public_key: [u8; 32], - /// The UDP port the peer's WireGuard interface listens on. - pub listen_port: u16, - /// Reachability the plugin gathered for itself. Advisory, may be empty. - pub endpoints: Vec, /// The overlay address the peer believes it has. /// /// Carried for diagnostics and cross-checking only. `AllowedIPs` are @@ -47,40 +42,16 @@ pub struct WgAnnouncement { pub struct ValidatedAnnouncement { /// The peer's WireGuard public key. pub public_key: WgPublicKey, - /// The peer's listening port. - pub listen_port: u16, - /// Usable endpoints, filtered. - pub endpoints: Vec, /// The overlay address derived locally for this key. Authoritative. pub overlay_address: Ipv6Addr, } -impl ValidatedAnnouncement { - /// The endpoint to configure for this peer, if any is usable. - /// - /// WireGuard takes a single endpoint. The first usable one wins, and - /// WireGuard itself will re-learn the peer's real source address from the - /// first authenticated packet it receives. - pub fn preferred_endpoint(&self) -> Option { - self.endpoints.first().copied() - } -} - impl WgAnnouncement { /// Builds this agent's announcement. - pub fn new( - network: NetworkId, - public_key: &WgPublicKey, - listen_port: u16, - endpoints: Vec, - ) -> Self { - let mut endpoints = endpoints; - endpoints.truncate(MAX_ENDPOINTS); + pub fn new(network: NetworkId, public_key: &WgPublicKey) -> Self { Self { version: ANNOUNCEMENT_VERSION, public_key: *public_key.as_bytes(), - listen_port, - endpoints, overlay_address: overlay_address(network, public_key), } } @@ -128,18 +99,6 @@ impl WgAnnouncement { "peer announced this agent's own WireGuard key".into(), )); } - if self.listen_port == 0 { - return Err(PluginError::Rejected( - "WireGuard listen port must not be zero".into(), - )); - } - if self.endpoints.len() > MAX_ENDPOINTS { - return Err(PluginError::Rejected(format!( - "announcement carries {} endpoints, at most {MAX_ENDPOINTS} are accepted", - self.endpoints.len() - ))); - } - // AllowedIPs are derived, never trusted. A mismatch means the peer is // confused or lying, and either way its own claim is discarded. let derived = overlay_address(network, &public_key); @@ -150,40 +109,13 @@ impl WgAnnouncement { )); } - let endpoints: Vec = self - .endpoints - .into_iter() - .filter(is_usable_endpoint) - .collect(); - Ok(ValidatedAnnouncement { public_key, - listen_port: self.listen_port, - endpoints, overlay_address: derived, }) } } -/// Whether an advertised endpoint is worth trying. -/// -/// Nothing here is trusted; this only discards addresses that cannot be a -/// peer, so the plugin does not waste a WireGuard endpoint slot on them. -fn is_usable_endpoint(endpoint: &SocketAddr) -> bool { - if endpoint.port() == 0 { - return false; - } - match endpoint.ip() { - IpAddr::V4(ip) => { - !ip.is_unspecified() - && !ip.is_multicast() - && !ip.is_broadcast() - && !ip.is_documentation() - } - IpAddr::V6(ip) => !ip.is_unspecified() && !ip.is_multicast(), - } -} - #[cfg(test)] mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] @@ -201,26 +133,31 @@ mod tests { .network_id() } - fn endpoint(text: &str) -> SocketAddr { - text.parse().unwrap() - } - #[test] fn a_well_formed_announcement_round_trips() { let id = network("round-trip"); let peer = WgSecretKey::generate().public(); let local = WgSecretKey::generate().public(); - let announcement = - WgAnnouncement::new(id, &peer, 51820, vec![endpoint("192.0.2.10:51820")]); - let payload = announcement.encode().unwrap(); + let payload = WgAnnouncement::new(id, &peer).encode().unwrap(); let validated = WgAnnouncement::decode_and_validate(&payload, id, &local).unwrap(); assert_eq!(validated.public_key, peer); - assert_eq!(validated.listen_port, 51820); assert_eq!(validated.overlay_address, overlay_address(id, &peer)); - // 192.0.2.0/24 is documentation space and is filtered out. - assert!(validated.endpoints.is_empty()); + } + + #[test] + fn the_announcement_says_who_not_where() { + // Reachability belongs to the transport. Nothing address-like is + // carried here, so there is nothing for a peer to lie about. + let id = network("identity-only"); + let peer = WgSecretKey::generate().public(); + let payload = WgAnnouncement::new(id, &peer).encode().unwrap(); + assert!( + payload.len() < 80, + "the announcement should stay tiny, got {} bytes", + payload.len() + ); } #[test] @@ -231,7 +168,7 @@ mod tests { let local = WgSecretKey::generate().public(); // An attacker claims the victim's overlay address with its own key. - let mut forged = WgAnnouncement::new(id, &attacker, 51820, Vec::new()); + let mut forged = WgAnnouncement::new(id, &attacker); forged.overlay_address = overlay_address(id, &victim); let result = WgAnnouncement::decode_and_validate(&forged.encode().unwrap(), id, &local); @@ -248,9 +185,7 @@ mod tests { let peer = WgSecretKey::generate().public(); let local = WgSecretKey::generate().public(); - let payload = WgAnnouncement::new(there, &peer, 51820, Vec::new()) - .encode() - .unwrap(); + let payload = WgAnnouncement::new(there, &peer).encode().unwrap(); assert!(WgAnnouncement::decode_and_validate(&payload, here, &local).is_err()); } @@ -260,13 +195,12 @@ mod tests { let local = WgSecretKey::generate().public(); let peer = WgSecretKey::generate().public(); - // Not postcard at all. assert!(WgAnnouncement::decode_and_validate(&[0xff; 64], id, &local).is_err()); assert!(WgAnnouncement::decode_and_validate(&[], id, &local).is_err()); let wrong_version = WgAnnouncement { version: ANNOUNCEMENT_VERSION + 1, - ..WgAnnouncement::new(id, &peer, 51820, Vec::new()) + ..WgAnnouncement::new(id, &peer) }; assert!( WgAnnouncement::decode_and_validate(&wrong_version.encode().unwrap(), id, &local) @@ -275,79 +209,26 @@ mod tests { let zero_key = WgAnnouncement { public_key: [0u8; 32], - ..WgAnnouncement::new(id, &peer, 51820, Vec::new()) + ..WgAnnouncement::new(id, &peer) }; assert!( WgAnnouncement::decode_and_validate(&zero_key.encode().unwrap(), id, &local).is_err() ); - - let zero_port = WgAnnouncement::new(id, &peer, 0, Vec::new()); - assert!( - WgAnnouncement::decode_and_validate(&zero_port.encode().unwrap(), id, &local).is_err() - ); - - let too_many = WgAnnouncement { - endpoints: (0..MAX_ENDPOINTS + 1) - .map(|index| endpoint(&format!("10.0.0.1:{}", 1000 + index))) - .collect(), - ..WgAnnouncement::new(id, &peer, 51820, Vec::new()) - }; - assert!( - WgAnnouncement::decode_and_validate(&too_many.encode().unwrap(), id, &local).is_err() - ); } #[test] fn a_peer_cannot_claim_our_own_key() { let id = network("self"); let local = WgSecretKey::generate().public(); - let payload = WgAnnouncement::new(id, &local, 51820, Vec::new()) - .encode() - .unwrap(); + let payload = WgAnnouncement::new(id, &local).encode().unwrap(); assert!(WgAnnouncement::decode_and_validate(&payload, id, &local).is_err()); } - #[test] - fn unusable_endpoints_are_filtered_and_the_rest_kept() { - let id = network("filter"); - let peer = WgSecretKey::generate().public(); - let local = WgSecretKey::generate().public(); - - let announcement = WgAnnouncement::new( - id, - &peer, - 51820, - vec![ - endpoint("0.0.0.0:51820"), - endpoint("224.0.0.1:51820"), - endpoint("10.1.2.3:0"), - endpoint("10.1.2.3:51820"), - endpoint("[2001:db8::1]:51820"), - ], - ); - let validated = - WgAnnouncement::decode_and_validate(&announcement.encode().unwrap(), id, &local) - .unwrap(); - assert_eq!( - validated.endpoints, - vec![endpoint("10.1.2.3:51820"), endpoint("[2001:db8::1]:51820")] - ); - assert_eq!( - validated.preferred_endpoint(), - Some(endpoint("10.1.2.3:51820")) - ); - } - #[test] fn announcements_stay_well_under_the_capability_payload_limit() { let id = network("size"); let peer = WgSecretKey::generate().public(); - let endpoints = (0..MAX_ENDPOINTS) - .map(|index| endpoint(&format!("[2001:db8::{index}]:51820"))) - .collect(); - let payload = WgAnnouncement::new(id, &peer, 51820, endpoints) - .encode() - .unwrap(); + let payload = WgAnnouncement::new(id, &peer).encode().unwrap(); assert!( payload.len() < crate::config::Limits::default().max_capability_data_len, "announcement is {} bytes", diff --git a/src/dataplane/wireguard/backend.rs b/src/dataplane/wireguard/backend.rs deleted file mode 100644 index 5406123..0000000 --- a/src/dataplane/wireguard/backend.rs +++ /dev/null @@ -1,214 +0,0 @@ -//! How a desired configuration reaches the operating system. -//! -//! The plugin computes *what* the interface should look like; a backend makes -//! it so. Splitting them keeps every interesting decision testable without -//! root and without touching the host's network. -//! -//! A backend only ever touches the interface named in the configuration it is -//! given. It never enumerates, adopts or modifies anything else. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use crate::dataplane::PluginError; - -use super::config::{InterfaceConfig, InterfaceState}; - -/// Applies a desired WireGuard configuration. -/// -/// Implementations are synchronous and may block; the plugin calls them from a -/// blocking task, never from the async runtime. -pub trait WireguardBackend: Send + Sync + std::fmt::Debug + 'static { - /// A short name used in diagnostics. - fn name(&self) -> &str; - - /// Reads back the current state of an interface. - /// - /// `Ok(None)` means the interface does not exist, which is different from - /// an error. - fn inspect(&self, interface: &str) -> Result, PluginError>; - - /// Creates or updates the interface so that it matches `desired`. - fn apply(&self, desired: &InterfaceConfig) -> Result<(), PluginError>; - - /// Removes an interface this plugin created. Removing an absent interface - /// succeeds. - fn remove(&self, interface: &str) -> Result<(), PluginError>; -} - -/// What a [`RecordingBackend`] was asked to do. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BackendCall { - /// An interface was inspected. - Inspect(String), - /// An interface was created or updated. - Apply(String), - /// An interface was removed. - Remove(String), -} - -/// An in-memory backend for tests and dry runs. -/// -/// It behaves like a working WireGuard implementation without needing root or -/// touching the host: applied configurations are remembered and can be read -/// back, drift can be injected, and failures can be simulated. -#[derive(Debug, Clone, Default)] -pub struct RecordingBackend { - inner: Arc>, -} - -#[derive(Debug, Default)] -struct Recorded { - interfaces: HashMap, - calls: Vec, - fail_next_apply: Option, -} - -impl RecordingBackend { - /// Creates an empty backend. - pub fn new() -> Self { - Self::default() - } - - fn with(&self, f: impl FnOnce(&mut Recorded) -> T) -> T { - let mut guard = match self.inner.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - f(&mut guard) - } - - /// The state currently configured for an interface, if any. - pub fn state(&self, interface: &str) -> Option { - self.with(|recorded| recorded.interfaces.get(interface).cloned()) - } - - /// Every interface currently configured. - pub fn interfaces(&self) -> Vec { - self.with(|recorded| { - let mut names: Vec = recorded.interfaces.keys().cloned().collect(); - names.sort(); - names - }) - } - - /// Everything the backend was asked to do, in order. - pub fn calls(&self) -> Vec { - self.with(|recorded| recorded.calls.clone()) - } - - /// How many times an interface was applied. - pub fn apply_count(&self, interface: &str) -> usize { - self.with(|recorded| { - recorded - .calls - .iter() - .filter(|call| matches!(call, BackendCall::Apply(name) if name == interface)) - .count() - }) - } - - /// Replaces an interface's state, simulating someone editing it by hand. - pub fn inject_drift(&self, interface: &str, state: InterfaceState) { - self.with(|recorded| { - recorded.interfaces.insert(interface.to_string(), state); - }); - } - - /// Makes the next `apply` fail, simulating a data plane error. - pub fn fail_next_apply(&self, reason: impl Into) { - let reason = reason.into(); - self.with(|recorded| recorded.fail_next_apply = Some(reason)); - } - - /// Forgets the recorded call history, keeping configured interfaces. - pub fn clear_calls(&self) { - self.with(|recorded| recorded.calls.clear()); - } -} - -impl WireguardBackend for RecordingBackend { - fn name(&self) -> &str { - "recording" - } - - fn inspect(&self, interface: &str) -> Result, PluginError> { - self.with(|recorded| { - recorded - .calls - .push(BackendCall::Inspect(interface.to_string())); - Ok(recorded.interfaces.get(interface).cloned()) - }) - } - - fn apply(&self, desired: &InterfaceConfig) -> Result<(), PluginError> { - let state = desired.to_state(); - self.with(|recorded| { - recorded - .calls - .push(BackendCall::Apply(desired.name.clone())); - if let Some(reason) = recorded.fail_next_apply.take() { - return Err(PluginError::Unavailable(reason)); - } - recorded.interfaces.insert(desired.name.clone(), state); - Ok(()) - }) - } - - fn remove(&self, interface: &str) -> Result<(), PluginError> { - self.with(|recorded| { - recorded - .calls - .push(BackendCall::Remove(interface.to_string())); - recorded.interfaces.remove(interface); - }); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - - use super::*; - use crate::dataplane::wireguard::config::{InterfaceParams, build_interface}; - use crate::dataplane::wireguard::keys::WgSecretKey; - use crate::identity::{NetworkKeys, NetworkName, NetworkSecret}; - - #[test] - fn the_recording_backend_behaves_like_a_working_one() { - let network = NetworkKeys::derive( - &NetworkName::new("backend").unwrap(), - &NetworkSecret::from_bytes(vec![2u8; 32]).unwrap(), - ) - .network_id(); - let backend = RecordingBackend::new(); - let config = build_interface( - InterfaceParams { - network, - name: "tsun0".into(), - private_key: WgSecretKey::generate(), - listen_port: 51820, - mtu: None, - keepalive: None, - }, - [WgSecretKey::generate().public()], - |_| None, - ); - - assert_eq!(backend.inspect("tsun0").unwrap(), None); - backend.apply(&config).unwrap(); - assert_eq!(backend.inspect("tsun0").unwrap(), Some(config.to_state())); - assert_eq!(backend.interfaces(), vec!["tsun0".to_string()]); - - backend.fail_next_apply("no permission"); - assert!(backend.apply(&config).is_err()); - backend.apply(&config).unwrap(); - - backend.remove("tsun0").unwrap(); - assert_eq!(backend.inspect("tsun0").unwrap(), None); - // Removing something absent is not an error. - backend.remove("tsun0").unwrap(); - assert_eq!(backend.apply_count("tsun0"), 3); - } -} diff --git a/src/dataplane/wireguard/config.rs b/src/dataplane/wireguard/config.rs index 57ab6c8..de7eab4 100644 --- a/src/dataplane/wireguard/config.rs +++ b/src/dataplane/wireguard/config.rs @@ -9,19 +9,12 @@ //! module re-serialises itself, so a hostile announcement cannot inject a //! configuration directive or a command argument. -use std::fmt::Write as _; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; - -use sha2::{Digest, Sha256}; -use zeroize::Zeroizing; +use std::net::{IpAddr, Ipv6Addr}; use crate::dataplane::PluginError; use crate::identity::NetworkId; -use super::keys::{WgPublicKey, WgSecretKey}; -use super::overlay::{ - OVERLAY_HOST_PREFIX_LEN, OVERLAY_PREFIX_LEN, overlay_address, overlay_prefix, -}; +use super::overlay::OVERLAY_HOST_PREFIX_LEN; /// Longest interface name Linux accepts, excluding the terminating NUL. pub const MAX_INTERFACE_NAME_LEN: usize = 15; @@ -68,150 +61,6 @@ impl std::fmt::Display for Cidr { } } -/// One remote participant, as this agent will configure it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PeerConfig { - /// The peer's WireGuard public key. - pub public_key: WgPublicKey, - /// Where to send the first packet, when the peer advertised somewhere. - pub endpoint: Option, - /// Prefixes accepted from and routed to this peer. - /// - /// Always derived locally from the peer's key. Never taken from what the - /// peer claims. - pub allowed_ips: Vec, - /// Keepalive interval, needed to hold a NAT mapping open. - pub persistent_keepalive: Option, -} - -/// The complete local configuration for one network's overlay interface. -#[derive(Debug, Clone)] -pub struct InterfaceConfig { - /// Interface name this plugin owns. - pub name: String, - /// This agent's private key for this network. - pub private_key: WgSecretKey, - /// UDP port the interface listens on. - pub listen_port: u16, - /// Addresses assigned to the interface. - pub addresses: Vec, - /// Interface MTU, when one is configured. - pub mtu: Option, - /// Remote participants. - pub peers: Vec, -} - -/// Observable state of a configured interface, without any private key. -/// -/// This is what desired and actual are compared on, so reconciliation never -/// needs to move a private key around. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct InterfaceState { - /// Interface name. - pub name: String, - /// Public key currently configured on the interface. - pub public_key: WgPublicKey, - /// Port currently listened on. - pub listen_port: u16, - /// Addresses currently assigned. - pub addresses: Vec, - /// Peers currently configured, sorted by public key. - pub peers: Vec, -} - -/// Observable state of one configured peer. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PeerState { - /// The peer's public key. - pub public_key: WgPublicKey, - /// Endpoint currently configured. - pub endpoint: Option, - /// Allowed prefixes currently configured, sorted. - pub allowed_ips: Vec, - /// Keepalive currently configured. - pub persistent_keepalive: Option, -} - -impl PeerState { - /// Puts the state in its canonical, comparable form. - pub fn normalised(mut self) -> Self { - self.allowed_ips.sort(); - self.allowed_ips.dedup(); - // WireGuard reports a disabled keepalive as zero. - if self.persistent_keepalive == Some(0) { - self.persistent_keepalive = None; - } - self - } -} - -impl InterfaceState { - /// Puts the state in its canonical, comparable form. - pub fn normalised(mut self) -> Self { - self.addresses.sort(); - self.addresses.dedup(); - self.peers = self.peers.into_iter().map(PeerState::normalised).collect(); - self.peers.sort_by_key(|peer| peer.public_key); - self - } -} - -impl InterfaceConfig { - /// The state this configuration is expected to produce. - pub fn to_state(&self) -> InterfaceState { - InterfaceState { - name: self.name.clone(), - public_key: self.private_key.public(), - listen_port: self.listen_port, - addresses: self.addresses.clone(), - peers: self - .peers - .iter() - .map(|peer| PeerState { - public_key: peer.public_key, - endpoint: peer.endpoint, - allowed_ips: peer.allowed_ips.clone(), - persistent_keepalive: peer.persistent_keepalive, - }) - .collect(), - } - .normalised() - } - - /// Renders the configuration in the format `wg setconf` and `wg syncconf` - /// read. - /// - /// Only WireGuard's own directives appear here. Addresses and MTU are not - /// part of this format — they belong to the network interface and are - /// applied separately. - /// - /// The result contains the private key and is zeroized on drop. - pub fn render(&self) -> Zeroizing { - let mut out = String::with_capacity(256 + self.peers.len() * 192); - out.push_str("[Interface]\n"); - let _ = writeln!(out, "PrivateKey = {}", self.private_key.encode().as_str()); - let _ = writeln!(out, "ListenPort = {}", self.listen_port); - - let mut peers = self.peers.clone(); - peers.sort_by_key(|peer| peer.public_key); - for peer in &peers { - out.push_str("\n[Peer]\n"); - let _ = writeln!(out, "PublicKey = {}", peer.public_key.encode()); - let mut allowed = peer.allowed_ips.clone(); - allowed.sort(); - let rendered: Vec = allowed.iter().map(Cidr::to_string).collect(); - let _ = writeln!(out, "AllowedIPs = {}", rendered.join(", ")); - if let Some(endpoint) = peer.endpoint { - let _ = writeln!(out, "Endpoint = {endpoint}"); - } - if let Some(keepalive) = peer.persistent_keepalive { - let _ = writeln!(out, "PersistentKeepalive = {keepalive}"); - } - } - Zeroizing::new(out) - } -} - /// Derives this plugin's interface name for a network. /// /// The name is stable across restarts and short enough for the platform. Two @@ -244,132 +93,6 @@ pub fn interface_name(prefix: &str, network: NetworkId) -> Result Self { - Self::Derived { - base: 51820, - span: 64, - } - } -} - -impl PortPolicy { - /// The port to listen on for `network`. - pub fn port_for(&self, network: NetworkId) -> Result { - match *self { - PortPolicy::Fixed(port) => { - if port == 0 { - return Err(PluginError::Other( - "a fixed WireGuard port must not be zero".into(), - )); - } - Ok(port) - } - PortPolicy::Derived { base, span } => { - if base == 0 || span == 0 { - return Err(PluginError::Other( - "a derived WireGuard port range must not be empty or start at zero".into(), - )); - } - let room = u16::MAX - base; - if span - 1 > room { - return Err(PluginError::Other( - "the derived WireGuard port range runs past port 65535".into(), - )); - } - let hash = Sha256::digest(network.as_bytes()); - let offset = u16::from_be_bytes([hash[0], hash[1]]) % span; - Ok(base + offset) - } - } - } -} - -/// Everything about the local side of one network's interface. -#[derive(Debug, Clone)] -pub struct InterfaceParams { - /// The network the interface serves. - pub network: NetworkId, - /// Interface name, derived by [`interface_name`]. - pub name: String, - /// This agent's private key for this network. - pub private_key: WgSecretKey, - /// Port to listen on. - pub listen_port: u16, - /// Interface MTU. - pub mtu: Option, - /// Keepalive applied to every peer. - pub keepalive: Option, -} - -/// Builds this agent's interface configuration for one network. -/// -/// `peers` is the set of participants the control plane agreed on; `endpoints` -/// supplies whatever reachability each of them advertised. -pub fn build_interface( - params: InterfaceParams, - peers: impl IntoIterator, - endpoints: impl Fn(&WgPublicKey) -> Option, -) -> InterfaceConfig { - let InterfaceParams { - network, - name, - private_key, - listen_port, - mtu, - keepalive, - } = params; - let local = overlay_address(network, &private_key.public()); - - let mut peer_configs: Vec = peers - .into_iter() - .filter(|key| !key.is_zero() && *key != private_key.public()) - .map(|key| PeerConfig { - endpoint: endpoints(&key), - // Derived locally. This is the whole reason a hostile member - // cannot route another member's traffic to itself. - allowed_ips: vec![Cidr::host(overlay_address(network, &key))], - public_key: key, - persistent_keepalive: keepalive, - }) - .collect(); - peer_configs.sort_by_key(|peer| peer.public_key); - peer_configs.dedup_by(|a, b| a.public_key == b.public_key); - - InterfaceConfig { - name, - private_key, - listen_port, - addresses: vec![ - Cidr::host(local), - // The shared /64 gives the interface a route for the overlay. - Cidr { - addr: IpAddr::V6(overlay_prefix(network)), - prefix_len: OVERLAY_PREFIX_LEN, - }, - ], - mtu, - peers: peer_configs, - } -} - #[cfg(test)] mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] @@ -385,132 +108,6 @@ mod tests { .network_id() } - #[test] - fn a_full_mesh_of_n_members_yields_n_minus_one_peers() { - let id = network("mesh"); - let me = WgSecretKey::generate(); - let others: Vec = (0..3).map(|_| WgSecretKey::generate().public()).collect(); - - let config = build_interface( - InterfaceParams { - network: id, - name: "tsun0".into(), - private_key: me.clone(), - listen_port: 51820, - mtu: None, - keepalive: Some(25), - }, - others.clone().into_iter().chain([me.public()]), - |_| None, - ); - - assert_eq!(config.peers.len(), 3, "our own key is never a peer"); - for peer in &config.peers { - assert_eq!(peer.allowed_ips.len(), 1); - assert_eq!( - peer.allowed_ips[0], - Cidr::host(overlay_address(id, &peer.public_key)) - ); - assert_eq!(peer.persistent_keepalive, Some(25)); - } - assert!( - config - .addresses - .contains(&Cidr::host(overlay_address(id, &me.public()))) - ); - } - - #[test] - fn duplicate_and_zero_peer_keys_are_dropped() { - let id = network("dupes"); - let me = WgSecretKey::generate(); - let other = WgSecretKey::generate().public(); - - let config = build_interface( - InterfaceParams { - network: id, - name: "tsun0".into(), - private_key: me, - listen_port: 51820, - mtu: None, - keepalive: None, - }, - [other, other, WgPublicKey::from_bytes([0u8; 32])], - |_| None, - ); - assert_eq!(config.peers.len(), 1); - assert_eq!(config.peers[0].public_key, other); - } - - #[test] - fn the_rendered_config_is_the_wg_setconf_format() { - let id = network("render"); - let me = WgSecretKey::generate(); - let peer = WgSecretKey::generate().public(); - let config = build_interface( - InterfaceParams { - network: id, - name: "tsun0".into(), - private_key: me.clone(), - listen_port: 51820, - mtu: Some(1380), - keepalive: Some(25), - }, - [peer], - |_| Some("10.0.0.7:51820".parse().unwrap()), - ); - - let rendered = config.render(); - let text = rendered.as_str(); - assert!(text.starts_with("[Interface]\n")); - assert!(text.contains(&format!("PrivateKey = {}", me.encode().as_str()))); - assert!(text.contains("ListenPort = 51820")); - assert!(text.contains(&format!("PublicKey = {}", peer.encode()))); - assert!(text.contains("Endpoint = 10.0.0.7:51820")); - assert!(text.contains("PersistentKeepalive = 25")); - assert!(text.contains(&format!( - "AllowedIPs = {}", - Cidr::host(overlay_address(id, &peer)) - ))); - // Address and MTU belong to the interface, not to wg's own format. - assert!(!text.contains("Address")); - assert!(!text.contains("MTU")); - } - - #[test] - fn rendering_is_deterministic_regardless_of_peer_order() { - let id = network("stable"); - let me = WgSecretKey::generate(); - let keys: Vec = (0..5).map(|_| WgSecretKey::generate().public()).collect(); - - let forward = build_interface( - InterfaceParams { - network: id, - name: "tsun0".into(), - private_key: me.clone(), - listen_port: 51820, - mtu: None, - keepalive: None, - }, - keys.clone(), - |_| None, - ); - let reversed = build_interface( - InterfaceParams { - network: id, - name: "tsun0".into(), - private_key: me, - listen_port: 51820, - mtu: None, - keepalive: None, - }, - keys.into_iter().rev().collect::>(), - |_| None, - ); - assert_eq!(forward.render().as_str(), reversed.render().as_str()); - assert_eq!(forward.to_state(), reversed.to_state()); - } - #[test] fn interface_names_fit_the_platform_limit_and_are_stable() { let id = network("naming"); @@ -532,56 +129,12 @@ mod tests { } #[test] - fn derived_ports_are_stable_and_inside_the_range() { - let policy = PortPolicy::default(); - let id = network("ports"); - let port = policy.port_for(id).unwrap(); - assert_eq!(port, policy.port_for(id).unwrap()); - assert!( - (51820..51884).contains(&port), - "port {port} outside the range" - ); - - assert_eq!(PortPolicy::Fixed(1234).port_for(id).unwrap(), 1234); - assert!(PortPolicy::Fixed(0).port_for(id).is_err()); - assert!( - PortPolicy::Derived { - base: 65500, - span: 1000 - } - .port_for(id) - .is_err() - ); - } - - #[test] - fn state_comparison_ignores_ordering_and_a_zero_keepalive() { - let peer_a = WgSecretKey::generate().public(); - let peer_b = WgSecretKey::generate().public(); - let make = |order: [WgPublicKey; 2], keepalive: Option| { - InterfaceState { - name: "tsun0".into(), - public_key: peer_a, - listen_port: 51820, - addresses: vec![ - Cidr::host("fd00::2".parse().unwrap()), - Cidr::host("fd00::1".parse().unwrap()), - ], - peers: order - .into_iter() - .map(|public_key| PeerState { - public_key, - endpoint: None, - allowed_ips: Vec::new(), - persistent_keepalive: keepalive, - }) - .collect(), - } - .normalised() - }; + fn a_cidr_rejects_an_impossible_prefix_length() { + assert!(Cidr::new("10.0.0.1".parse().unwrap(), 33).is_err()); + assert!(Cidr::new("fd00::1".parse().unwrap(), 129).is_err()); assert_eq!( - make([peer_a, peer_b], None), - make([peer_b, peer_a], Some(0)) + Cidr::host("fd00::1".parse().unwrap()).to_string(), + "fd00::1/128" ); } } diff --git a/src/dataplane/wireguard/device.rs b/src/dataplane/wireguard/device.rs new file mode 100644 index 0000000..00f8678 --- /dev/null +++ b/src/dataplane/wireguard/device.rs @@ -0,0 +1,554 @@ +//! Userspace WireGuard. +//! +//! The protocol itself is [`boringtun::noise::Tunn`], which is pure state +//! machine: no sockets, no TUN, no kernel module. That is what lets this work +//! the same way on any platform and be tested end to end without privileges. +//! +//! ```text +//! TunDevice (IP packets) PacketLink per peer +//! | | +//! v v +//! destination address -> peer --Tunn.encapsulate--> ciphertext +//! source address checked <--Tunn.decapsulate-- ciphertext +//! ``` +//! +//! # Address ownership is enforced here +//! +//! Kernel WireGuard enforces `AllowedIPs`; in userspace we must do it +//! ourselves, and we do: +//! +//! * outbound, a packet is routed to the peer that **owns** its destination +//! address, where ownership is the derivation in [`super::overlay`]; +//! * inbound, a decrypted packet is dropped unless its **source** is exactly +//! the address derived for the peer whose tunnel decrypted it. +//! +//! So a participant cannot receive traffic addressed to someone else, and +//! cannot forge traffic that appears to come from someone else, no matter +//! what it announced. + +use std::collections::HashMap; +use std::net::Ipv6Addr; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +use boringtun::noise::{Tunn, TunnResult}; +use bytes::Bytes; +use iroh::EndpointId; +use tokio::task::JoinHandle; + +use crate::dataplane::PluginError; +use crate::dataplane::transport::{SharedLink, TransportError}; +use crate::identity::NetworkId; + +use super::keys::{WgPublicKey, WgSecretKey}; +use super::overlay::overlay_address; +use super::packet::IpHeader; +use super::tun::TunDevice; + +/// How often WireGuard's own timers are driven. +/// +/// boringtun expects this at least every few hundred milliseconds; it is what +/// drives handshakes, rekeying and keepalives. +const TIMER_INTERVAL: Duration = Duration::from_millis(250); + +/// Scratch space for one encapsulate or decapsulate call. +const SCRATCH: usize = 4096; + +/// Counters for one peer's tunnel. +#[derive(Debug, Default)] +struct PeerCounters { + tx_packets: AtomicU64, + tx_bytes: AtomicU64, + rx_packets: AtomicU64, + rx_bytes: AtomicU64, + dropped_wrong_source: AtomicU64, + dropped_oversize: AtomicU64, + protocol_errors: AtomicU64, +} + +/// A snapshot of one peer's tunnel counters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PeerStats { + /// Plaintext packets encrypted and sent to this peer. + pub tx_packets: u64, + /// Plaintext bytes encrypted and sent to this peer. + pub tx_bytes: u64, + /// Plaintext packets decrypted from this peer and given to the OS. + pub rx_packets: u64, + /// Plaintext bytes decrypted from this peer and given to the OS. + pub rx_bytes: u64, + /// Packets dropped because their source was not this peer's address. + /// + /// A non-zero value means a peer tried to use an address it does not own. + pub dropped_wrong_source: u64, + /// Packets dropped because they did not fit in one link datagram. + pub dropped_oversize: u64, + /// WireGuard protocol errors, including packets that failed to decrypt. + pub protocol_errors: u64, +} + +/// Whether a peer's tunnel has completed a handshake. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PeerHealth { + /// Time since the last successful WireGuard handshake. + /// + /// `None` means no handshake has completed yet, so the tunnel is not + /// carrying traffic. This is reported as it is, never guessed. + pub since_handshake: Option, +} + +impl PeerHealth { + /// Whether the tunnel has ever completed a handshake. + pub fn is_up(&self) -> bool { + self.since_handshake.is_some() + } +} + +struct Peer { + endpoint_id: EndpointId, + public_key: WgPublicKey, + overlay: Ipv6Addr, + tunn: Mutex, + link: SharedLink, + counters: Arc, + task: Mutex>>, +} + +impl std::fmt::Debug for Peer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Peer") + .field("peer", &self.endpoint_id.fmt_short().to_string()) + .field("public_key", &self.public_key) + .field("overlay", &self.overlay) + .finish() + } +} + +impl Drop for Peer { + fn drop(&mut self) { + if let Ok(mut guard) = self.task.lock() + && let Some(task) = guard.take() + { + task.abort(); + } + } +} + +impl Peer { + fn stats(&self) -> PeerStats { + PeerStats { + tx_packets: self.counters.tx_packets.load(Ordering::Relaxed), + tx_bytes: self.counters.tx_bytes.load(Ordering::Relaxed), + rx_packets: self.counters.rx_packets.load(Ordering::Relaxed), + rx_bytes: self.counters.rx_bytes.load(Ordering::Relaxed), + dropped_wrong_source: self.counters.dropped_wrong_source.load(Ordering::Relaxed), + dropped_oversize: self.counters.dropped_oversize.load(Ordering::Relaxed), + protocol_errors: self.counters.protocol_errors.load(Ordering::Relaxed), + } + } + + fn health(&self) -> PeerHealth { + let guard = match self.tunn.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + PeerHealth { + since_handshake: guard.time_since_last_handshake(), + } + } +} + +/// What a peer's tunnel looks like from outside. +#[derive(Debug, Clone)] +pub struct PeerSummary { + /// The peer's control plane identity. + pub endpoint_id: EndpointId, + /// The peer's WireGuard public key. + pub public_key: WgPublicKey, + /// The overlay address this agent derived for it. + pub overlay_address: Ipv6Addr, + /// Whether the tunnel has handshaken. + pub health: PeerHealth, + /// Traffic counters. + pub stats: PeerStats, + /// What the transport reports about the path carrying this tunnel. + pub path: String, + /// Largest datagram the link accepts. + pub max_datagram: usize, +} + +struct Inner { + network: NetworkId, + private_key: WgSecretKey, + tun: Arc, + peers: RwLock>>, + routes: RwLock>, + next_index: AtomicU32, + unroutable: AtomicU64, +} + +impl std::fmt::Debug for Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Inner") + .field("network", &self.network.fmt_short()) + .field("tun", &self.tun.name()) + .finish() + } +} + +/// A userspace WireGuard interface for one network. +#[derive(Debug)] +pub struct WireguardDevice { + inner: Arc, + tasks: Vec>, +} + +impl WireguardDevice { + /// Starts a device on top of `tun`. + pub fn start(network: NetworkId, private_key: WgSecretKey, tun: Arc) -> Self { + let inner = Arc::new(Inner { + network, + private_key, + tun, + peers: RwLock::new(HashMap::new()), + routes: RwLock::new(HashMap::new()), + next_index: AtomicU32::new(1), + unroutable: AtomicU64::new(0), + }); + + let reader = tokio::spawn(read_from_os(Arc::clone(&inner))); + let timers = tokio::spawn(drive_timers(Arc::clone(&inner))); + + Self { + inner, + tasks: vec![reader, timers], + } + } + + /// The interface name in use. + pub fn interface(&self) -> &str { + self.inner.tun.name() + } + + /// The interface MTU. + pub fn mtu(&self) -> u32 { + self.inner.tun.mtu() + } + + /// Adds or replaces a peer and starts its tunnel. + pub fn add_peer( + &self, + endpoint_id: EndpointId, + public_key: WgPublicKey, + link: SharedLink, + keepalive: Option, + ) -> Result<(), PluginError> { + if public_key == self.inner.private_key.public() { + return Err(PluginError::Rejected( + "refusing to add ourselves as a WireGuard peer".into(), + )); + } + + let index = self.inner.next_index.fetch_add(1, Ordering::Relaxed); + let tunn = Tunn::new( + self.inner.private_key.to_static_secret(), + public_key.into_x25519(), + None, + keepalive, + index, + None, + ); + + let overlay = overlay_address(self.inner.network, &public_key); + let peer = Arc::new(Peer { + endpoint_id, + public_key, + overlay, + tunn: Mutex::new(tunn), + link, + counters: Arc::new(PeerCounters::default()), + task: Mutex::new(None), + }); + + let task = tokio::spawn(read_from_link(Arc::clone(&self.inner), Arc::clone(&peer))); + if let Ok(mut guard) = peer.task.lock() { + *guard = Some(task); + } + + write_lock(&self.inner.peers).insert(public_key, Arc::clone(&peer)); + write_lock(&self.inner.routes).insert(overlay, public_key); + + // Start the handshake now instead of waiting for the next timer tick, + // so the tunnel is usable as soon as the link exists. + kick_handshake(&peer); + Ok(()) + } + + /// Removes a peer and stops its tunnel. + pub fn remove_peer(&self, public_key: &WgPublicKey) { + if let Some(peer) = write_lock(&self.inner.peers).remove(public_key) { + write_lock(&self.inner.routes).remove(&peer.overlay); + } + } + + /// Removes every peer whose key is not in `keep`. + pub fn retain_peers(&self, keep: &[WgPublicKey]) { + let stale: Vec = read_lock(&self.inner.peers) + .keys() + .filter(|key| !keep.contains(key)) + .copied() + .collect(); + for key in stale { + self.remove_peer(&key); + } + } + + /// Whether a peer's tunnel exists. + pub fn has_peer(&self, public_key: &WgPublicKey) -> bool { + read_lock(&self.inner.peers).contains_key(public_key) + } + + /// A snapshot of every peer. + pub fn peers(&self) -> Vec { + let mut peers: Vec = read_lock(&self.inner.peers) + .values() + .map(|peer| PeerSummary { + endpoint_id: peer.endpoint_id, + public_key: peer.public_key, + overlay_address: peer.overlay, + health: peer.health(), + stats: peer.stats(), + path: peer.link.path_description(), + max_datagram: peer.link.max_datagram_size(), + }) + .collect(); + peers.sort_by_key(|peer| peer.public_key); + peers + } + + /// Packets the operating system sent that no peer owns the address for. + pub fn unroutable_packets(&self) -> u64 { + self.inner.unroutable.load(Ordering::Relaxed) + } +} + +impl Drop for WireguardDevice { + fn drop(&mut self) { + for task in &self.tasks { + task.abort(); + } + } +} + +fn read_lock(lock: &RwLock) -> std::sync::RwLockReadGuard<'_, T> { + match lock.read() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +fn write_lock(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, T> { + match lock.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +/// Asks boringtun for a handshake initiation and sends it. +/// +/// Encapsulating an empty packet is how the protocol state machine is told +/// "there is something to say"; with no session yet it answers with the +/// handshake initiation. +fn kick_handshake(peer: &Peer) { + let mut scratch = vec![0u8; SCRATCH]; + let len = { + let mut tunn = match peer.tunn.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + match tunn.encapsulate(&[], &mut scratch) { + TunnResult::WriteToNetwork(out) => Some(out.len()), + _ => None, + } + }; + if let Some(len) = len { + send_to_peer(peer, &scratch[..len]); + } +} + +/// Sends whatever boringtun produced, without holding the tunnel lock. +fn send_to_peer(peer: &Peer, payload: &[u8]) { + match peer.link.send(Bytes::copy_from_slice(payload)) { + Ok(()) => {} + Err(TransportError::TooLarge { .. }) => { + peer.counters + .dropped_oversize + .fetch_add(1, Ordering::Relaxed); + } + Err(TransportError::Closed) => {} + Err(err) => { + tracing::trace!(%err, "dropping a WireGuard packet the link refused"); + } + } +} + +/// Operating system -> peer. +async fn read_from_os(inner: Arc) { + loop { + let Some(packet) = inner.tun.recv().await else { + return; + }; + + // Route by destination: only the peer that owns that overlay address + // may receive it. + let Some(destination) = IpHeader::parse(&packet).and_then(|header| header.v6_destination()) + else { + inner.unroutable.fetch_add(1, Ordering::Relaxed); + continue; + }; + let target = read_lock(&inner.routes).get(&destination).copied(); + let Some(target) = target else { + inner.unroutable.fetch_add(1, Ordering::Relaxed); + continue; + }; + let peer = read_lock(&inner.peers).get(&target).cloned(); + let Some(peer) = peer else { + inner.unroutable.fetch_add(1, Ordering::Relaxed); + continue; + }; + + let mut scratch = vec![0u8; SCRATCH]; + let outcome = { + let mut tunn = match peer.tunn.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + match tunn.encapsulate(&packet, &mut scratch) { + TunnResult::WriteToNetwork(out) => Some(out.len()), + TunnResult::Done => None, + TunnResult::Err(err) => { + tracing::trace!(?err, "wireguard encapsulation failed"); + peer.counters + .protocol_errors + .fetch_add(1, Ordering::Relaxed); + None + } + _ => None, + } + }; + + if let Some(len) = outcome { + send_to_peer(&peer, &scratch[..len]); + peer.counters.tx_packets.fetch_add(1, Ordering::Relaxed); + peer.counters + .tx_bytes + .fetch_add(packet.len() as u64, Ordering::Relaxed); + } + } +} + +/// Peer -> operating system. +async fn read_from_link(inner: Arc, peer: Arc) { + loop { + let Some(datagram) = peer.link.recv().await else { + return; + }; + + let mut scratch = vec![0u8; SCRATCH]; + // boringtun may need several passes: a handshake reply first, then + // any packets that were queued while the session was coming up. + let mut input: Option<&[u8]> = Some(&datagram); + loop { + let outcome = { + let mut tunn = match peer.tunn.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + match tunn.decapsulate(None, input.unwrap_or(&[]), &mut scratch) { + TunnResult::WriteToNetwork(out) => Outcome::ToNetwork(out.len()), + TunnResult::WriteToTunnelV6(out, source) => { + Outcome::ToTunnel(out.len(), Some(source)) + } + TunnResult::WriteToTunnelV4(out, _) => Outcome::ToTunnel(out.len(), None), + TunnResult::Done => Outcome::Done, + TunnResult::Err(err) => { + tracing::trace!(?err, "wireguard decapsulation failed"); + Outcome::Failed + } + } + }; + + match outcome { + Outcome::ToNetwork(len) => { + send_to_peer(&peer, &scratch[..len]); + // Keep draining with an empty datagram, as boringtun asks. + input = None; + continue; + } + Outcome::ToTunnel(len, source) => { + let payload = Bytes::copy_from_slice(&scratch[..len]); + // Enforce address ownership: a peer may only send from the + // address derived for its own key. + if source != Some(peer.overlay) { + peer.counters + .dropped_wrong_source + .fetch_add(1, Ordering::Relaxed); + break; + } + if inner.tun.send(payload).await.is_ok() { + peer.counters.rx_packets.fetch_add(1, Ordering::Relaxed); + peer.counters + .rx_bytes + .fetch_add(len as u64, Ordering::Relaxed); + } + break; + } + Outcome::Failed => { + peer.counters + .protocol_errors + .fetch_add(1, Ordering::Relaxed); + break; + } + Outcome::Done => break, + } + } + } +} + +enum Outcome { + ToNetwork(usize), + ToTunnel(usize, Option), + Done, + Failed, +} + +/// Drives WireGuard's handshake, rekey and keepalive timers. +async fn drive_timers(inner: Arc) { + let mut ticker = tokio::time::interval(TIMER_INTERVAL); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + ticker.tick().await; + let peers: Vec> = read_lock(&inner.peers).values().cloned().collect(); + for peer in peers { + let mut scratch = vec![0u8; SCRATCH]; + let len = { + let mut tunn = match peer.tunn.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + match tunn.update_timers(&mut scratch) { + TunnResult::WriteToNetwork(out) => Some(out.len()), + TunnResult::Err(err) => { + tracing::trace!(?err, "wireguard timer produced an error"); + None + } + _ => None, + } + }; + if let Some(len) = len { + send_to_peer(&peer, &scratch[..len]); + } + } + } +} diff --git a/src/dataplane/wireguard/keys.rs b/src/dataplane/wireguard/keys.rs index a6f0ab6..7f7b1de 100644 --- a/src/dataplane/wireguard/keys.rs +++ b/src/dataplane/wireguard/keys.rs @@ -7,6 +7,7 @@ //! Keys are X25519, encoded the way WireGuard encodes them: standard base64 //! with padding, 44 characters. +use boringtun::x25519; use data_encoding::BASE64; use zeroize::{Zeroize, Zeroizing}; @@ -46,6 +47,11 @@ impl WgPublicKey { &self.0 } + /// The key in the form the WireGuard implementation expects. + pub(crate) fn into_x25519(self) -> x25519::PublicKey { + x25519::PublicKey::from(self.0) + } + /// Whether this is the all-zero key, which is never a valid peer. pub fn is_zero(&self) -> bool { self.0 == [0u8; KEY_LEN] @@ -121,9 +127,12 @@ impl WgSecretKey { /// The matching public key. pub fn public(&self) -> WgPublicKey { - let secret = x25519_dalek::StaticSecret::from(*self.0); - let public = x25519_dalek::PublicKey::from(&secret); - WgPublicKey(public.to_bytes()) + WgPublicKey(x25519::PublicKey::from(&self.to_static_secret()).to_bytes()) + } + + /// The key in the form the WireGuard implementation expects. + pub(crate) fn to_static_secret(&self) -> x25519::StaticSecret { + x25519::StaticSecret::from(*self.0) } /// The base64 form, for the WireGuard configuration. Zeroized on drop. diff --git a/src/dataplane/wireguard/mod.rs b/src/dataplane/wireguard/mod.rs index 9b7529d..ba2f54f 100644 --- a/src/dataplane/wireguard/mod.rs +++ b/src/dataplane/wireguard/mod.rs @@ -6,15 +6,18 @@ //! //! The two planes stay separate: //! -//! * **No user IP traffic goes through iroh.** iroh carries this plugin's -//! announcements and nothing else; the packets themselves travel over -//! WireGuard's own UDP sockets. -//! * **An iroh address is not a WireGuard address.** The plugin gathers its -//! own reachability and advertises that. +//! * **The plugin knows nothing about reachability.** It is handed a +//! [`PacketLink`](crate::dataplane::transport::PacketLink) per peer and +//! bridges the kernel WireGuard device onto it. Hole punching and relaying +//! belong to the transport. +//! * **The announcement says who, not where.** It carries a public key, so +//! there is no address for a peer to lie about. //! * **The core never parses these announcements.** It moves a bounded opaque //! blob; only [`announcement`] interprets it. //! * **Keys are separate.** The plugin has its own key per network, in its own //! store, unrelated to the iroh device key and to the network secret. +//! * **WireGuard's own crypto is untouched.** The bridge is a pipe; the +//! handshake and encryption run end to end between the two kernels. //! //! # How a mesh forms //! @@ -24,34 +27,41 @@ //! peer's `AllowedIPs` itself instead of believing what the peer claims — a //! member cannot route another member's traffic to itself. //! -//! Each agent then builds its own local configuration with one peer entry per -//! other participant ([`config`]) and hands it to a [`backend`]. The -//! [`backend::RecordingBackend`] applies it in memory, which is what the test -//! suite uses; [`wgtool::WgToolBackend`] drives the real `wg` and `ip` tools -//! and needs Linux with `CAP_NET_ADMIN`. +//! WireGuard itself is [`boringtun`]'s protocol state machine, running in this +//! process: no kernel module, no `wg` tool, the same code on every platform. +//! [`device::WireguardDevice`] drives one tunnel per peer and routes packets +//! between them and a [`tun::TunDevice`]. +//! +//! The only part that needs privileges is the packet interface. With +//! [`tun::MemoryTunFactory`] the whole data plane — handshake, encryption, +//! routing, address ownership — runs and is tested with no privileges at all; +//! `SystemTunFactory` swaps in a real interface when you want traffic to +//! reach the operating system. //! //! See `docs/wireguard.md` for the full picture. pub mod announcement; -pub mod backend; pub mod config; +pub mod device; pub mod keys; pub mod overlay; +pub mod packet; pub mod plugin; pub mod store; -pub mod wgtool; +pub mod tun; pub use announcement::{ValidatedAnnouncement, WgAnnouncement}; -pub use backend::{BackendCall, RecordingBackend, WireguardBackend}; -pub use config::{ - Cidr, InterfaceConfig, InterfaceParams, InterfaceState, PeerConfig, PeerState, PortPolicy, - build_interface, interface_name, -}; +pub use config::{Cidr, DEFAULT_INTERFACE_PREFIX, MAX_INTERFACE_NAME_LEN, interface_name}; +pub use device::{PeerHealth, PeerStats, PeerSummary, WireguardDevice}; pub use keys::{WgPublicKey, WgSecretKey}; -pub use overlay::{overlay_address, overlay_prefix}; +pub use overlay::{OVERLAY_PREFIX_LEN, overlay_address, overlay_prefix}; +pub use packet::IpHeader; pub use plugin::{ - AdvertisePolicy, NetworkOverview, PeerOverview, WIREGUARD_PROTOCOL, WireguardConfig, + DEFAULT_MTU, NetworkOverview, PeerOverview, WIREGUARD_PROTOCOL, WireguardConfig, WireguardPlugin, }; pub use store::WgKeyStore; -pub use wgtool::{WgToolBackend, plan_apply, plan_remove}; +pub use tun::{MemoryTun, MemoryTunFactory, TunDevice, TunFactory, TunRequest}; + +#[cfg(feature = "tun-device")] +pub use tun::SystemTunFactory; diff --git a/src/dataplane/wireguard/packet.rs b/src/dataplane/wireguard/packet.rs new file mode 100644 index 0000000..60ae5ab --- /dev/null +++ b/src/dataplane/wireguard/packet.rs @@ -0,0 +1,125 @@ +//! The little bit of IP parsing the data plane needs. +//! +//! Two questions only: which peer should carry this packet, and did the packet +//! that came back really come from that peer? Everything is bounds checked and +//! nothing here can panic on a hostile packet. + +use std::net::{Ipv4Addr, Ipv6Addr}; + +/// The addresses of an IP packet, as far as routing cares. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IpHeader { + /// An IPv4 packet. + V4 { + /// Source address. + source: Ipv4Addr, + /// Destination address. + destination: Ipv4Addr, + }, + /// An IPv6 packet. + V6 { + /// Source address. + source: Ipv6Addr, + /// Destination address. + destination: Ipv6Addr, + }, +} + +impl IpHeader { + /// Reads the addresses out of a packet, or `None` if it is not one. + pub fn parse(packet: &[u8]) -> Option { + let version = packet.first()? >> 4; + match version { + 4 => { + let source: [u8; 4] = packet.get(12..16)?.try_into().ok()?; + let destination: [u8; 4] = packet.get(16..20)?.try_into().ok()?; + Some(IpHeader::V4 { + source: Ipv4Addr::from(source), + destination: Ipv4Addr::from(destination), + }) + } + 6 => { + let source: [u8; 16] = packet.get(8..24)?.try_into().ok()?; + let destination: [u8; 16] = packet.get(24..40)?.try_into().ok()?; + Some(IpHeader::V6 { + source: Ipv6Addr::from(source), + destination: Ipv6Addr::from(destination), + }) + } + _ => None, + } + } + + /// The destination, when the packet is IPv6. + pub fn v6_destination(&self) -> Option { + match self { + IpHeader::V6 { destination, .. } => Some(*destination), + IpHeader::V4 { .. } => None, + } + } + + /// The source, when the packet is IPv6. + pub fn v6_source(&self) -> Option { + match self { + IpHeader::V6 { source, .. } => Some(*source), + IpHeader::V4 { .. } => None, + } + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + fn ipv6_packet(source: Ipv6Addr, destination: Ipv6Addr) -> Vec { + let mut packet = vec![0u8; 48]; + packet[0] = 6 << 4; + packet[8..24].copy_from_slice(&source.octets()); + packet[24..40].copy_from_slice(&destination.octets()); + packet + } + + #[test] + fn ipv6_addresses_are_read_correctly() { + let source: Ipv6Addr = "fd00::1".parse().unwrap(); + let destination: Ipv6Addr = "fd00::2".parse().unwrap(); + let header = IpHeader::parse(&ipv6_packet(source, destination)).unwrap(); + assert_eq!(header.v6_source(), Some(source)); + assert_eq!(header.v6_destination(), Some(destination)); + } + + #[test] + fn ipv4_addresses_are_read_correctly() { + let mut packet = vec![0u8; 20]; + packet[0] = 4 << 4; + packet[12..16].copy_from_slice(&[10, 0, 0, 1]); + packet[16..20].copy_from_slice(&[10, 0, 0, 2]); + let header = IpHeader::parse(&packet).unwrap(); + assert_eq!( + header, + IpHeader::V4 { + source: Ipv4Addr::new(10, 0, 0, 1), + destination: Ipv4Addr::new(10, 0, 0, 2), + } + ); + // The overlay is IPv6, so the v6 accessors correctly report nothing. + assert_eq!(header.v6_destination(), None); + } + + #[test] + fn truncated_and_nonsense_packets_are_rejected_without_panicking() { + assert!(IpHeader::parse(&[]).is_none()); + assert!(IpHeader::parse(&[0x60]).is_none()); + assert!(IpHeader::parse(&[0x40; 19]).is_none(), "short IPv4"); + assert!(IpHeader::parse(&[0x60; 39]).is_none(), "short IPv6"); + assert!(IpHeader::parse(&[0x00; 64]).is_none(), "version 0"); + assert!(IpHeader::parse(&[0xf0; 64]).is_none(), "version 15"); + // Every possible first byte is safe to feed in. + for byte in 0..=u8::MAX { + let _ = IpHeader::parse(&[byte; 64]); + let _ = IpHeader::parse(&[byte]); + } + } +} diff --git a/src/dataplane/wireguard/plugin.rs b/src/dataplane/wireguard/plugin.rs index 4c88187..363407d 100644 --- a/src/dataplane/wireguard/plugin.rs +++ b/src/dataplane/wireguard/plugin.rs @@ -1,26 +1,31 @@ //! The WireGuard IP plugin. //! -//! Each agent builds its **own** local configuration from the set of -//! participants the control plane agreed on. For a full mesh of `N` members -//! that is `N - 1` peers locally. Nobody is handed a configuration by anybody -//! else, and no participant is authoritative. +//! Each agent builds its own view of the overlay from the set of participants +//! the control plane agreed on. For a full mesh of `N` members that is `N - 1` +//! tunnels locally. Nobody is handed a configuration by anybody else, and no +//! participant is authoritative. //! -//! What the plugin owns and what it never touches: +//! # What this plugin does and does not know //! -//! * it owns one WireGuard key per network, in its own store; -//! * it owns one interface per network, named deterministically from the -//! network id and its configured prefix; -//! * it never enumerates, adopts or edits an interface it did not create, and -//! it never changes routing, DNS or firewall settings. +//! * It does **not** know where a peer is. It is handed a +//! [`PacketLink`](crate::dataplane::transport::PacketLink) per peer and runs +//! a WireGuard tunnel over it. Reachability, hole punching and relaying are +//! the transport's problem. +//! * It owns one WireGuard key per network, in its own store, unrelated to the +//! iroh device key and to the network secret. +//! * It owns one packet interface per network, named deterministically. +//! * It never touches an interface it did not create, and never changes +//! routing, DNS or firewall settings beyond its own device. //! -//! Reconciliation runs on every change and on a timer, so a configuration -//! edited by hand is put back the way it should be. +//! WireGuard runs in userspace via [`boringtun`], so there is no kernel module +//! and no `wg` tool to depend on. The only privileged step is creating the +//! packet interface, and even that is behind [`TunFactory`] so the whole data +//! plane can run unprivileged in tests. //! -//! A failure here is reported and retried. It never stops the control plane: -//! the agent keeps receiving state and stays manageable. +//! A failure here is reported and retried. It never stops the control plane. use std::collections::{BTreeSet, HashMap}; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::path::PathBuf; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; @@ -30,39 +35,28 @@ use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; use crate::BoxFuture; +use crate::dataplane::transport::SharedLink; use crate::dataplane::{IpPlugin, PluginCapability, PluginContext, PluginError}; use crate::identity::NetworkId; use super::announcement::{ValidatedAnnouncement, WgAnnouncement}; -use super::backend::WireguardBackend; -use super::config::{ - DEFAULT_INTERFACE_PREFIX, InterfaceConfig, InterfaceParams, PortPolicy, build_interface, - interface_name, -}; +use super::config::{DEFAULT_INTERFACE_PREFIX, interface_name}; +use super::device::{PeerSummary, WireguardDevice}; use super::keys::{WgPublicKey, WgSecretKey}; -use super::overlay::{overlay_address, overlay_prefix}; +use super::overlay::{OVERLAY_PREFIX_LEN, overlay_address, overlay_prefix}; use super::store::WgKeyStore; +use super::tun::{TunFactory, TunRequest}; /// The protocol identifier this plugin announces. pub const WIREGUARD_PROTOCOL: &str = "wireguard"; -/// How the plugin advertises its own reachability. +/// Default interface MTU. /// -/// An iroh address is an address for iroh. WireGuard needs its own, so the -/// plugin gathers its own rather than reusing the control plane's. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AdvertisePolicy { - /// Advertise nothing. - /// - /// Peers can still reach this agent if they are reachable themselves: - /// WireGuard learns a peer's real source address from the first - /// authenticated packet it receives. - None, - /// Advertise exactly these addresses, combined with the listening port. - Explicit(Vec), - /// Advertise the host's own non-loopback addresses. - LocalInterfaces, -} +/// Every packet rides in one transport datagram, and WireGuard adds 32 bytes. +/// A QUIC datagram on a relayed path can be as small as roughly 1160 bytes, so +/// 1100 leaves headroom instead of relying on the best case. Packets that do +/// not fit are dropped and counted, never truncated. +pub const DEFAULT_MTU: u32 = 1100; /// Configuration of the WireGuard plugin. #[derive(Debug, Clone)] @@ -74,18 +68,14 @@ pub struct WireguardConfig { /// Two agents on one host in the same network need different prefixes, /// because the rest of the name is derived from the network id. pub interface_prefix: String, - /// How the listening port is chosen. - pub ports: PortPolicy, - /// What reachability to advertise. - pub advertise: AdvertisePolicy, - /// Keepalive interval, which holds a NAT mapping open. + /// WireGuard keepalive, which keeps tunnels and their links warm. pub keepalive: Option, - /// Interface MTU. - pub mtu: Option, + /// Interface MTU. See [`DEFAULT_MTU`]. + pub mtu: u32, /// How long to coalesce changes before reconciling. pub reconcile_debounce: Duration, - /// How often to reconcile even when nothing changed, which is what - /// corrects a configuration someone edited by hand. + /// How often to reconcile anyway, which is also when a packet interface + /// that could not be created before is retried. pub reconcile_interval: Duration, } @@ -95,12 +85,10 @@ impl WireguardConfig { Self { state_dir: state_dir.into(), interface_prefix: DEFAULT_INTERFACE_PREFIX.to_string(), - ports: PortPolicy::default(), - advertise: AdvertisePolicy::LocalInterfaces, keepalive: Some(25), - mtu: Some(1380), + mtu: DEFAULT_MTU, reconcile_debounce: Duration::from_millis(200), - reconcile_interval: Duration::from_secs(30), + reconcile_interval: Duration::from_secs(15), } } @@ -110,15 +98,9 @@ impl WireguardConfig { self } - /// Sets the port policy. - pub fn with_ports(mut self, ports: PortPolicy) -> Self { - self.ports = ports; - self - } - - /// Sets what reachability to advertise. - pub fn with_advertise(mut self, advertise: AdvertisePolicy) -> Self { - self.advertise = advertise; + /// Sets the interface MTU. + pub fn with_mtu(mut self, mtu: u32) -> Self { + self.mtu = mtu; self } @@ -140,23 +122,32 @@ impl WireguardConfig { pub struct NetworkOverview { /// The network. pub network: NetworkId, - /// Interface this plugin created for it. + /// Packet interface this plugin created for it. pub interface: String, + /// Interface MTU. + pub mtu: u32, /// This agent's WireGuard public key in this network. pub public_key: WgPublicKey, /// This agent's overlay address. pub overlay_address: IpAddr, /// The overlay subnet every member shares. pub overlay_prefix: IpAddr, - /// Port the interface listens on. - pub listen_port: u16, - /// Reachability advertised to peers. - pub advertised: Vec, - /// Peers whose announcements were accepted. + /// Prefix length of the overlay subnet. + pub overlay_prefix_len: u8, + /// Peers this agent knows about. pub peers: Vec, + /// Packets the operating system sent to an address no peer owns. + pub unroutable_packets: u64, } -/// One accepted peer. +impl NetworkOverview { + /// Peers whose tunnel has completed a handshake. + pub fn established_peers(&self) -> usize { + self.peers.iter().filter(|peer| peer.is_up()).count() + } +} + +/// One peer of the overlay. #[derive(Debug, Clone)] pub struct PeerOverview { /// The peer's control plane identity. @@ -165,17 +156,28 @@ pub struct PeerOverview { pub public_key: WgPublicKey, /// The overlay address derived for it locally. pub overlay_address: IpAddr, - /// Endpoint that will be configured for it, if any. - pub endpoint: Option, + /// Whether a data plane link to it exists. + pub has_link: bool, + /// The running tunnel, once there is a link. + pub tunnel: Option, +} + +impl PeerOverview { + /// Whether the tunnel to this peer has handshaken and can carry traffic. + pub fn is_up(&self) -> bool { + self.tunnel + .as_ref() + .is_some_and(|tunnel| tunnel.health.is_up()) + } } #[derive(Debug)] struct NetworkState { key: WgSecretKey, interface: String, - listen_port: u16, - advertised: Vec, - peers: HashMap, + device: Option>, + announcements: HashMap, + links: HashMap, } #[derive(Debug, Default)] @@ -185,19 +187,20 @@ struct Shared { #[derive(Debug)] enum Command { - /// Make sure a network has keys, a name and a port. Prepare(NetworkId), - /// Bring the interface in line with the known peers. Sync(NetworkId), - /// Remove the interface for a network. + Link { + network: NetworkId, + peer: EndpointId, + link: SharedLink, + }, Teardown(NetworkId), - /// Tear everything down and stop. Stop(oneshot::Sender<()>), } struct Worker { config: WireguardConfig, - backend: Arc, + tun_factory: Arc, store: WgKeyStore, shared: Mutex, context: OnceLock, @@ -206,7 +209,7 @@ struct Worker { impl std::fmt::Debug for Worker { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Worker") - .field("backend", &self.backend.name()) + .field("tun", &self.tun_factory.name()) .field("store", &self.store.path()) .finish() } @@ -227,7 +230,7 @@ impl WireguardPlugin { /// runtime of its own. pub async fn open( config: WireguardConfig, - backend: Arc, + tun_factory: Arc, ) -> Result, PluginError> { // Validate the prefix once, here, rather than failing per network. interface_name(&config.interface_prefix, NetworkId::from_bytes([0u8; 32]))?; @@ -239,7 +242,7 @@ impl WireguardPlugin { let worker = Arc::new(Worker { config, - backend, + tun_factory, store, shared: Mutex::new(Shared::default()), context: OnceLock::new(), @@ -259,14 +262,28 @@ impl WireguardPlugin { pub fn overview(&self, network: NetworkId) -> Option { let shared = self.worker.lock_shared(); let state = shared.networks.get(&network)?; + + let tunnels: HashMap = state + .device + .as_ref() + .map(|device| { + device + .peers() + .into_iter() + .map(|summary| (summary.public_key, summary)) + .collect() + }) + .unwrap_or_default(); + let mut peers: Vec = state - .peers + .announcements .iter() .map(|(endpoint_id, announcement)| PeerOverview { endpoint_id: *endpoint_id, public_key: announcement.public_key, overlay_address: IpAddr::V6(announcement.overlay_address), - endpoint: announcement.preferred_endpoint(), + has_link: state.links.contains_key(endpoint_id), + tunnel: tunnels.get(&announcement.public_key).cloned(), }) .collect(); peers.sort_by_key(|peer| peer.public_key); @@ -274,18 +291,21 @@ impl WireguardPlugin { Some(NetworkOverview { network, interface: state.interface.clone(), + mtu: self.worker.config.mtu, public_key: state.key.public(), overlay_address: IpAddr::V6(overlay_address(network, &state.key.public())), overlay_prefix: IpAddr::V6(overlay_prefix(network)), - listen_port: state.listen_port, - advertised: state.advertised.clone(), + overlay_prefix_len: OVERLAY_PREFIX_LEN, peers, + unroutable_packets: state + .device + .as_ref() + .map(|device| device.unroutable_packets()) + .unwrap_or(0), }) } - /// Asks the reconciliation task to run now, and waits for it to be queued. - /// - /// Tests use it to avoid waiting for the periodic tick. + /// Asks the reconciliation task to run now. pub async fn reconcile_now(&self, network: NetworkId) { let _ = self.commands.send(Command::Sync(network)).await; } @@ -293,7 +313,7 @@ impl WireguardPlugin { fn nudge(&self, command: Command) { if let Err(err) = self.commands.try_send(command) { // A full queue means work is already scheduled; the periodic - // reconcile will pick anything up that was missed. + // reconcile picks up anything that was missed. tracing::debug!(%err, "wireguard command queue is busy"); } } @@ -320,55 +340,27 @@ impl Worker { } } - /// Gathers the addresses to advertise for our own listening port. - async fn advertised_endpoints(&self, listen_port: u16) -> Vec { - let addresses: Vec = match &self.config.advertise { - AdvertisePolicy::None => Vec::new(), - AdvertisePolicy::Explicit(addresses) => addresses.clone(), - AdvertisePolicy::LocalInterfaces => { - let state = netwatch::interfaces::State::new().await; - state.local_addresses.regular - } - }; - - let mut endpoints: Vec = addresses - .into_iter() - .filter(|addr| !addr.is_loopback() && !addr.is_unspecified() && !is_link_local(addr)) - .map(|addr| SocketAddr::new(addr, listen_port)) - .collect(); - endpoints.sort(); - endpoints.dedup(); - endpoints.truncate(super::announcement::MAX_ENDPOINTS); - endpoints - } - - /// Makes sure a network has a key, an interface name and a port. + /// Makes sure a network has a key, a name and a running packet interface. /// - /// Returns `true` when something changed and peers should be told. + /// Returns `true` when the key became available now, so peers should be + /// told. Creating the interface may fail without privileges; the key and + /// the announcement still work, and the interface is retried. async fn prepare(self: &Arc, network: NetworkId) -> Result { let existing = { let shared = self.lock_shared(); shared .networks .get(&network) - .map(|state| (state.listen_port, state.advertised.clone())) + .map(|state| state.device.is_some()) }; - let listen_port = match existing { - Some((port, _)) => port, - None => self.config.ports.port_for(network)?, - }; - let advertised = self.advertised_endpoints(listen_port).await; - - if let Some((_, previous)) = existing { - if previous == advertised { + if let Some(has_device) = existing { + if has_device { return Ok(false); } - let mut shared = self.lock_shared(); - if let Some(state) = shared.networks.get_mut(&network) { - state.advertised = advertised; - } - return Ok(true); + // The key is there but the interface is not. Try again. + self.ensure_device(network).await?; + return Ok(false); } let name = interface_name(&self.config.interface_prefix, network)?; @@ -377,86 +369,97 @@ impl Worker { .await .map_err(|err| PluginError::Other(format!("key store task failed: {err}")))??; - let mut shared = self.lock_shared(); - shared.networks.entry(network).or_insert(NetworkState { - key, - interface: name, - listen_port, - advertised, - peers: HashMap::new(), - }); + { + let mut shared = self.lock_shared(); + shared.networks.entry(network).or_insert(NetworkState { + key, + interface: name, + device: None, + announcements: HashMap::new(), + links: HashMap::new(), + }); + } + + // The announcement only needs the key, so peers can be told even if + // the interface is not up yet. + let device = self.ensure_device(network).await; + if let Err(err) = device { + self.report(network, err); + } Ok(true) } - /// Builds the configuration this agent wants for a network. - fn desired_config(&self, network: NetworkId) -> Option { - let shared = self.lock_shared(); - let state = shared.networks.get(&network)?; - - let endpoints: HashMap = state - .peers - .values() - .filter_map(|announcement| { - announcement - .preferred_endpoint() - .map(|endpoint| (announcement.public_key, endpoint)) - }) - .collect(); - let keys: Vec = state - .peers - .values() - .map(|announcement| announcement.public_key) - .collect(); - - Some(build_interface( - InterfaceParams { - network, - name: state.interface.clone(), - private_key: state.key.clone(), - listen_port: state.listen_port, - mtu: self.config.mtu, - keepalive: self.config.keepalive, - }, - keys, - |key| endpoints.get(key).copied(), - )) - } - - /// Brings the interface in line with the desired configuration. - async fn sync(&self, network: NetworkId) -> Result<(), PluginError> { - let Some(desired) = self.desired_config(network) else { - return Ok(()); - }; - let backend = Arc::clone(&self.backend); - tokio::task::spawn_blocking(move || { - let current = backend.inspect(&desired.name)?; - // Reconciliation: anything that drifted, including an edit made by - // hand, is corrected here. - if current.as_ref() == Some(&desired.to_state()) { - return Ok(()); + /// Creates the packet interface and starts the WireGuard device. + async fn ensure_device(&self, network: NetworkId) -> Result<(), PluginError> { + let (name, key) = { + let shared = self.lock_shared(); + match shared.networks.get(&network) { + Some(state) if state.device.is_none() => { + (state.interface.clone(), state.key.clone()) + } + _ => return Ok(()), } - backend.apply(&desired) - }) - .await - .map_err(|err| PluginError::Other(format!("wireguard apply task failed: {err}")))? + }; + + let request = TunRequest { + name: name.clone(), + address: overlay_address(network, &key.public()), + prefix_len: OVERLAY_PREFIX_LEN, + mtu: self.config.mtu, + }; + let tun = self.tun_factory.create(request).await?; + let device = Arc::new(WireguardDevice::start(network, key, tun)); + + let mut shared = self.lock_shared(); + if let Some(state) = shared.networks.get_mut(&network) { + state.interface = device.interface().to_string(); + state.device = Some(device); + } + Ok(()) } - /// Removes the interface for a network, keeping its key. - async fn teardown(&self, network: NetworkId) -> Result<(), PluginError> { - let interface = { - let mut shared = self.lock_shared(); - shared - .networks - .remove(&network) - .map(|state| state.interface) + /// Brings the running tunnels in line with what is known. + /// + /// A peer gets a tunnel once both halves have arrived: its announcement, + /// which says who it is, and a link, which says packets can reach it. + fn sync(&self, network: NetworkId) { + let mut shared = self.lock_shared(); + let Some(state) = shared.networks.get_mut(&network) else { + return; }; - let Some(interface) = interface else { - return Ok(()); + let Some(device) = state.device.clone() else { + return; }; - let backend = Arc::clone(&self.backend); - tokio::task::spawn_blocking(move || backend.remove(&interface)) - .await - .map_err(|err| PluginError::Other(format!("wireguard remove task failed: {err}")))? + + let mut wanted: Vec = Vec::new(); + for (endpoint_id, announcement) in &state.announcements { + let Some(link) = state.links.get(endpoint_id) else { + continue; + }; + if link.is_closed() { + continue; + } + wanted.push(announcement.public_key); + if device.has_peer(&announcement.public_key) { + continue; + } + if let Err(err) = device.add_peer( + *endpoint_id, + announcement.public_key, + Arc::clone(link), + self.config.keepalive, + ) { + tracing::debug!(%err, "cannot start a WireGuard tunnel"); + } + } + device.retain_peers(&wanted); + } + + /// Removes a network's interface and tunnels, keeping its key. + fn teardown(&self, network: NetworkId) { + // Dropping the state drops the device, which stops its tasks and + // closes the packet interface. + self.lock_shared().networks.remove(&network); } fn known_networks(&self) -> Vec { @@ -465,16 +468,11 @@ impl Worker { } /// The reconciliation task. -/// -/// Changes are coalesced over a short debounce so that a burst of peer -/// announcements produces one apply, and a periodic tick reconciles even when -/// nothing changed locally. async fn run(worker: Arc, mut commands: mpsc::Receiver) { let mut pending: BTreeSet = BTreeSet::new(); let mut deadline: Option = None; let mut ticker = tokio::time::interval(worker.config.reconcile_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - // The first tick fires immediately and would reconcile nothing. ticker.tick().await; loop { @@ -495,18 +493,23 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { Command::Sync(network) => { pending.insert(network); } + Command::Link { network, peer, link } => { + { + let mut shared = worker.lock_shared(); + if let Some(state) = shared.networks.get_mut(&network) { + state.links.insert(peer, link); + } + } + pending.insert(network); + } Command::Teardown(network) => { pending.remove(&network); - if let Err(err) = worker.teardown(network).await { - worker.report(network, err); - } + worker.teardown(network); continue; } Command::Stop(reply) => { for network in worker.known_networks() { - if let Err(err) = worker.teardown(network).await { - tracing::warn!(%err, "wireguard teardown failed during shutdown"); - } + worker.teardown(network); } let _ = reply.send(()); return; @@ -517,37 +520,28 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { _ = async { match wait_until { Some(at) => tokio::time::sleep_until(at).await, - // Never resolves; the branch is disabled by the guard. None => std::future::pending::<()>().await, } }, if wait_until.is_some() => { deadline = None; for network in std::mem::take(&mut pending) { - if let Err(err) = worker.sync(network).await { - worker.report(network, err); - } + worker.sync(network); } } _ = ticker.tick() => { - // Periodic reconciliation is what corrects drift nobody told - // us about. for network in worker.known_networks() { - if let Err(err) = worker.sync(network).await { - worker.report(network, err); + // Also the retry for an interface that could not be + // created earlier. + if let Err(err) = worker.ensure_device(network).await { + tracing::debug!(%err, "packet interface still unavailable"); } + worker.sync(network); } } } } } -fn is_link_local(addr: &IpAddr) -> bool { - match addr { - IpAddr::V4(ip) => ip.is_link_local(), - IpAddr::V6(ip) => (ip.segments()[0] & 0xffc0) == 0xfe80, - } -} - impl IpPlugin for WireguardPlugin { fn protocol_id(&self) -> &str { WIREGUARD_PROTOCOL @@ -567,19 +561,15 @@ impl IpPlugin for WireguardPlugin { ) -> Result, PluginError> { let shared = self.worker.lock_shared(); let Some(state) = shared.networks.get(&network) else { - // Not ready yet. Ask for preparation; once it finishes the plugin - // asks the agent to re-announce, so peers are not left waiting. + // Not ready yet. Ask for preparation; once the key exists the + // plugin asks the agent to re-announce. drop(shared); self.nudge(Command::Prepare(network)); return Ok(None); }; - let announcement = WgAnnouncement::new( - network, - &state.key.public(), - state.listen_port, - state.advertised.clone(), - ); + // Identity only. Where to send packets is the transport's business. + let announcement = WgAnnouncement::new(network, &state.key.public()); Ok(Some(PluginCapability { protocol: WIREGUARD_PROTOCOL.to_string(), version: super::announcement::ANNOUNCEMENT_VERSION, @@ -614,7 +604,7 @@ impl IpPlugin for WireguardPlugin { let mut shared = self.worker.lock_shared(); match shared.networks.get_mut(&network) { Some(state) => { - state.peers.insert(peer, validated) != state.peers.get(&peer).cloned() + state.announcements.insert(peer, validated.clone()) != Some(validated) } None => false, } @@ -625,14 +615,24 @@ impl IpPlugin for WireguardPlugin { Ok(()) } + fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) { + self.nudge(Command::Link { + network, + peer, + link, + }); + } + fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) { let removed = { let mut shared = self.worker.lock_shared(); - shared - .networks - .get_mut(&network) - .and_then(|state| state.peers.remove(&peer)) - .is_some() + match shared.networks.get_mut(&network) { + Some(state) => { + let had_link = state.links.remove(&peer).is_some(); + state.announcements.remove(&peer).is_some() || had_link + } + None => false, + } }; if removed { self.nudge(Command::Sync(network)); @@ -659,7 +659,6 @@ impl IpPlugin for WireguardPlugin { impl Drop for WireguardPlugin { fn drop(&mut self) { - // Safety net for a plugin dropped without an explicit shutdown. if let Ok(mut guard) = self.task.lock() && let Some(task) = guard.take() { diff --git a/src/dataplane/wireguard/tun.rs b/src/dataplane/wireguard/tun.rs new file mode 100644 index 0000000..d753e8b --- /dev/null +++ b/src/dataplane/wireguard/tun.rs @@ -0,0 +1,307 @@ +//! The boundary to the operating system's packet interface. +//! +//! The WireGuard implementation in [`super::device`] is pure userspace and +//! needs no kernel WireGuard module and no `wg` tool. It does still need a way +//! to hand IP packets to the operating system, which is what this trait is. +//! +//! Two implementations: +//! +//! * [`MemoryTun`] keeps packets in memory. It needs no privileges at all and +//! is what the test suite uses, so the entire data plane — handshake, +//! encryption, routing — is exercised without touching the host. +//! * `SystemTun`, behind the `tun-device` feature, is a real TUN interface. +//! Creating one needs `CAP_NET_ADMIN` on Linux or the equivalent elsewhere. + +use std::net::Ipv6Addr; +use std::sync::Arc; + +use bytes::Bytes; + +use crate::BoxFuture; +use crate::dataplane::PluginError; + +/// What a device should look like once created. +#[derive(Debug, Clone)] +pub struct TunRequest { + /// Interface name to ask for. + pub name: String, + /// The overlay address this host answers to. + pub address: Ipv6Addr, + /// Prefix length of the overlay subnet, so the OS routes it here. + pub prefix_len: u8, + /// Interface MTU. + pub mtu: u32, +} + +/// A packet interface. +/// +/// `recv` yields packets the operating system wants sent; `send` delivers +/// packets that arrived from a peer. +pub trait TunDevice: Send + Sync + std::fmt::Debug + 'static { + /// The interface name the operating system actually gave us. + fn name(&self) -> &str; + + /// The interface MTU. + fn mtu(&self) -> u32; + + /// The next packet the operating system wants to send, or `None` once the + /// device is gone. + fn recv(&self) -> BoxFuture<'_, Option>; + + /// Delivers a packet to the operating system. + fn send<'a>(&'a self, packet: Bytes) -> BoxFuture<'a, Result<(), PluginError>>; +} + +/// Creates packet interfaces. +pub trait TunFactory: Send + Sync + std::fmt::Debug + 'static { + /// A short name used in diagnostics. + fn name(&self) -> &str; + + /// Creates a device. + fn create<'a>( + &'a self, + request: TunRequest, + ) -> BoxFuture<'a, Result, PluginError>>; +} + +/// An in-memory packet interface. +/// +/// Nothing reaches the operating system. Packets the device "sends" can be +/// read back with [`MemoryTun::pop_to_os`], and packets can be injected as if +/// the operating system produced them with [`MemoryTun::push_from_os`]. +#[derive(Debug)] +pub struct MemoryTun { + name: String, + mtu: u32, + from_os_tx: tokio::sync::mpsc::UnboundedSender, + from_os_rx: tokio::sync::Mutex>, + to_os_tx: tokio::sync::mpsc::UnboundedSender, + to_os_rx: tokio::sync::Mutex>, +} + +impl MemoryTun { + /// Creates a device with the given name and MTU. + pub fn new(name: impl Into, mtu: u32) -> Arc { + let (from_os_tx, from_os_rx) = tokio::sync::mpsc::unbounded_channel(); + let (to_os_tx, to_os_rx) = tokio::sync::mpsc::unbounded_channel(); + Arc::new(Self { + name: name.into(), + mtu, + from_os_tx, + from_os_rx: tokio::sync::Mutex::new(from_os_rx), + to_os_tx, + to_os_rx: tokio::sync::Mutex::new(to_os_rx), + }) + } + + /// Injects a packet as if the operating system had produced it. + pub fn push_from_os(&self, packet: Bytes) { + let _ = self.from_os_tx.send(packet); + } + + /// Takes the next packet the device delivered to the operating system. + pub async fn pop_to_os(&self) -> Option { + self.to_os_rx.lock().await.recv().await + } +} + +impl TunDevice for MemoryTun { + fn name(&self) -> &str { + &self.name + } + + fn mtu(&self) -> u32 { + self.mtu + } + + fn recv(&self) -> BoxFuture<'_, Option> { + Box::pin(async move { self.from_os_rx.lock().await.recv().await }) + } + + fn send<'a>(&'a self, packet: Bytes) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + let _ = self.to_os_tx.send(packet); + Ok(()) + }) + } +} + +/// Creates [`MemoryTun`] devices. +#[derive(Debug, Clone, Default)] +pub struct MemoryTunFactory { + created: Arc>>>, +} + +impl MemoryTunFactory { + /// Creates a factory. + pub fn new() -> Self { + Self::default() + } + + /// The device created for an interface name, if any. + pub fn device(&self, name: &str) -> Option> { + let guard = match self.created.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + guard + .iter() + .find(|device| device.name() == name) + .map(Arc::clone) + } + + /// Every device created so far. + pub fn devices(&self) -> Vec> { + let guard = match self.created.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + guard.clone() + } +} + +impl TunFactory for MemoryTunFactory { + fn name(&self) -> &str { + "memory" + } + + fn create<'a>( + &'a self, + request: TunRequest, + ) -> BoxFuture<'a, Result, PluginError>> { + Box::pin(async move { + let device = MemoryTun::new(request.name, request.mtu); + let mut guard = match self.created.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + guard.push(Arc::clone(&device)); + Ok(device as Arc) + }) + } +} + +#[cfg(feature = "tun-device")] +pub use system::SystemTunFactory; + +#[cfg(feature = "tun-device")] +mod system { + use std::sync::Arc; + + use bytes::Bytes; + use tokio::sync::Mutex; + + use super::{TunDevice, TunFactory, TunRequest}; + use crate::BoxFuture; + use crate::dataplane::PluginError; + + /// A real TUN interface. + /// + /// Creating one needs `CAP_NET_ADMIN` on Linux, or the platform + /// equivalent. Failure is reported, never fatal for the agent. + pub struct SystemTun { + name: String, + mtu: u32, + reader: Mutex>, + writer: Mutex>, + } + + impl std::fmt::Debug for SystemTun { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SystemTun") + .field("name", &self.name) + .field("mtu", &self.mtu) + .finish() + } + } + + impl TunDevice for SystemTun { + fn name(&self) -> &str { + &self.name + } + + fn mtu(&self) -> u32 { + self.mtu + } + + fn recv(&self) -> BoxFuture<'_, Option> { + Box::pin(async move { + use tokio::io::AsyncReadExt; + let mut buffer = vec![0u8; self.mtu as usize + 64]; + let mut reader = self.reader.lock().await; + match reader.read(&mut buffer).await { + Ok(0) => None, + Ok(read) => { + buffer.truncate(read); + Some(Bytes::from(buffer)) + } + Err(err) => { + tracing::debug!(%err, "tun read failed"); + None + } + } + }) + } + + fn send<'a>(&'a self, packet: Bytes) -> BoxFuture<'a, Result<(), PluginError>> { + Box::pin(async move { + use tokio::io::AsyncWriteExt; + let mut writer = self.writer.lock().await; + writer + .write_all(&packet) + .await + .map_err(|err| PluginError::Other(format!("tun write failed: {err}"))) + }) + } + } + + /// Creates real TUN interfaces. + #[derive(Debug, Clone, Default)] + pub struct SystemTunFactory; + + impl SystemTunFactory { + /// Creates the factory. + pub fn new() -> Self { + Self + } + } + + impl TunFactory for SystemTunFactory { + fn name(&self) -> &str { + "system" + } + + fn create<'a>( + &'a self, + request: TunRequest, + ) -> BoxFuture<'a, Result, PluginError>> { + Box::pin(async move { + let mut config = tun::Configuration::default(); + config.tun_name(&request.name).mtu(request.mtu as u16).up(); + // The overlay address and its subnet, so the operating system + // routes overlay traffic into this interface. + let _ = (&request.address, request.prefix_len); + + let device = tun::create_as_async(&config).map_err(|err| { + PluginError::Unavailable(format!( + "cannot create the TUN interface `{}`: {err}. \ + This needs CAP_NET_ADMIN (try running as root).", + request.name + )) + })?; + + // The name was requested explicitly; creation fails rather + // than silently picking another one. + let name = request.name.clone(); + let (reader, writer) = tokio::io::split(device); + + Ok(Arc::new(SystemTun { + name, + mtu: request.mtu, + reader: Mutex::new(reader), + writer: Mutex::new(writer), + }) as Arc) + }) + } + } +} diff --git a/src/dataplane/wireguard/wgtool.rs b/src/dataplane/wireguard/wgtool.rs deleted file mode 100644 index db3be84..0000000 --- a/src/dataplane/wireguard/wgtool.rs +++ /dev/null @@ -1,667 +0,0 @@ -//! A backend that drives the standard `wg` and `ip` tools. -//! -//! This is the one part of the plugin that changes the operating system. It is -//! split in two deliberately: -//! -//! * a **pure planner** that turns a desired configuration into an exact list -//! of commands, and pure **parsers** for the tools' output — both fully -//! unit tested on every platform; -//! * a thin executor that runs the plan, which needs Linux and -//! `CAP_NET_ADMIN`. -//! -//! Nothing that arrives from the network is ever passed through as text. Peer -//! keys, endpoints, allowed prefixes and keepalives are typed values that this -//! module re-serialises itself, so an announcement cannot inject an argument -//! or a configuration directive. The only names involved are derived locally. -//! -//! The interface is created by this plugin and removed by this plugin. An -//! interface that already exists and is not a WireGuard device is refused, not -//! adopted, so the agent never takes over something it did not create. - -use std::collections::BTreeSet; -use std::net::SocketAddr; - -use zeroize::Zeroizing; - -use crate::dataplane::PluginError; - -use super::config::{Cidr, InterfaceConfig, InterfaceState, PeerState}; -use super::keys::{WgPublicKey, WgSecretKey}; - -/// Which external programs to use. -#[derive(Debug, Clone)] -pub struct Tools { - /// The `wg` executable. - pub wg: String, - /// The `ip` executable. - pub ip: String, -} - -impl Default for Tools { - fn default() -> Self { - Self { - wg: "wg".into(), - ip: "ip".into(), - } - } -} - -/// One command to run, with optional data for its standard input. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct WgCommand { - /// Program to execute. - pub program: String, - /// Arguments, already separated. Never a shell string. - pub args: Vec, - /// Data piped to the program's standard input. - /// - /// Used for the WireGuard configuration so the private key never reaches - /// the filesystem. Zeroized on drop. - pub stdin: Option>, -} - -impl WgCommand { - fn new(program: &str, args: &[&str]) -> Self { - Self { - program: program.to_string(), - args: args.iter().map(|arg| arg.to_string()).collect(), - stdin: None, - } - } - - /// A redacted rendering, safe for logs. - pub fn describe(&self) -> String { - format!("{} {}", self.program, self.args.join(" ")) - } -} - -/// Builds the commands that bring `desired` into being. -/// -/// `current` is what the interface looks like now, or `None` if it does not -/// exist yet. The plan is minimal: an interface that already matches produces -/// only the idempotent link-up command. -pub fn plan_apply( - desired: &InterfaceConfig, - current: Option<&InterfaceState>, - tools: &Tools, -) -> Vec { - let mut plan = Vec::new(); - let name = desired.name.as_str(); - - if current.is_none() { - plan.push(WgCommand::new( - &tools.ip, - &["link", "add", "dev", name, "type", "wireguard"], - )); - } - - // `setconf` replaces everything, `syncconf` applies a difference without - // tearing down live peers. Use each where it belongs. - let subcommand = if current.is_none() { - "setconf" - } else { - "syncconf" - }; - let mut configure = WgCommand::new(&tools.wg, &[subcommand, name, "/dev/stdin"]); - configure.stdin = Some(desired.render()); - plan.push(configure); - - let desired_addrs: BTreeSet = desired.addresses.iter().copied().collect(); - let current_addrs: BTreeSet = current - .map(|state| state.addresses.iter().copied().collect()) - .unwrap_or_default(); - - for addr in desired_addrs.difference(¤t_addrs) { - plan.push(WgCommand::new( - &tools.ip, - &["address", "add", &addr.to_string(), "dev", name], - )); - } - // Addresses on an interface this plugin owns that are not wanted any more - // were either put there by an older configuration or by hand. Either way - // reconciliation removes them. - for addr in current_addrs.difference(&desired_addrs) { - plan.push(WgCommand::new( - &tools.ip, - &["address", "del", &addr.to_string(), "dev", name], - )); - } - - if let Some(mtu) = desired.mtu { - plan.push(WgCommand::new( - &tools.ip, - &["link", "set", "mtu", &mtu.to_string(), "dev", name], - )); - } - - plan.push(WgCommand::new( - &tools.ip, - &["link", "set", "up", "dev", name], - )); - plan -} - -/// Builds the commands that remove an interface this plugin created. -pub fn plan_remove(interface: &str, tools: &Tools) -> Vec { - vec![WgCommand::new( - &tools.ip, - &["link", "del", "dev", interface], - )] -} - -/// Parses the output of `wg showconf `. -/// -/// The private key present in that output is used only to derive the -/// interface's public key and is dropped immediately. -pub fn parse_showconf(interface: &str, text: &str) -> Result { - #[derive(Default)] - struct PartialPeer { - public_key: Option, - endpoint: Option, - allowed_ips: Vec, - persistent_keepalive: Option, - } - - let mut public_key: Option = None; - let mut listen_port = 0u16; - let mut peers: Vec = Vec::new(); - let mut current: Option = None; - - let finish = |peer: PartialPeer, peers: &mut Vec| -> Result<(), PluginError> { - let key = peer - .public_key - .ok_or_else(|| PluginError::Other("wg showconf peer without a public key".into()))?; - peers.push( - PeerState { - public_key: key, - endpoint: peer.endpoint, - allowed_ips: peer.allowed_ips, - persistent_keepalive: peer.persistent_keepalive, - } - .normalised(), - ); - Ok(()) - }; - - for raw in text.lines() { - let line = raw.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - if line.eq_ignore_ascii_case("[interface]") { - if let Some(peer) = current.take() { - finish(peer, &mut peers)?; - } - continue; - } - if line.eq_ignore_ascii_case("[peer]") { - if let Some(peer) = current.take() { - finish(peer, &mut peers)?; - } - current = Some(PartialPeer::default()); - continue; - } - - let Some((key, value)) = line.split_once('=') else { - continue; - }; - let name = key.trim().to_ascii_lowercase(); - let value = value.trim(); - - match current.as_mut() { - None => match name.as_str() { - "privatekey" => { - // Derive the public key, then let the secret drop. - let raw = data_encoding::BASE64 - .decode(value.as_bytes()) - .map_err(|_| { - PluginError::Other("wg showconf private key is not base64".into()) - })?; - let bytes = <[u8; 32]>::try_from(raw.as_slice()).map_err(|_| { - PluginError::Other("wg showconf private key is not 32 bytes".into()) - })?; - public_key = Some(WgSecretKey::from_bytes(&bytes).public()); - } - "listenport" => { - listen_port = value - .parse() - .map_err(|_| PluginError::Other(format!("bad listen port {value:?}")))?; - } - _ => {} - }, - Some(peer) => match name.as_str() { - "publickey" => peer.public_key = Some(WgPublicKey::decode(value)?), - "endpoint" => peer.endpoint = value.parse().ok(), - "allowedips" => { - for entry in value.split(',') { - let entry = entry.trim(); - if entry.is_empty() { - continue; - } - peer.allowed_ips.push(parse_cidr(entry)?); - } - } - "persistentkeepalive" => { - peer.persistent_keepalive = - match value { - "off" => None, - other => Some(other.parse().map_err(|_| { - PluginError::Other(format!("bad keepalive {other:?}")) - })?), - }; - } - _ => {} - }, - } - } - if let Some(peer) = current.take() { - finish(peer, &mut peers)?; - } - - let public_key = public_key - .ok_or_else(|| PluginError::Other("wg showconf did not report a private key".into()))?; - - Ok(InterfaceState { - name: interface.to_string(), - public_key, - listen_port, - addresses: Vec::new(), - peers, - } - .normalised()) -} - -/// Parses the addresses out of `ip -o address show dev `. -pub fn parse_ip_addresses(text: &str) -> Result, PluginError> { - let mut out = Vec::new(); - for line in text.lines() { - let mut tokens = line.split_whitespace(); - while let Some(token) = tokens.next() { - if token != "inet" && token != "inet6" { - continue; - } - let Some(value) = tokens.next() else { - continue; - }; - // A link-local address is added by the kernel, not by us. - let cidr = parse_cidr(value)?; - if cidr.addr.is_loopback() { - continue; - } - if let std::net::IpAddr::V6(ip) = cidr.addr - && (ip.segments()[0] & 0xffc0) == 0xfe80 - { - continue; - } - out.push(cidr); - } - } - out.sort(); - out.dedup(); - Ok(out) -} - -fn parse_cidr(text: &str) -> Result { - let (addr, prefix) = text - .split_once('/') - .ok_or_else(|| PluginError::Other(format!("{text:?} is not an address with a prefix")))?; - let addr = addr - .parse() - .map_err(|_| PluginError::Other(format!("{addr:?} is not an IP address")))?; - let prefix_len = prefix - .parse() - .map_err(|_| PluginError::Other(format!("{prefix:?} is not a prefix length")))?; - Cidr::new(addr, prefix_len) -} - -pub use executor::WgToolBackend; - -#[cfg(target_os = "linux")] -mod executor { - use std::io::Write; - use std::process::{Command, Stdio}; - - use super::*; - use crate::dataplane::wireguard::backend::WireguardBackend; - - /// Drives the real `wg` and `ip` tools. - /// - /// Requires Linux and `CAP_NET_ADMIN`, so it is never used by the default - /// test suite. - #[derive(Debug, Clone)] - pub struct WgToolBackend { - tools: Tools, - } - - impl WgToolBackend { - /// Creates a backend using `wg` and `ip` from `PATH`. - pub fn new() -> Result { - Self::with_tools(Tools::default()) - } - - /// Creates a backend using explicitly located tools. - pub fn with_tools(tools: Tools) -> Result { - Ok(Self { tools }) - } - - fn run(&self, command: &WgCommand) -> Result { - let mut child = Command::new(&command.program) - .args(&command.args) - .stdin(if command.stdin.is_some() { - Stdio::piped() - } else { - Stdio::null() - }) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|err| { - PluginError::Unavailable(format!("cannot run `{}`: {err}", command.program)) - })?; - - if let Some(stdin) = &command.stdin { - let mut handle = child.stdin.take().ok_or_else(|| { - PluginError::Other("could not open the child's standard input".into()) - })?; - handle.write_all(stdin.as_bytes()).map_err(|err| { - PluginError::Other(format!("cannot write the WireGuard configuration: {err}")) - })?; - drop(handle); - } - - let output = child.wait_with_output().map_err(|err| { - PluginError::Other(format!("`{}` did not complete: {err}", command.describe())) - })?; - if !output.status.success() { - // The configuration went to stdin, so stderr cannot contain it. - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - return Err(PluginError::Unavailable(format!( - "`{}` failed: {stderr}", - command.describe() - ))); - } - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } - - fn link_exists(&self, interface: &str) -> bool { - Command::new(&self.tools.ip) - .args(["link", "show", "dev", interface]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|status| status.success()) - .unwrap_or(false) - } - } - - impl WireguardBackend for WgToolBackend { - fn name(&self) -> &str { - "wg-tools" - } - - fn inspect(&self, interface: &str) -> Result, PluginError> { - if !self.link_exists(interface) { - return Ok(None); - } - let showconf = WgCommand::new(&self.tools.wg, &["showconf", interface]); - let text = match self.run(&showconf) { - Ok(text) => text, - Err(_) => { - // The link exists but is not a WireGuard device. It is not - // ours, so it is refused rather than adopted or modified. - return Err(PluginError::Rejected(format!( - "interface `{interface}` already exists and is not a WireGuard device; \ - refusing to touch it" - ))); - } - }; - let mut state = parse_showconf(interface, &text)?; - let addresses = self.run(&WgCommand::new( - &self.tools.ip, - &["-o", "address", "show", "dev", interface], - ))?; - state.addresses = parse_ip_addresses(&addresses)?; - Ok(Some(state.normalised())) - } - - fn apply(&self, desired: &InterfaceConfig) -> Result<(), PluginError> { - let current = self.inspect(&desired.name)?; - for command in plan_apply(desired, current.as_ref(), &self.tools) { - self.run(&command)?; - } - Ok(()) - } - - fn remove(&self, interface: &str) -> Result<(), PluginError> { - if !self.link_exists(interface) { - return Ok(()); - } - for command in plan_remove(interface, &self.tools) { - self.run(&command)?; - } - Ok(()) - } - } -} - -#[cfg(not(target_os = "linux"))] -mod executor { - use super::*; - - /// Placeholder on platforms where this backend is not implemented. - /// - /// The planner and the parsers in this module work everywhere; only - /// applying a configuration is Linux-specific. - #[derive(Debug, Clone)] - pub struct WgToolBackend { - _private: (), - } - - impl WgToolBackend { - /// Always fails: this backend drives `ip link ... type wireguard`, - /// which exists on Linux only. - pub fn new() -> Result { - Self::with_tools(Tools::default()) - } - - /// Always fails, see [`WgToolBackend::new`]. - pub fn with_tools(_tools: Tools) -> Result { - Err(PluginError::Unavailable( - "the wg-tools backend is implemented for Linux only".into(), - )) - } - } -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - - use super::*; - use crate::dataplane::wireguard::config::{InterfaceParams, build_interface}; - use crate::dataplane::wireguard::keys::WgSecretKey; - use crate::dataplane::wireguard::overlay::overlay_address; - use crate::identity::{NetworkId, NetworkKeys, NetworkName, NetworkSecret}; - - fn network(name: &str) -> NetworkId { - NetworkKeys::derive( - &NetworkName::new(name).unwrap(), - &NetworkSecret::from_bytes(vec![4u8; 32]).unwrap(), - ) - .network_id() - } - - fn sample() -> (NetworkId, InterfaceConfig, WgPublicKey) { - let id = network("plan"); - let peer = WgSecretKey::generate().public(); - let config = build_interface( - InterfaceParams { - network: id, - name: "tsun0".into(), - private_key: WgSecretKey::generate(), - listen_port: 51820, - mtu: Some(1380), - keepalive: Some(25), - }, - [peer], - |_| Some("10.0.0.5:51820".parse().unwrap()), - ); - (id, config, peer) - } - - #[test] - fn creating_an_interface_plans_every_step_in_order() { - let (_, config, _) = sample(); - let plan = plan_apply(&config, None, &Tools::default()); - let described: Vec = plan.iter().map(WgCommand::describe).collect(); - - assert_eq!(described[0], "ip link add dev tsun0 type wireguard"); - assert_eq!(described[1], "wg setconf tsun0 /dev/stdin"); - assert!(plan[1].stdin.is_some(), "the config goes over stdin"); - assert!(described.iter().any(|c| c.starts_with("ip address add"))); - assert!(described.contains(&"ip link set mtu 1380 dev tsun0".to_string())); - assert_eq!(described.last().unwrap(), "ip link set up dev tsun0"); - } - - #[test] - fn updating_an_existing_interface_syncs_instead_of_replacing() { - let (_, config, _) = sample(); - let current = config.to_state(); - let plan = plan_apply(&config, Some(¤t), &Tools::default()); - let described: Vec = plan.iter().map(WgCommand::describe).collect(); - - assert!( - !described.iter().any(|c| c.contains("link add")), - "an existing interface must not be recreated" - ); - assert_eq!(described[0], "wg syncconf tsun0 /dev/stdin"); - assert!( - !described.iter().any(|c| c.starts_with("ip address add")), - "matching addresses need no change: {described:?}" - ); - } - - #[test] - fn addresses_that_should_not_be_there_are_removed() { - let (_, config, _) = sample(); - let mut current = config.to_state(); - current.addresses.push(Cidr { - addr: "192.0.2.1".parse().unwrap(), - prefix_len: 32, - }); - let plan = plan_apply(&config, Some(¤t), &Tools::default()); - let described: Vec = plan.iter().map(WgCommand::describe).collect(); - assert!( - described.contains(&"ip address del 192.0.2.1/32 dev tsun0".to_string()), - "{described:?}" - ); - } - - #[test] - fn nothing_from_the_network_reaches_an_argument_as_text() { - let (id, config, peer) = sample(); - let plan = plan_apply(&config, None, &Tools::default()); - for command in &plan { - for arg in &command.args { - assert!( - !arg.contains(' ') && !arg.contains(';') && !arg.contains('\n'), - "argument {arg:?} is not a single clean token" - ); - } - } - // The peer's key and derived prefix travel in the piped configuration, - // which is a value this crate rendered itself. - let rendered = plan[1].stdin.as_ref().unwrap(); - assert!(rendered.contains(&peer.encode())); - assert!(rendered.contains(&Cidr::host(overlay_address(id, &peer)).to_string())); - } - - #[test] - fn removal_only_touches_the_named_interface() { - let plan = plan_remove("tsun0", &Tools::default()); - assert_eq!( - plan.iter().map(WgCommand::describe).collect::>(), - vec!["ip link del dev tsun0".to_string()] - ); - } - - #[test] - fn showconf_output_parses_into_comparable_state() { - let secret = WgSecretKey::generate(); - let peer_a = WgSecretKey::generate().public(); - let peer_b = WgSecretKey::generate().public(); - let text = format!( - "[Interface]\n\ - ListenPort = 51821\n\ - PrivateKey = {}\n\ - \n\ - [Peer]\n\ - PublicKey = {}\n\ - AllowedIPs = fd00::2/128, fd00::3/128\n\ - Endpoint = 10.0.0.9:51820\n\ - PersistentKeepalive = 25\n\ - \n\ - [Peer]\n\ - PublicKey = {}\n\ - AllowedIPs = fd00::4/128\n\ - PersistentKeepalive = off\n", - secret.encode().as_str(), - peer_a.encode(), - peer_b.encode(), - ); - - let state = parse_showconf("tsun0", &text).unwrap(); - assert_eq!(state.public_key, secret.public()); - assert_eq!(state.listen_port, 51821); - assert_eq!(state.peers.len(), 2); - - let a = state - .peers - .iter() - .find(|peer| peer.public_key == peer_a) - .unwrap(); - assert_eq!(a.endpoint, Some("10.0.0.9:51820".parse().unwrap())); - assert_eq!(a.allowed_ips.len(), 2); - assert_eq!(a.persistent_keepalive, Some(25)); - - let b = state - .peers - .iter() - .find(|peer| peer.public_key == peer_b) - .unwrap(); - assert_eq!(b.endpoint, None); - assert_eq!(b.persistent_keepalive, None); - } - - #[test] - fn malformed_tool_output_is_an_error_not_a_panic() { - assert!(parse_showconf("tsun0", "").is_err()); - assert!(parse_showconf("tsun0", "[Interface]\nListenPort = nope\n").is_err()); - assert!(parse_showconf("tsun0", "[Peer]\nAllowedIPs = fd00::1/128\n").is_err()); - assert!(parse_showconf("tsun0", "[Interface]\nPrivateKey = zzzz\n").is_err()); - assert!(parse_ip_addresses("1: tsun0 inet6 not-an-address scope global").is_err()); - } - - #[test] - fn interface_addresses_parse_and_skip_kernel_managed_ones() { - let text = "3: tsun0 inet6 fd12:3456::1/128 scope global \\ valid_lft forever\n\ - 3: tsun0 inet6 fd12:3456::/64 scope global \\ valid_lft forever\n\ - 3: tsun0 inet6 fe80::1/64 scope link \\ valid_lft forever\n"; - let addresses = parse_ip_addresses(text).unwrap(); - assert_eq!( - addresses.iter().map(Cidr::to_string).collect::>(), - vec!["fd12:3456::/64".to_string(), "fd12:3456::1/128".to_string()] - ); - } - - #[test] - fn a_rendered_config_round_trips_through_the_parser() { - let (_, config, _) = sample(); - let rendered = config.render(); - let parsed = parse_showconf(&config.name, &rendered).unwrap(); - let mut expected = config.to_state(); - // showconf does not report interface addresses. - expected.addresses.clear(); - assert_eq!(parsed, expected); - } -} diff --git a/src/net.rs b/src/net.rs index 2ac444a..62304d9 100644 --- a/src/net.rs +++ b/src/net.rs @@ -32,7 +32,7 @@ use iroh::{Endpoint, EndpointAddr, EndpointId, RelayMode}; use crate::config::{AgentConfig, TransportPolicy}; use crate::error::{Error, Result}; use crate::identity::DeviceIdentity; -use crate::proto::message::ALPN; +use crate::proto::message::{ALPN, DATA_ALPN}; /// A network path address as reported by iroh. #[derive(Debug, Clone, PartialEq, Eq)] @@ -193,6 +193,10 @@ fn local_addr_string(addr: &iroh::endpoint::LocalTransportAddr) -> Option Result { let mut builder = Endpoint::builder(presets::Minimal) .secret_key(identity.secret_key()) - .alpns(vec![ALPN.to_vec()]); + .alpns(vec![ALPN.to_vec(), DATA_ALPN.to_vec()]); builder = match config.transport { TransportPolicy::LocalOnly => builder diff --git a/src/proto/message.rs b/src/proto/message.rs index c2ecd0c..9439499 100644 --- a/src/proto/message.rs +++ b/src/proto/message.rs @@ -22,6 +22,17 @@ use crate::error::ProtocolError; /// bumping it must not change any existing [`crate::NetworkId`]. pub const ALPN: &[u8] = b"tsunagi/ctrl/1"; +/// ALPN of the tsunagi data plane. +/// +/// Data plane connections are deliberately separate from control plane ones. +/// They carry one IP plugin's packets for one network and nothing else, so a +/// saturated or broken data plane cannot disturb control traffic, and the +/// transport underneath can be replaced without touching the control protocol. +pub const DATA_ALPN: &[u8] = b"tsunagi/data/1"; + +/// Largest plugin protocol identifier accepted when opening a data channel. +pub const MAX_DATA_PROTOCOL_LEN: usize = 32; + /// Control protocol version carried inside the handshake. pub const PROTOCOL_VERSION: u16 = 1; @@ -61,6 +72,26 @@ pub struct Announcement { pub capabilities: Vec, } +/// Opens a data channel, sent by the initiator right after the membership +/// handshake on a [`DATA_ALPN`] connection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataOpen { + /// Which IP plugin's packets this channel will carry. + pub protocol: String, +} + +/// The responder's answer to [`DataOpen`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataOpenAck { + /// Whether the channel was accepted. + /// + /// A channel is refused when the responder has no plugin for that + /// protocol in that network. That is an ordinary outcome, not an error. + pub accepted: bool, + /// Largest datagram the responder is willing to receive, in bytes. + pub max_datagram: u32, +} + /// A control message exchanged after a successful handshake. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] diff --git a/src/proto/mod.rs b/src/proto/mod.rs index 8c36c86..5283380 100644 --- a/src/proto/mod.rs +++ b/src/proto/mod.rs @@ -15,6 +15,13 @@ //! re-checked against the session's network id. //! //! Nothing here adds its own encryption on top of iroh. +//! +//! # The data plane speaks a different protocol +//! +//! IP plugin packets never travel on a control connection. They use their own +//! ALPN, [`message::DATA_ALPN`], with the same membership handshake followed by +//! [`message::DataOpen`]. Keeping them apart is what lets the data plane's +//! transport be replaced without touching anything above. pub mod frame; pub mod handshake; @@ -23,5 +30,6 @@ pub mod message; pub use frame::{read_frame, write_frame}; pub use handshake::{HandshakeOutcome, Role}; pub use message::{ - ALPN, Announcement, AuthProof, ControlMessage, Envelope, Hello, HelloAck, PROTOCOL_VERSION, + ALPN, Announcement, AuthProof, ControlMessage, DATA_ALPN, DataOpen, DataOpenAck, Envelope, + Hello, HelloAck, PROTOCOL_VERSION, }; diff --git a/tests/wireguard.rs b/tests/wireguard.rs index 1bae3b0..57b9f3f 100644 --- a/tests/wireguard.rs +++ b/tests/wireguard.rs @@ -1,66 +1,58 @@ -//! The WireGuard data plane plugin, driven over real iroh connections. +//! The WireGuard data plane, driven over real iroh connections. //! -//! Every agent here is a real agent with a real control plane. Only the part -//! that would change the host's network is substituted: the plugin runs -//! against [`RecordingBackend`], so the suite needs no root and touches no -//! interfaces, while the key handling, the announcements, the derived -//! addressing, the configuration builder and reconciliation are all the real -//! ones. +//! Everything here is real except the packet interface: real agents, real +//! control plane, real iroh data links, real WireGuard handshakes and +//! encryption from `boringtun`. Only the TUN device is in memory, which is why +//! the whole data plane can be tested with no privileges and without touching +//! the host's network. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod common; -use std::net::{IpAddr, SocketAddr}; +use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; use std::time::Duration; -use common::{DEADLINE, config_with, network, settle, wait_event, wait_for_peers, wait_until}; +use bytes::Bytes; +use common::{config_with, network, settle, wait_event, wait_for_peers, wait_until}; use iroh::EndpointId; use tempfile::TempDir; use tsunagi::agent::Event; -use tsunagi::config::AgentConfig; +use tsunagi::dataplane::IpPlugin; use tsunagi::dataplane::wireguard::{ - AdvertisePolicy, Cidr, InterfaceState, PortPolicy, RecordingBackend, WIREGUARD_PROTOCOL, - WgAnnouncement, WgPublicKey, WgSecretKey, WireguardConfig, WireguardPlugin, overlay_address, - overlay_prefix, + MemoryTun, MemoryTunFactory, WIREGUARD_PROTOCOL, WgAnnouncement, WgSecretKey, WireguardConfig, + WireguardPlugin, overlay_address, overlay_prefix, }; -use tsunagi::dataplane::{IpPlugin, PluginCapability, PluginError}; use tsunagi::discovery::SharedMemoryDiscovery; use tsunagi::identity::{NetworkId, NetworkName, NetworkSecret}; use tsunagi::{Agent, NetworkStatus}; -/// An agent, its WireGuard plugin and the backend that plugin drives. +/// An agent with a WireGuard plugin backed by an in-memory packet interface. struct WgAgent { dir: TempDir, agent: Agent, plugin: Arc, - backend: RecordingBackend, + tuns: MemoryTunFactory, } impl WgAgent { - /// Starts an agent whose WireGuard plugin writes into an in-memory backend. - /// - /// Each agent gets its own interface prefix, because the rest of the name - /// is derived from the network id and these all share one host. - async fn spawn(discovery: &SharedMemoryDiscovery, tag: &str, advertise: IpAddr) -> Self { - Self::spawn_with(discovery, tag, advertise, |config| config).await + async fn spawn(discovery: &SharedMemoryDiscovery, tag: &str) -> Self { + Self::spawn_with(discovery, tag, |config| config).await } async fn spawn_with( discovery: &SharedMemoryDiscovery, tag: &str, - advertise: IpAddr, tune: impl FnOnce(WireguardConfig) -> WireguardConfig, ) -> Self { let dir = TempDir::new().unwrap(); - let (agent, plugin, backend) = - Self::open(dir.path(), discovery, tag, advertise, tune).await; + let (agent, plugin, tuns) = Self::open(dir.path(), discovery, tag, tune).await; Self { dir, agent, plugin, - backend, + tuns, } } @@ -68,49 +60,61 @@ impl WgAgent { root: &std::path::Path, discovery: &SharedMemoryDiscovery, tag: &str, - advertise: IpAddr, tune: impl FnOnce(WireguardConfig) -> WireguardConfig, - ) -> (Agent, Arc, RecordingBackend) { - let backend = RecordingBackend::new(); + ) -> (Agent, Arc, MemoryTunFactory) { + let tuns = MemoryTunFactory::new(); let wg = tune( WireguardConfig::new(root.join("wireguard")) .with_interface_prefix(tag) - .with_advertise(AdvertisePolicy::Explicit(vec![advertise])) - .with_ports(PortPolicy::Fixed(51820)) .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), ); - let plugin = WireguardPlugin::open(wg, Arc::new(backend.clone())) + let plugin = WireguardPlugin::open(wg, Arc::new(tuns.clone())) .await .unwrap(); - let config: AgentConfig = - config_with(root, discovery).with_plugin(plugin.clone() as Arc); - let agent = Agent::spawn(config).await.unwrap(); - (agent, plugin, backend) + let agent = Agent::spawn( + config_with(root, discovery).with_plugin(plugin.clone() as Arc), + ) + .await + .unwrap(); + (agent, plugin, tuns) } fn endpoint_id(&self) -> EndpointId { self.agent.endpoint_id() } - /// The interface this agent's plugin owns for a network. - async fn interface(&self, network: NetworkId) -> String { - wait_until("the plugin prepared the network", || async { - self.plugin.overview(network).map(|view| view.interface) + /// This agent's overlay address in a network. + async fn overlay(&self, network: NetworkId) -> Ipv6Addr { + let view = wait_until("the plugin prepared the network", || async { + self.plugin.overview(network) }) - .await + .await; + match view.overlay_address { + IpAddr::V6(addr) => addr, + IpAddr::V4(_) => panic!("the overlay is IPv6"), + } } - /// Waits until the applied configuration has exactly `count` peers. - async fn wait_for_wg_peers(&self, network: NetworkId, count: usize) -> InterfaceState { - let interface = self.interface(network).await; + /// The in-memory packet interface for a network. + async fn tun(&self, network: NetworkId) -> Arc { + let name = wait_until("the packet interface exists", || async { + let view = self.plugin.overview(network)?; + self.tuns.device(&view.interface).map(|_| view.interface) + }) + .await; + self.tuns.device(&name).unwrap() + } + + /// Waits until `count` tunnels have completed a WireGuard handshake. + async fn wait_for_tunnels(&self, network: NetworkId, count: usize) { wait_until( - &format!("{count} WireGuard peers on {interface}"), + &format!("{count} established WireGuard tunnels"), || async { - let state = self.backend.state(&interface)?; - (state.peers.len() == count).then_some(state) + let view = self.plugin.overview(network)?; + (view.established_peers() == count).then_some(()) }, ) - .await + .await; } async fn shutdown(self) -> TempDir { @@ -119,78 +123,158 @@ impl WgAgent { } } -fn addr(text: &str) -> IpAddr { - text.parse().unwrap() -} - -/// The overlay address a peer should have been given, derived independently. -fn expected_allowed_ips(network: NetworkId, key: &WgPublicKey) -> Vec { - vec![Cidr::host(overlay_address(network, key))] +/// Builds a minimal well-formed IPv6 packet. +fn ipv6_packet(source: Ipv6Addr, destination: Ipv6Addr, payload: &[u8]) -> Bytes { + let mut packet = Vec::with_capacity(40 + payload.len()); + packet.push(6 << 4); // version 6 + packet.extend_from_slice(&[0, 0, 0]); // traffic class and flow label + packet.extend_from_slice(&(payload.len() as u16).to_be_bytes()); + packet.push(59); // "no next header" + packet.push(64); // hop limit + packet.extend_from_slice(&source.octets()); + packet.extend_from_slice(&destination.octets()); + packet.extend_from_slice(payload); + Bytes::from(packet) } #[tokio::test] -async fn two_agents_build_each_others_wireguard_configuration() { +async fn two_agents_carry_real_ip_packets_through_a_wireguard_tunnel() { let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("wg-pair"); + let (name, secret) = network("wg-traffic"); - let a = WgAgent::spawn(&discovery, "ta", addr("10.77.0.1")).await; - let b = WgAgent::spawn(&discovery, "tb", addr("10.77.0.2")).await; + let a = WgAgent::spawn(&discovery, "ta").await; + let b = WgAgent::spawn(&discovery, "tb").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); wait_for_peers(&a.agent, network_id, 1).await; - let state_a = a.wait_for_wg_peers(network_id, 1).await; - let state_b = b.wait_for_wg_peers(network_id, 1).await; + // Both tunnels must actually handshake, not merely be configured. + a.wait_for_tunnels(network_id, 1).await; + b.wait_for_tunnels(network_id, 1).await; - let key_a = a.plugin.overview(network_id).unwrap().public_key; - let key_b = b.plugin.overview(network_id).unwrap().public_key; - assert_ne!(key_a, key_b); - - // Each side configured exactly the other, with locally derived AllowedIPs. - assert_eq!(state_a.peers[0].public_key, key_b); + let addr_a = a.overlay(network_id).await; + let addr_b = b.overlay(network_id).await; + assert_ne!(addr_a, addr_b); + // One shared /64, derived by both sides independently. + assert_eq!(addr_a.octets()[0..8], addr_b.octets()[0..8]); assert_eq!( - state_a.peers[0].allowed_ips, - expected_allowed_ips(network_id, &key_b) - ); - assert_eq!(state_b.peers[0].public_key, key_a); - assert_eq!( - state_b.peers[0].allowed_ips, - expected_allowed_ips(network_id, &key_a) + &overlay_prefix(network_id).octets()[0..8], + &addr_a.octets()[0..8] ); - // The endpoint each side uses is the one the *plugin* advertised, not an - // iroh address. - assert_eq!( - state_a.peers[0].endpoint, - Some(SocketAddr::new(addr("10.77.0.2"), 51820)) - ); - let iroh_addrs: Vec = b - .agent - .status() + let tun_a = a.tun(network_id).await; + let tun_b = b.tun(network_id).await; + + // A real IP packet, encrypted by WireGuard, carried over iroh, decrypted + // on the other side and handed to that host's packet interface. + let payload = b"hello over the overlay"; + tun_a.push_from_os(ipv6_packet(addr_a, addr_b, payload)); + + let received = tokio::time::timeout(common::DEADLINE, tun_b.pop_to_os()) .await - .unwrap() - .bound_sockets - .iter() - .map(SocketAddr::ip) - .collect(); - assert!( - !iroh_addrs.contains(&addr("10.77.0.2")), - "the WireGuard endpoint must not come from iroh's addresses" - ); + .expect("the packet should arrive") + .expect("the interface should still be open"); + assert_eq!(&received[40..], payload); + assert_eq!(&received[8..24], &addr_a.octets(), "source preserved"); + assert_eq!(&received[24..40], &addr_b.octets(), "destination preserved"); - // Both derive the same overlay subnet and their own distinct address. - let view_a = a.plugin.overview(network_id).unwrap(); - let view_b = b.plugin.overview(network_id).unwrap(); - assert_eq!(view_a.overlay_prefix, view_b.overlay_prefix); - assert_eq!( - view_a.overlay_prefix, - IpAddr::V6(overlay_prefix(network_id)) - ); - assert_ne!(view_a.overlay_address, view_b.overlay_address); - assert_eq!( - view_a.overlay_address, - IpAddr::V6(overlay_address(network_id, &key_a)) + // And back the other way. + tun_b.push_from_os(ipv6_packet(addr_b, addr_a, b"and back")); + let back = tokio::time::timeout(common::DEADLINE, tun_a.pop_to_os()) + .await + .expect("the reply should arrive") + .unwrap(); + assert_eq!(&back[40..], b"and back"); + + let view = a.plugin.overview(network_id).unwrap(); + let tunnel = view.peers[0].tunnel.as_ref().unwrap(); + assert!(tunnel.health.is_up()); + assert!(tunnel.stats.tx_packets >= 1); + assert!(tunnel.stats.rx_packets >= 1); + assert_eq!(tunnel.stats.dropped_wrong_source, 0); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn a_peer_cannot_send_from_an_address_it_does_not_own() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("wg-spoof"); + + let a = WgAgent::spawn(&discovery, "ta").await; + let b = WgAgent::spawn(&discovery, "tb").await; + + let network_id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + a.wait_for_tunnels(network_id, 1).await; + b.wait_for_tunnels(network_id, 1).await; + + let addr_a = a.overlay(network_id).await; + let addr_b = b.overlay(network_id).await; + let tun_a = a.tun(network_id).await; + let tun_b = b.tun(network_id).await; + + // A sends a packet claiming to come from a third party's address. + let someone_else: Ipv6Addr = { + let mut octets = addr_a.octets(); + octets[15] ^= 0xff; + Ipv6Addr::from(octets) + }; + tun_a.push_from_os(ipv6_packet(someone_else, addr_b, b"spoofed")); + + // B must drop it: the source is not the address derived for A's key. + wait_until("the spoofed packet is dropped", || async { + let view = b.plugin.overview(network_id)?; + let tunnel = view.peers.first()?.tunnel.as_ref()?; + (tunnel.stats.dropped_wrong_source >= 1).then_some(()) + }) + .await; + + // A legitimate packet still goes through, so the tunnel is not broken. + tun_a.push_from_os(ipv6_packet(addr_a, addr_b, b"honest")); + let received = tokio::time::timeout(common::DEADLINE, tun_b.pop_to_os()) + .await + .expect("the honest packet should arrive") + .unwrap(); + assert_eq!(&received[40..], b"honest"); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn packets_for_an_unknown_address_are_counted_not_broadcast() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("wg-unroutable"); + + let a = WgAgent::spawn(&discovery, "ta").await; + let b = WgAgent::spawn(&discovery, "tb").await; + let network_id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + a.wait_for_tunnels(network_id, 1).await; + + let addr_a = a.overlay(network_id).await; + let tun_a = a.tun(network_id).await; + let tun_b = b.tun(network_id).await; + + // Nobody owns this address, so it must not be sent to anybody. + let nowhere: Ipv6Addr = "fd00:dead:beef::1".parse().unwrap(); + tun_a.push_from_os(ipv6_packet(addr_a, nowhere, b"lost")); + + wait_until("the packet is counted as unroutable", || async { + let view = a.plugin.overview(network_id)?; + (view.unroutable_packets >= 1).then_some(()) + }) + .await; + + settle().await; + assert!( + tokio::time::timeout(Duration::from_millis(200), tun_b.pop_to_os()) + .await + .is_err(), + "an unroutable packet must not reach another member" ); a.shutdown().await; @@ -198,13 +282,13 @@ async fn two_agents_build_each_others_wireguard_configuration() { } #[tokio::test] -async fn a_mesh_of_three_gives_every_agent_two_peers() { +async fn a_mesh_of_three_establishes_every_tunnel() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-mesh"); - let a = WgAgent::spawn(&discovery, "ta", addr("10.77.1.1")).await; - let b = WgAgent::spawn(&discovery, "tb", addr("10.77.1.2")).await; - let c = WgAgent::spawn(&discovery, "tc", addr("10.77.1.3")).await; + let a = WgAgent::spawn(&discovery, "ta").await; + let b = WgAgent::spawn(&discovery, "tb").await; + let c = WgAgent::spawn(&discovery, "tc").await; let network_id = a.agent.join_network(&name, &secret).await.unwrap(); b.agent.join_network(&name, &secret).await.unwrap(); @@ -212,37 +296,37 @@ async fn a_mesh_of_three_gives_every_agent_two_peers() { for agent in [&a, &b, &c] { wait_for_peers(&agent.agent, network_id, 2).await; - // N - 1 remote peers in the local configuration. - let state = agent.wait_for_wg_peers(network_id, 2).await; - assert_eq!(state.listen_port, 51820); - let own_key = agent.plugin.overview(network_id).unwrap().public_key; - assert_eq!(state.public_key, own_key); - assert!( - state.peers.iter().all(|peer| peer.public_key != own_key), - "an agent must never configure itself as a peer" - ); + // N - 1 tunnels, all handshaken. + agent.wait_for_tunnels(network_id, 2).await; } - // Everybody agrees on who is in the overlay and at which address. - let keys: Vec = [&a, &b, &c] - .iter() - .map(|agent| agent.plugin.overview(network_id).unwrap().public_key) - .collect(); + // Everyone agrees on the subnet and nobody configured themselves. + let mut addresses = Vec::new(); for agent in [&a, &b, &c] { - let own = agent.plugin.overview(network_id).unwrap().public_key; - let state = agent - .backend - .state(&agent.interface(network_id).await) - .unwrap(); - for peer in &state.peers { - assert!(keys.contains(&peer.public_key)); - assert_eq!( - peer.allowed_ips, - expected_allowed_ips(network_id, &peer.public_key) - ); - assert_ne!(peer.public_key, own); - } + let view = agent.plugin.overview(network_id).unwrap(); + assert_eq!(view.overlay_prefix, IpAddr::V6(overlay_prefix(network_id))); + assert!( + view.peers + .iter() + .all(|peer| peer.public_key != view.public_key) + ); + addresses.push(view.overlay_address); } + addresses.sort(); + addresses.dedup(); + assert_eq!(addresses.len(), 3, "every member has its own address"); + + // A packet from A reaches C directly, not via B. + let addr_a = a.overlay(network_id).await; + let addr_c = c.overlay(network_id).await; + a.tun(network_id) + .await + .push_from_os(ipv6_packet(addr_a, addr_c, b"a to c")); + let received = tokio::time::timeout(common::DEADLINE, c.tun(network_id).await.pop_to_os()) + .await + .expect("the packet should arrive") + .unwrap(); + assert_eq!(&received[40..], b"a to c"); a.shutdown().await; b.shutdown().await; @@ -250,16 +334,16 @@ async fn a_mesh_of_three_gives_every_agent_two_peers() { } #[tokio::test] -async fn a_departing_peer_is_removed_from_the_configuration() { +async fn a_departing_peer_loses_its_tunnel() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-departure"); - let stayer = WgAgent::spawn(&discovery, "ta", addr("10.77.2.1")).await; - let leaver = WgAgent::spawn(&discovery, "tb", addr("10.77.2.2")).await; + let stayer = WgAgent::spawn(&discovery, "ta").await; + let leaver = WgAgent::spawn(&discovery, "tb").await; let network_id = stayer.agent.join_network(&name, &secret).await.unwrap(); leaver.agent.join_network(&name, &secret).await.unwrap(); - stayer.wait_for_wg_peers(network_id, 1).await; + stayer.wait_for_tunnels(network_id, 1).await; let leaver_id = leaver.endpoint_id(); let mut events = stayer.agent.subscribe(); @@ -271,13 +355,13 @@ async fn a_departing_peer_is_removed_from_the_configuration() { }) .await; - // The interface stays, the peer goes. - let state = stayer.wait_for_wg_peers(network_id, 0).await; - assert!(state.peers.is_empty()); - assert_eq!( - state.public_key, - stayer.plugin.overview(network_id).unwrap().public_key - ); + wait_until("the tunnel is removed", || async { + let view = stayer.plugin.overview(network_id)?; + view.peers.is_empty().then_some(()) + }) + .await; + // The interface itself stays; only the peer went. + assert!(stayer.plugin.overview(network_id).is_some()); stayer.shutdown().await; } @@ -288,193 +372,94 @@ async fn two_networks_get_separate_interfaces_keys_and_overlays() { let (name_a, secret_a) = network("wg-left"); let (name_b, secret_b) = network("wg-right"); - let hub = WgAgent::spawn(&discovery, "th", addr("10.77.3.1")).await; - let left = WgAgent::spawn(&discovery, "tl", addr("10.77.3.2")).await; - let right = WgAgent::spawn(&discovery, "tr", addr("10.77.3.3")).await; + let hub = WgAgent::spawn(&discovery, "th").await; + let left = WgAgent::spawn(&discovery, "tl").await; + let right = WgAgent::spawn(&discovery, "tr").await; let alpha = hub.agent.join_network(&name_a, &secret_a).await.unwrap(); let beta = hub.agent.join_network(&name_b, &secret_b).await.unwrap(); left.agent.join_network(&name_a, &secret_a).await.unwrap(); right.agent.join_network(&name_b, &secret_b).await.unwrap(); - hub.wait_for_wg_peers(alpha, 1).await; - hub.wait_for_wg_peers(beta, 1).await; + hub.wait_for_tunnels(alpha, 1).await; + hub.wait_for_tunnels(beta, 1).await; let view_alpha = hub.plugin.overview(alpha).unwrap(); let view_beta = hub.plugin.overview(beta).unwrap(); - assert_ne!(view_alpha.interface, view_beta.interface); assert_ne!( view_alpha.public_key, view_beta.public_key, - "one identity per network, not one per host" + "one WireGuard identity per network, not one per host" ); assert_ne!(view_alpha.overlay_prefix, view_beta.overlay_prefix); - assert_eq!(hub.backend.interfaces().len(), 2); + assert_eq!(hub.tuns.devices().len(), 2); - // Neither interface knows anything about the other network's member. - let left_key = left.plugin.overview(alpha).unwrap().public_key; - let right_key = right.plugin.overview(beta).unwrap().public_key; - let state_alpha = hub.backend.state(&view_alpha.interface).unwrap(); - let state_beta = hub.backend.state(&view_beta.interface).unwrap(); - assert_eq!(state_alpha.peers[0].public_key, left_key); - assert_eq!(state_beta.peers[0].public_key, right_key); - assert!(state_alpha.peers.iter().all(|p| p.public_key != right_key)); - assert!(state_beta.peers.iter().all(|p| p.public_key != left_key)); + // Traffic in one overlay never surfaces in the other. + let hub_alpha = match view_alpha.overlay_address { + IpAddr::V6(addr) => addr, + IpAddr::V4(_) => panic!("ipv6"), + }; + let left_addr = left.overlay(alpha).await; + hub.tun(alpha) + .await + .push_from_os(ipv6_packet(hub_alpha, left_addr, b"alpha only")); + let seen = tokio::time::timeout(common::DEADLINE, left.tun(alpha).await.pop_to_os()) + .await + .expect("the packet should arrive") + .unwrap(); + assert_eq!(&seen[40..], b"alpha only"); + assert!( + tokio::time::timeout( + Duration::from_millis(200), + right.tun(beta).await.pop_to_os() + ) + .await + .is_err(), + "the other overlay must see nothing" + ); // Deactivating one network removes only its interface. hub.agent.deactivate_network(alpha).await.unwrap(); wait_until("the alpha interface is gone", || async { - hub.backend - .state(&view_alpha.interface) - .is_none() - .then_some(()) + hub.plugin.overview(alpha).is_none().then_some(()) }) .await; - assert!(hub.backend.state(&view_beta.interface).is_some()); + assert!(hub.plugin.overview(beta).is_some()); hub.shutdown().await; left.shutdown().await; right.shutdown().await; } -#[tokio::test] -async fn reconciliation_repairs_a_configuration_edited_by_hand() { - let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("wg-drift"); - - let a = WgAgent::spawn(&discovery, "ta", addr("10.77.4.1")).await; - let b = WgAgent::spawn(&discovery, "tb", addr("10.77.4.2")).await; - - let network_id = a.agent.join_network(&name, &secret).await.unwrap(); - b.agent.join_network(&name, &secret).await.unwrap(); - - let good = a.wait_for_wg_peers(network_id, 1).await; - let interface = a.interface(network_id).await; - - // Someone edits the interface: the peer's allowed prefix is widened to - // everything and a bogus address is added. - let mut tampered = good.clone(); - tampered.peers[0].allowed_ips = vec![Cidr::new(addr("::"), 0).unwrap()]; - tampered - .addresses - .push(Cidr::new(addr("192.0.2.1"), 32).unwrap()); - a.backend.inject_drift(&interface, tampered.clone()); - assert_ne!(a.backend.state(&interface).unwrap(), good); - - // The periodic reconcile puts it back without anybody asking. - let repaired = wait_until("the drift is corrected", || async { - let state = a.backend.state(&interface)?; - (state == good).then_some(state) - }) - .await; - assert_eq!(repaired.peers[0].allowed_ips, good.peers[0].allowed_ips); - assert!( - !repaired - .addresses - .contains(&Cidr::new(addr("192.0.2.1"), 32).unwrap()) - ); - - a.shutdown().await; - b.shutdown().await; -} - -#[tokio::test] -async fn a_data_plane_failure_does_not_stop_the_control_plane() { - let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("wg-failure"); - - let a = WgAgent::spawn(&discovery, "ta", addr("10.77.5.1")).await; - let b = WgAgent::spawn(&discovery, "tb", addr("10.77.5.2")).await; - - let mut events = a.agent.subscribe(); - a.backend - .fail_next_apply("simulated: no permission to configure the device"); - - let network_id = a.agent.join_network(&name, &secret).await.unwrap(); - b.agent.join_network(&name, &secret).await.unwrap(); - - // The failure is surfaced on the agent's event stream. - let reason = wait_event(&mut events, |event| match event { - Event::PluginError { - network, - protocol, - reason, - } if *network == network_id && protocol == WIREGUARD_PROTOCOL => Some(reason.clone()), - _ => None, - }) - .await; - assert!(reason.contains("simulated"), "unexpected reason: {reason}"); - - // The control plane is untouched: peers stay authenticated and messages - // keep flowing. - wait_for_peers(&a.agent, network_id, 1).await; - a.agent - .send( - network_id, - b.endpoint_id(), - tsunagi::proto::ControlMessage::Ping { - seq: 1, - payload: b"alive".to_vec(), - }, - ) - .await - .unwrap(); - wait_event(&mut events, |event| match event { - Event::MessageReceived { - message: tsunagi::proto::ControlMessage::Pong { seq: 1, .. }, - .. - } => Some(()), - _ => None, - }) - .await; - - // And the plugin retries, so the interface converges anyway. - a.wait_for_wg_peers(network_id, 1).await; - let status: NetworkStatus = a.agent.network_status(network_id).await.unwrap(); - assert!(status.metrics.plugin_errors >= 1); - - a.shutdown().await; - b.shutdown().await; -} - #[tokio::test] async fn restarting_keeps_the_wireguard_identity_and_overlay_address() { let discovery = SharedMemoryDiscovery::new(); let (name, secret) = network("wg-restart"); - let peer = WgAgent::spawn(&discovery, "tp", addr("10.77.6.9")).await; - let subject = WgAgent::spawn(&discovery, "ts", addr("10.77.6.1")).await; + let peer = WgAgent::spawn(&discovery, "tp").await; + let subject = WgAgent::spawn(&discovery, "ts").await; let network_id = peer.agent.join_network(&name, &secret).await.unwrap(); subject.agent.join_network(&name, &secret).await.unwrap(); - peer.wait_for_wg_peers(network_id, 1).await; + peer.wait_for_tunnels(network_id, 1).await; let before = subject.plugin.overview(network_id).unwrap(); let dir = subject.shutdown().await; - let (agent, plugin, backend) = - WgAgent::open(dir.path(), &discovery, "ts", addr("10.77.6.1"), |config| { - config - }) - .await; + let (agent, plugin, _tuns) = WgAgent::open(dir.path(), &discovery, "ts", |config| config).await; let after = wait_until("the restarted plugin is ready", || async { plugin.overview(network_id) }) .await; - assert_eq!( - after.public_key, before.public_key, - "the WireGuard key must survive a restart" - ); + assert_eq!(after.public_key, before.public_key); assert_eq!(after.overlay_address, before.overlay_address); assert_eq!(after.interface, before.interface); - // The peer reconnects and both configurations come back. - wait_for_peers(&agent, network_id, 1).await; - wait_until("the restarted interface has its peer", || async { - backend - .state(&after.interface) - .filter(|state| state.peers.len() == 1) + // The tunnel comes back on its own. + wait_until("the tunnel is re-established", || async { + let view = plugin.overview(network_id)?; + (view.established_peers() == 1).then_some(()) }) .await; @@ -490,86 +475,64 @@ async fn shutdown_removes_every_interface_the_plugin_created() { let (name_a, secret_a) = network("wg-teardown-a"); let (name_b, secret_b) = network("wg-teardown-b"); - let agent = WgAgent::spawn(&discovery, "ta", addr("10.77.7.1")).await; + let agent = WgAgent::spawn(&discovery, "ta").await; let alpha = agent.agent.join_network(&name_a, &secret_a).await.unwrap(); let beta = agent.agent.join_network(&name_b, &secret_b).await.unwrap(); - agent.interface(alpha).await; - agent.interface(beta).await; - wait_until("both interfaces exist", || async { - (agent.backend.interfaces().len() == 2).then_some(()) - }) - .await; + agent.tun(alpha).await; + agent.tun(beta).await; agent.agent.shutdown().await; - assert!( - agent.backend.interfaces().is_empty(), - "a clean shutdown must not leave interfaces behind: {:?}", - agent.backend.interfaces() - ); -} - -/// A plugin that announces whatever bytes it is told to, under the WireGuard -/// protocol id. Used to test what a hostile member can do. -#[derive(Debug)] -struct ForgingPlugin { - payload: std::sync::Mutex>>, -} - -impl IpPlugin for ForgingPlugin { - fn protocol_id(&self) -> &str { - WIREGUARD_PROTOCOL - } - - fn local_capability( - &self, - _network: NetworkId, - ) -> Result, PluginError> { - let payload = match self.payload.lock() { - Ok(guard) => guard.clone(), - Err(poisoned) => poisoned.into_inner().clone(), - }; - Ok(payload.map(|data| PluginCapability { - protocol: WIREGUARD_PROTOCOL.to_string(), - version: 1, - enabled: true, - data, - })) - } - - fn on_peer_capability( - &self, - _network: NetworkId, - _peer: EndpointId, - _capability: &PluginCapability, - ) -> Result<(), PluginError> { - Ok(()) - } - - fn on_peer_gone(&self, _network: NetworkId, _peer: EndpointId) {} - fn on_network_deactivated(&self, _network: NetworkId) {} + assert!(agent.plugin.overview(alpha).is_none()); + assert!(agent.plugin.overview(beta).is_none()); } #[tokio::test] -async fn a_member_cannot_claim_another_members_overlay_address() { +async fn the_core_carries_the_payload_without_interpreting_it() { + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("wg-opaque"); + + let a = WgAgent::spawn(&discovery, "ta").await; + let b = WgAgent::spawn(&discovery, "tb").await; + let network_id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&a.agent, network_id, 1).await; + + let capability = wait_until("the peer's capability arrived", || async { + let status: NetworkStatus = a.agent.network_status(network_id).await.ok()?; + status + .peers + .first() + .and_then(|peer| peer.capabilities.first().cloned()) + }) + .await; + assert_eq!(capability.protocol, WIREGUARD_PROTOCOL); + + let view_b = b.plugin.overview(network_id).unwrap(); + let expected = WgAnnouncement::new(network_id, &view_b.public_key) + .encode() + .unwrap(); + assert_eq!(capability.data, expected); + assert!(capability.data.len() < tsunagi::Limits::default().max_capability_data_len); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn a_forged_overlay_claim_is_rejected_and_never_reaches_a_tunnel() { let discovery = SharedMemoryDiscovery::new(); let name = NetworkName::new("wg-hijack").unwrap(); let secret = NetworkSecret::generate(); - let victim = WgAgent::spawn(&discovery, "tv", addr("10.77.8.1")).await; + let victim = WgAgent::spawn(&discovery, "tv").await; let network_id = victim.agent.join_network(&name, &secret).await.unwrap(); - let victim_view = wait_until("the victim is ready", || async { - victim.plugin.overview(network_id) - }) - .await; + let victim_address = victim.overlay(network_id).await; - // The attacker knows the secret — it is a legitimate member — and tries to - // take over the victim's overlay address with its own WireGuard key. + // A legitimate member — it knows the secret — claims the victim's overlay + // address with its own WireGuard key. let attacker_key = WgSecretKey::generate().public(); - let mut forged = WgAnnouncement::new(network_id, &attacker_key, 51820, Vec::new()); - let IpAddr::V6(victim_address) = victim_view.overlay_address else { - panic!("overlay addresses are IPv6"); - }; + let mut forged = WgAnnouncement::new(network_id, &attacker_key); forged.overlay_address = victim_address; let forger = Arc::new(ForgingPlugin { @@ -587,7 +550,6 @@ async fn a_member_cannot_claim_another_members_overlay_address() { attacker.join_network(&name, &secret).await.unwrap(); wait_for_peers(&victim.agent, network_id, 1).await; - // The victim rejects the claim and says why. let reason = wait_event(&mut events, |event| match event { Event::PluginError { protocol, reason, .. @@ -595,130 +557,72 @@ async fn a_member_cannot_claim_another_members_overlay_address() { _ => None, }) .await; - assert!( - reason.contains("does not match"), - "unexpected reason: {reason}" - ); + assert!(reason.contains("does not match"), "unexpected: {reason}"); - // Nothing was configured for the attacker, and the victim keeps its own - // address. settle().await; - let interface = victim.interface(network_id).await; - let state = victim.backend.state(&interface); + let view = victim.plugin.overview(network_id).unwrap(); assert!( - state - .map(|state| state - .peers - .iter() - .all(|peer| peer.public_key != attacker_key)) - .unwrap_or(true), - "a rejected announcement must not reach the configuration" - ); - assert_eq!( - victim.plugin.overview(network_id).unwrap().overlay_address, - victim_view.overlay_address + view.peers + .iter() + .all(|peer| peer.public_key != attacker_key), + "a rejected announcement must never become a tunnel" ); + assert_eq!(view.overlay_address, IpAddr::V6(victim_address)); attacker.shutdown().await; victim.shutdown().await; drop(attacker_dir); } -#[tokio::test] -async fn the_core_carries_the_payload_without_interpreting_it() { - let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("wg-opaque"); - - let a = WgAgent::spawn(&discovery, "ta", addr("10.77.9.1")).await; - let b = WgAgent::spawn(&discovery, "tb", addr("10.77.9.2")).await; - - let network_id = a.agent.join_network(&name, &secret).await.unwrap(); - b.agent.join_network(&name, &secret).await.unwrap(); - a.wait_for_wg_peers(network_id, 1).await; - - // What the control plane stored for the peer is exactly the bytes the - // peer's plugin produced, and the core never looked inside. - let capability = wait_until("the peer's capability arrived", || async { - let status = a.agent.network_status(network_id).await.ok()?; - status - .peers - .first() - .and_then(|peer| peer.capabilities.first().cloned()) - }) - .await; - assert_eq!(capability.protocol, WIREGUARD_PROTOCOL); - assert!(capability.enabled); - - let view_b = b.plugin.overview(network_id).unwrap(); - let expected = WgAnnouncement::new( - network_id, - &view_b.public_key, - view_b.listen_port, - view_b.advertised.clone(), - ) - .encode() - .unwrap(); - assert_eq!(capability.data, expected); - - // And the payload stays well inside the control protocol's bound. - assert!(capability.data.len() < tsunagi::Limits::default().max_capability_data_len); - - a.shutdown().await; - b.shutdown().await; +/// A plugin that announces whatever bytes it is told to, under the WireGuard +/// protocol id. Used to test what a hostile member can do. +#[derive(Debug)] +struct ForgingPlugin { + payload: std::sync::Mutex>>, } -#[tokio::test] -async fn local_interface_advertising_produces_usable_endpoints() { - let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("wg-advertise"); - - let agent = WgAgent::spawn_with(&discovery, "ta", addr("10.77.10.1"), |config| { - config.with_advertise(AdvertisePolicy::LocalInterfaces) - }) - .await; - let network_id = agent.agent.join_network(&name, &secret).await.unwrap(); - - let view = wait_until("the plugin is ready", || async { - agent.plugin.overview(network_id) - }) - .await; - - // Whatever this host has, nothing loopback, unspecified or link-local may - // be advertised, and every entry carries the WireGuard port. - for endpoint in &view.advertised { - assert_eq!(endpoint.port(), view.listen_port); - assert!(!endpoint.ip().is_loopback()); - assert!(!endpoint.ip().is_unspecified()); +impl IpPlugin for ForgingPlugin { + fn protocol_id(&self) -> &str { + WIREGUARD_PROTOCOL } - assert!(view.advertised.len() <= 8, "the list stays bounded"); - agent.shutdown().await; + fn local_capability( + &self, + _network: NetworkId, + ) -> Result, tsunagi::dataplane::PluginError> { + let payload = match self.payload.lock() { + Ok(guard) => guard.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + }; + Ok(payload.map(|data| tsunagi::dataplane::PluginCapability { + protocol: WIREGUARD_PROTOCOL.to_string(), + version: 1, + enabled: true, + data, + })) + } + + fn on_peer_capability( + &self, + _network: NetworkId, + _peer: EndpointId, + _capability: &tsunagi::dataplane::PluginCapability, + ) -> Result<(), tsunagi::dataplane::PluginError> { + Ok(()) + } + + fn on_peer_gone(&self, _network: NetworkId, _peer: EndpointId) {} + fn on_network_deactivated(&self, _network: NetworkId) {} } #[tokio::test] -async fn the_plugin_converges_within_the_test_deadline() { - // A guard against the announce/re-announce handshake regressing into - // something that only converges on the slow periodic timer. - let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("wg-timing"); - - let a = WgAgent::spawn_with(&discovery, "ta", addr("10.77.11.1"), |config| { - // A deliberately slow periodic reconcile: convergence must come from - // the event path, not from the timer. - config.with_reconcile(Duration::from_millis(20), DEADLINE * 2) - }) - .await; - let b = WgAgent::spawn_with(&discovery, "tb", addr("10.77.11.2"), |config| { - config.with_reconcile(Duration::from_millis(20), DEADLINE * 2) - }) - .await; - - let network_id = a.agent.join_network(&name, &secret).await.unwrap(); - b.agent.join_network(&name, &secret).await.unwrap(); - - a.wait_for_wg_peers(network_id, 1).await; - b.wait_for_wg_peers(network_id, 1).await; - - a.shutdown().await; - b.shutdown().await; +async fn the_overlay_address_is_derived_from_the_key_alone() { + let (name, secret) = network("wg-derivation"); + let id = tsunagi::identity::NetworkKeys::derive(&name, &secret).network_id(); + let key = WgSecretKey::generate().public(); + assert_eq!(overlay_address(id, &key), overlay_address(id, &key)); + assert_ne!( + overlay_address(id, &key), + overlay_address(id, &WgSecretKey::generate().public()) + ); } diff --git a/tests/wireguard_system.rs b/tests/wireguard_system.rs deleted file mode 100644 index 0b9f18e..0000000 --- a/tests/wireguard_system.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! Opt-in tests for the real `wg` / `ip` backend. -//! -//! These are the only tests in this repository that change the host's network, -//! so they are **ignored by default** and are not part of the standard suite. -//! -//! They need Linux, the `wireguard` kernel module, `wg` from wireguard-tools, -//! `ip` from iproute2, and `CAP_NET_ADMIN` (in practice: root): -//! -//! ```text -//! sudo -E cargo test --test wireguard_system -- --ignored --test-threads=1 -//! ``` -//! -//! Everything they exercise — the command plan, the parsers, the configuration -//! builder and reconciliation — is already covered without root by the unit -//! tests in `src/dataplane/wireguard/` and by `tests/wireguard.rs`. What these -//! add is confirmation that the plan the planner produces is one the real -//! tools accept. - -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::net::SocketAddr; - -use tsunagi::dataplane::wireguard::wgtool::WgToolBackend; -use tsunagi::dataplane::wireguard::{ - Cidr, InterfaceParams, WgSecretKey, WireguardBackend, build_interface, overlay_address, -}; -use tsunagi::identity::{NetworkKeys, NetworkName, NetworkSecret}; - -/// A name unlikely to collide with anything already on the host. -const TEST_INTERFACE: &str = "tsunagitest0"; - -fn test_network() -> tsunagi::NetworkId { - NetworkKeys::derive( - &NetworkName::new("system-test").unwrap(), - &NetworkSecret::from_bytes(vec![11u8; 32]).unwrap(), - ) - .network_id() -} - -#[test] -#[ignore = "changes the host's network; needs Linux, wg, ip and CAP_NET_ADMIN"] -fn the_real_backend_applies_reads_back_and_removes_a_configuration() { - let backend = WgToolBackend::new().expect("the wg-tools backend should be available on Linux"); - let network = test_network(); - let local = WgSecretKey::generate(); - let peer = WgSecretKey::generate().public(); - - // Start from a clean slate even if a previous run was interrupted. - backend.remove(TEST_INTERFACE).unwrap(); - assert_eq!(backend.inspect(TEST_INTERFACE).unwrap(), None); - - let desired = build_interface( - InterfaceParams { - network, - name: TEST_INTERFACE.to_string(), - private_key: local.clone(), - listen_port: 51899, - mtu: Some(1380), - keepalive: Some(25), - }, - [peer], - |_| Some("10.99.0.2:51899".parse::().unwrap()), - ); - - backend.apply(&desired).unwrap(); - - let observed = backend - .inspect(TEST_INTERFACE) - .unwrap() - .expect("the interface should exist after apply"); - assert_eq!(observed.public_key, local.public()); - assert_eq!(observed.listen_port, 51899); - assert_eq!(observed.peers.len(), 1); - assert_eq!(observed.peers[0].public_key, peer); - assert_eq!( - observed.peers[0].allowed_ips, - vec![Cidr::host(overlay_address(network, &peer))] - ); - assert!( - observed - .addresses - .contains(&Cidr::host(overlay_address(network, &local.public()))), - "the overlay address should be assigned: {:?}", - observed.addresses - ); - - // Applying the same configuration again must be a no-op, not a rebuild. - backend.apply(&desired).unwrap(); - assert_eq!(backend.inspect(TEST_INTERFACE).unwrap(), Some(observed)); - - backend.remove(TEST_INTERFACE).unwrap(); - assert_eq!(backend.inspect(TEST_INTERFACE).unwrap(), None); - // Removing something that is not there is not an error. - backend.remove(TEST_INTERFACE).unwrap(); -} - -#[test] -#[ignore = "changes the host's network; needs Linux, wg, ip and CAP_NET_ADMIN"] -fn the_real_backend_refuses_an_interface_it_did_not_create() { - let backend = WgToolBackend::new().expect("the wg-tools backend should be available on Linux"); - let name = "tsunagitest1"; - - // A plain dummy link, not a WireGuard device: something that belongs to - // somebody else. - let created = std::process::Command::new("ip") - .args(["link", "add", "dev", name, "type", "dummy"]) - .status() - .expect("ip should be available"); - assert!(created.success(), "could not create the dummy link"); - - let result = backend.inspect(name); - let _ = std::process::Command::new("ip") - .args(["link", "del", "dev", name]) - .status(); - - assert!( - result.is_err(), - "an interface the plugin did not create must be refused, not adopted" - ); -}