From 2166ba480015488847b67ad13bd57141f6a003d8 Mon Sep 17 00:00:00 2001 From: Aleksandr Bogomiakov Date: Thu, 22 Oct 2026 01:07:52 +0100 Subject: [PATCH] Separate internal WireGuard encryption from data carriers AWG-only policy also selected a competing end-to-end plugin, so compatible external links could still have no compatible encrypted data channel. Carrier changes replaced tunnel state and status selected counters from different engines. Use one internal WireGuard plugin and a separate DataCarrier contract for external capabilities/lifecycle. Keep the logical link, keys, handshake, interface and counter epoch across policy, fallback and direct/relay changes. Control/discovery transport selection and membership remain independent. Share the bounded per-neighbour UDP WireGuard engine between ordinary WG and Amnezia masking. Encrypt the relay envelope inside the outer hop channel with fresh pair keys/PSK. Remove duplicate WG key/device code and the TCP/TLS inner cipher. TLS carries opaque internal WireGuard frames. Refuse missing native transports instead of letting default iroh impersonate their protocol names. Probe the actual authenticated carrier, independently of the shared IP tunnel; failed local probe submissions count as loss too. Apply the vendored iroh interface exclusion to actual direct candidates, not just public discovery records. Exclude overlay subnets from introductions and propagate the actual owned interface name, including kernel-assigned utun names, to every carrier. Production logs showed paths through the overlay itself. This removes a recursive transport path; it does not establish the historical cause of the reported 100 GB absolute counter. Exit-node traffic is also real peer IP traffic even when local applications are idle. Hide inner cipher choices/handshake clutter in CLI and tray. Report external routes, readiness and actionable policy/version failures. Count original IP packets only when their ciphertext is accepted by the logical transport, including queued packets' exact unpadded lengths. Exclude physical probes and keepalives. Export counter epochs; sum per-peer rate deltas and discard stale histories so joins/restarts cannot turn old totals into fresh traffic. Status also prints exact IP byte totals for comparison with the tray. Coordinated upgrade: data ALPN 6, internal wireguard-e2e capability v4, wg/awg v2, QUIC carrier v1, TLS v2, local IPC v23. Control ALPN 8, device/network identity derivation, signed records and saved network choices stay unchanged. Keep the historical wg-quic key store; transactionally import missing keys from wg then awg without replacing canonical keys. Forget removes legacy copies too, on the blocking pool, to prevent resurrection after restart. Preserve historic cipher options as shared settings, rejecting contradictory values. Validation on Linux in the Nix development shell: - cargo fmt --all -- --check - cargo clippy --locked --workspace --all-targets -- -D warnings - cargo test --locked --workspace --all-targets (562 passed, 3 ignored) The inherited service socket/secret/group environment overrides were removed for isolated CLI process tests. A new carrier test initially raced the first peer announcement; its deadline polling now waits for a peer row explicitly. Real loopback endpoints, membership, SQLite, WG/AWG/TLS encryption and memory TUNs cover mixed-carrier relaying and one smoltcp TCP connection surviving relay/direct/policy changes and a dropped transmission. Counter tests check exact plaintext lengths, refused sends, epochs, joins and stale histories. Public DHT, privileged kernel and manual benchmark tests remain ignored; macOS/Windows native TUNs, desktop GUI and WAN/DPI behavior were not run. --- AGENTS.md | 2 +- Cargo.lock | 109 +- README.md | 11 +- crates/tsunagi-awg/Cargo.toml | 18 +- crates/tsunagi-awg/UPSTREAM.md | 6 +- crates/tsunagi-awg/src/lib.rs | 21 +- crates/tsunagi-awg/src/plugin.rs | 132 -- crates/tsunagi-awg/src/transport.rs | 720 -------- crates/tsunagi-awg/tests/common/tcp.rs | 221 +++ crates/tsunagi-awg/tests/pairs.rs | 260 ++- crates/tsunagi-core/src/agent/mod.rs | 81 +- crates/tsunagi-core/src/agent/network.rs | 317 ++-- crates/tsunagi-core/src/agent/status.rs | 2 + crates/tsunagi-core/src/config.rs | 27 +- crates/tsunagi-core/src/dataplane/mod.rs | 34 +- .../src/dataplane/transport/carrier.rs | 44 + .../src/dataplane/transport/health.rs | 74 +- .../src/dataplane/transport/iroh_link.rs | 56 +- .../src/dataplane/transport/mod.rs | 3 + crates/tsunagi-core/src/dataplane/tunnel.rs | 380 ----- crates/tsunagi-core/src/ipc/mod.rs | 8 +- crates/tsunagi-core/src/net.rs | 37 +- crates/tsunagi-core/src/overlay/router.rs | 18 - crates/tsunagi-core/src/proto/message.rs | 6 +- crates/tsunagi-core/tests/iroh_candidates.rs | 135 ++ crates/tsunagi-core/tests/mixed_transports.rs | 221 +-- crates/tsunagi-core/tests/multi_peer.rs | 4 +- crates/tsunagi-tcp-tls/Cargo.toml | 11 +- crates/tsunagi-tcp-tls/src/crypto.rs | 104 -- crates/tsunagi-tcp-tls/src/lib.rs | 7 +- crates/tsunagi-tcp-tls/src/plugin.rs | 465 ++--- crates/tsunagi-tcp-tls/src/transport.rs | 11 + crates/tsunagi-tcp-tls/tests/crypto_tests.rs | 73 - crates/tsunagi-tcp-tls/tests/plugin_tests.rs | 170 +- crates/tsunagi-tray/src/devices.rs | 140 +- crates/tsunagi-tray/src/exit.rs | 1 + crates/tsunagi-tray/src/format.rs | 13 - crates/tsunagi-tray/src/stats.rs | 213 ++- crates/tsunagi-tray/src/ui.rs | 2 +- crates/tsunagi-wg-quic/Cargo.toml | 4 +- crates/tsunagi-wg-quic/src/device.rs | 77 +- crates/tsunagi-wg-quic/src/plugin.rs | 63 +- crates/tsunagi-wg-quic/src/store.rs | 109 ++ crates/tsunagi-wg-quic/tests/carriers.rs | 285 ++++ crates/tsunagi-wg-quic/tests/exit_node.rs | 5 +- .../tests/interface_provisioning.rs | 5 +- crates/tsunagi-wg-quic/tests/local_control.rs | 11 +- crates/tsunagi-wg-quic/tests/wireguard.rs | 113 +- crates/tsunagi-wireguard/Cargo.toml | 10 +- crates/tsunagi-wireguard/LICENSE-AmneziaWG-RS | 201 +++ crates/tsunagi-wireguard/src/addresses.rs | 168 ++ crates/tsunagi-wireguard/src/announcement.rs | 142 -- crates/tsunagi-wireguard/src/device.rs | 686 -------- crates/tsunagi-wireguard/src/keys.rs | 160 -- crates/tsunagi-wireguard/src/lib.rs | 45 +- .../src/masking.rs | 0 crates/tsunagi-wireguard/src/plugin.rs | 1061 +----------- .../src/profile.rs | 30 +- .../src/session.rs | 22 +- crates/tsunagi-wireguard/src/store.rs | 160 -- .../src/stun.rs | 0 crates/tsunagi-wireguard/src/transport.rs | 1501 ++++++++--------- .../tests/announcement_tests.rs | 86 - .../tsunagi-wireguard/tests/plugin_tests.rs | 378 ----- .../tests/transport_tests.rs | 290 +--- crates/tsunagi/src/main.rs | 615 +++---- docs/amnezia.md | 43 +- docs/architecture.md | 21 +- docs/protocol.md | 8 +- docs/routing.md | 24 +- docs/testing.md | 26 +- docs/threat-model.md | 6 +- docs/wireguard.md | 159 +- vendor/iroh/TSUNAGI-PATCH.md | 17 + vendor/iroh/src/endpoint.rs | 10 + vendor/iroh/src/socket.rs | 16 +- 76 files changed, 3903 insertions(+), 6811 deletions(-) delete mode 100644 crates/tsunagi-awg/src/plugin.rs delete mode 100644 crates/tsunagi-awg/src/transport.rs create mode 100644 crates/tsunagi-awg/tests/common/tcp.rs create mode 100644 crates/tsunagi-core/src/dataplane/transport/carrier.rs delete mode 100644 crates/tsunagi-core/src/dataplane/tunnel.rs create mode 100644 crates/tsunagi-core/tests/iroh_candidates.rs delete mode 100644 crates/tsunagi-tcp-tls/src/crypto.rs delete mode 100644 crates/tsunagi-tcp-tls/tests/crypto_tests.rs create mode 100644 crates/tsunagi-wg-quic/tests/carriers.rs create mode 100644 crates/tsunagi-wireguard/LICENSE-AmneziaWG-RS create mode 100644 crates/tsunagi-wireguard/src/addresses.rs delete mode 100644 crates/tsunagi-wireguard/src/announcement.rs delete mode 100644 crates/tsunagi-wireguard/src/device.rs delete mode 100644 crates/tsunagi-wireguard/src/keys.rs rename crates/{tsunagi-awg => tsunagi-wireguard}/src/masking.rs (100%) rename crates/{tsunagi-awg => tsunagi-wireguard}/src/profile.rs (91%) rename crates/{tsunagi-awg => tsunagi-wireguard}/src/session.rs (95%) delete mode 100644 crates/tsunagi-wireguard/src/store.rs rename crates/{tsunagi-awg => tsunagi-wireguard}/src/stun.rs (100%) delete mode 100644 crates/tsunagi-wireguard/tests/announcement_tests.rs delete mode 100644 crates/tsunagi-wireguard/tests/plugin_tests.rs diff --git a/AGENTS.md b/AGENTS.md index d40fa5b..937f3f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,7 +162,7 @@ Keep these separate. Crossing them is the main thing to review for. | `crates/tsunagi-core/src/overlay/` | the one interface an agent owns: provisioning, the TUN, routing, source checks, broadcast host rules | | `crates/tsunagi-core/src/dataplane/` | the protocol contract and the authenticated packet transport | | `crates/tsunagi-core/src/dns/` | the DNS view of a network: zone, server, resolver publication | -| `crates/tsunagi-wg-quic/` | the `wg-quic` protocol: its keys, its announcement, its tunnels | +| `crates/tsunagi-wg-quic/` | the single internal WireGuard cipher, keys and opaque announcement; historical crate name | | `crates/tsunagi/` | the command line agent, built as `tsng`; the only place that owns a runtime, a logger and signals | | `tests/` | integration tests; `tests/common/` is the shared harness | diff --git a/Cargo.lock b/Cargo.lock index f12f337..0a9a83e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -601,6 +601,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "byteorder-lite" version = "0.1.0" @@ -1171,6 +1177,46 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "defmt" +version = "0.3.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4deef977660e373c458df605a913d4ab121db0ccd18d1ad6976dff7d37cddb53" +dependencies = [ + "defmt 1.2.0", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror 2.0.20", +] + [[package]] name = "der" version = "0.8.2" @@ -2366,6 +2412,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2404,6 +2459,16 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -3315,6 +3380,12 @@ dependencies = [ "libc", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "matchers" version = "0.2.0" @@ -5516,6 +5587,20 @@ dependencies = [ "serde", ] +[[package]] +name = "smoltcp" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad095989c1533c1c266d9b1e8d70a1329dd3723c3edac6d03bbd67e7bf6f4bb" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "cfg-if", + "defmt 0.3.101", + "heapless", + "managed", +] + [[package]] name = "socket2" version = "0.6.5" @@ -6257,20 +6342,15 @@ dependencies = [ name = "tsunagi-awg" version = "0.1.0" dependencies = [ - "boringtun", "bytes", - "chacha20 0.9.1", "iroh", - "postcard", "rand 0.10.3", - "serde", + "smoltcp", "tempfile", "tokio", - "tracing", "tsunagi-core", "tsunagi-wg-quic", "tsunagi-wireguard", - "zeroize", ] [[package]] @@ -6321,27 +6401,18 @@ version = "0.1.0" dependencies = [ "bytes", "caps", - "chacha20poly1305", - "hex", - "hkdf", "iroh", "netdev 0.46.3", "postcard", - "rand 0.10.3", "rcgen", "rustls", "serde", - "sha2", - "subtle", "tempfile", - "thiserror 2.0.20", "tokio", "tokio-rustls", "tracing", "tracing-subscriber", "tsunagi-core", - "x25519-dalek", - "zeroize", ] [[package]] @@ -6381,6 +6452,8 @@ dependencies = [ "tracing", "tracing-subscriber", "tsunagi-core", + "tsunagi-tcp-tls", + "tsunagi-wireguard", "zeroize", ] @@ -6390,17 +6463,13 @@ version = "0.1.0" dependencies = [ "boringtun", "bytes", - "data-encoding", - "hex", + "chacha20 0.9.1", "iroh", "netdev 0.46.3", "postcard", "rand 0.10.3", - "rusqlite", "serde", - "subtle", "tempfile", - "thiserror 2.0.20", "tokio", "tracing", "tracing-subscriber", diff --git a/README.md b/README.md index 37ee180..f32595d 100644 --- a/README.md +++ b/README.md @@ -136,9 +136,14 @@ tsng protocols home none # control plane only Exact network names, full ids and unique id prefixes work here. The choice is local to this device, saved per network and applied immediately; an offline -change applies on the next start. In the tray, expand **protocols** on a network -tile to toggle protocols or change their preference order. Both endpoints need -a common protocol at the same wire version. `tsng up --protocol` supplies the +change applies on the next start. In the tray, expand **data transports** on a network +tile to toggle protocols or change their preference order. Adjacent peers need a common external transport at the same wire version. +Selection applies to this device's incoming and outgoing data links; control and +discovery are independent. A relay's next hop may use a different transport. +Every pair uses one internal WireGuard cipher, hidden from these controls. +Changing transport or moving direct → relay → direct preserves that channel, +addresses and interface. An existing TCP connection can resume after temporary +loss, subject to its own timeouts. `tsng up --protocol` supplies the initial defaults for networks without a saved choice; it does not restrict which built-in protocols can be enabled later. diff --git a/crates/tsunagi-awg/Cargo.toml b/crates/tsunagi-awg/Cargo.toml index 7522090..b812757 100644 --- a/crates/tsunagi-awg/Cargo.toml +++ b/crates/tsunagi-awg/Cargo.toml @@ -8,23 +8,17 @@ license = "WTFPL AND Apache-2.0" repository.workspace = true [dependencies] -tsunagi-core = { path = "../tsunagi-core" } -tsunagi-wg-quic = { path = "../tsunagi-wg-quic" } tsunagi-wireguard = { path = "../tsunagi-wireguard" } -iroh.workspace = true -tokio = { workspace = true, features = ["net", "time", "sync", "macros"] } -serde.workspace = true -postcard.workspace = true -bytes.workspace = true -tracing.workspace = true -rand = "0.10" -zeroize = { version = "1.9", features = ["derive"] } -boringtun = { version = "0.7.1", default-features = false } -chacha20 = "0.9" [dev-dependencies] +tsunagi-wg-quic = { path = "../tsunagi-wg-quic" } +iroh.workspace = true +tokio = { workspace = true, features = ["net", "time", "sync", "macros"] } +bytes.workspace = true +rand = "0.10" tsunagi-core = { path = "../tsunagi-core", features = ["testing"] } tempfile.workspace = true +smoltcp = { version = "0.12", default-features = false, features = ["std", "medium-ip", "proto-ipv4", "socket-tcp"] } [lints] workspace = true diff --git a/crates/tsunagi-awg/UPSTREAM.md b/crates/tsunagi-awg/UPSTREAM.md index d5a16c3..dc1e5b2 100644 --- a/crates/tsunagi-awg/UPSTREAM.md +++ b/crates/tsunagi-awg/UPSTREAM.md @@ -1,9 +1,9 @@ # Amnezia packet engine -`src/masking.rs` is adapted from `src/core/amnezia.rs` in +`../tsunagi-wireguard/src/masking.rs` is adapted from `src/core/amnezia.rs` in https://github.com/RunaRoo/AmneziaWG-RS at `51579a7d6e8882d1d8d6d00822f09e18a9423b73`, under Apache-2.0. -The upstream license is retained in `LICENSE-AmneziaWG-RS`. +The upstream license is retained in both packages as `LICENSE-AmneziaWG-RS`. The adaptation replaces configuration-file loading with bounded typed pair profiles, updates RNG calls to rand 0.10, removes an unnecessary unwrap, and @@ -11,7 +11,7 @@ does not embed upstream's Node, global executors, OS interface operations or detached tasks. Noise/WireGuard is supplied by boringtun, already used by Tsunagi. Each pair owns one socket and one cancellable packet-engine task. -This is Tsunagi AWG wire version 1, not a claim of compatibility with external +This is Tsunagi AWG wire version 2, not a claim of compatibility with external Amnezia servers. The masking engine substitutes message headers outside the standard WireGuard state machine and restores them before Noise validation. There are no remote file paths, shell hooks, or persisted pair profiles. diff --git a/crates/tsunagi-awg/src/lib.rs b/crates/tsunagi-awg/src/lib.rs index 54f842a..d5a880f 100644 --- a/crates/tsunagi-awg/src/lib.rs +++ b/crates/tsunagi-awg/src/lib.rs @@ -1,16 +1,5 @@ -//! Pairwise, ephemeral Amnezia packet transport with independent end-to-end tunnels. -//! All negotiation travels over authenticated Tsunagi control sessions. -mod masking; -mod plugin; -pub mod profile; -mod session; -mod stun; -mod transport; - -pub use plugin::AwgPlugin; -pub use transport::{AwgTransport, PairOverview}; - -/// Protocol identifier and default second-choice direct transport. -pub const AWG_PROTOCOL: &str = "awg"; -/// Tsunagi AWG wire version, independent of software releases. -pub const AWG_WIRE_VERSION: u16 = 1; +//! Amnezia-masked UDP carrier using the independent per-neighbour WireGuard engine. +pub use tsunagi_wireguard::{ + AWG_PROTOCOL, AWG_WIRE_VERSION, PairOverview, UdpCarrier as AwgPlugin, + UdpTransport as AwgTransport, profile, +}; diff --git a/crates/tsunagi-awg/src/plugin.rs b/crates/tsunagi-awg/src/plugin.rs deleted file mode 100644 index 6733e8e..0000000 --- a/crates/tsunagi-awg/src/plugin.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! End-to-end encryption remains independent of physical Amnezia pair links. -use crate::{AWG_PROTOCOL, AWG_WIRE_VERSION, AwgTransport}; -use iroh::EndpointId; -use std::sync::Arc; -use tsunagi::dataplane::transport::SharedLink; -use tsunagi::dataplane::{IpPlugin, PluginCapability, PluginContext, PluginError, ProtocolOption}; -use tsunagi::state::Ipv4Range; -use tsunagi::{BoxFuture, NetworkId}; -use tsunagi_wg_quic::{NetworkOverview, WireguardConfig, WireguardPlugin}; - -/// AWG capability and an independent end-to-end WireGuard engine. -/// Pair transport keys/profiles never enter its durable end-to-end key store. -#[derive(Debug)] -pub struct AwgPlugin { - inner: Arc, - transport: Arc, -} -impl AwgPlugin { - /// Uses standard tunnel MTU/keepalive options; pair ports and profiles are ephemeral. - pub const OPTIONS: &'static [ProtocolOption] = &[ - ProtocolOption { - key: "keepalive", - value: "SECONDS", - help: "end-to-end tunnel keepalive; 0 disables it", - default: Some("25"), - }, - ProtocolOption { - key: "mtu", - value: "BYTES", - help: "maximum overlay packet size", - default: Some("1280"), - }, - ProtocolOption { - key: "stun", - value: "HOST:PORT|off", - help: "UDP public mapping discovery; disabled in local reach mode", - default: Some("stun.l.google.com:19302"), - }, - ]; - /// Opens the end-to-end engine, retaining separate pair transport state. - pub async fn open( - mut config: WireguardConfig, - transport: Arc, - ) -> Result, PluginError> { - config.protocol = AWG_PROTOCOL; - Ok(Arc::new(Self { - inner: WireguardPlugin::open(config).await?, - transport, - })) - } - /// Reports end-to-end tunnel health and traffic counters. - pub fn overview(&self, network: NetworkId) -> Option { - self.inner.overview(network) - } -} -impl IpPlugin for AwgPlugin { - fn protocol_id(&self) -> &str { - AWG_PROTOCOL - } - fn protocol_version(&self) -> u16 { - AWG_WIRE_VERSION - } - fn options(&self) -> &'static [ProtocolOption] { - Self::OPTIONS - } - fn attach(&self, context: PluginContext) { - self.inner.attach(context); - } - fn local_capability( - &self, - network: NetworkId, - ) -> Result, PluginError> { - self.inner.local_capability(network).map(|cap| { - cap.map(|mut cap| { - cap.protocol = AWG_PROTOCOL.into(); - cap.version = AWG_WIRE_VERSION; - cap - }) - }) - } - fn on_network_activated(&self, network: NetworkId) { - self.inner.on_network_activated(network); - } - fn on_peer_capability( - &self, - network: NetworkId, - peer: EndpointId, - capability: &PluginCapability, - ) -> Result<(), PluginError> { - if capability.protocol != AWG_PROTOCOL - || capability.version != AWG_WIRE_VERSION - || !capability.enabled - { - return Err(PluginError::Rejected("unsupported AWG capability".into())); - } - self.inner.on_peer_capability(network, peer, capability)?; - self.transport.allow_peer(network, peer); - Ok(()) - } - fn on_address_allocation( - &self, - network: NetworkId, - range: Ipv4Range, - allocations: &[(EndpointId, std::net::Ipv4Addr)], - ) { - self.inner - .on_address_allocation(network, range, allocations); - } - fn carry(&self, network: NetworkId, peer: EndpointId, packet: bytes::Bytes) -> bool { - self.inner.carry(network, peer, packet) - } - fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) { - self.inner.on_peer_link(network, peer, link); - } - fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) { - self.transport.forget_peer(network, peer); - self.inner.on_peer_gone(network, peer); - } - fn on_network_deactivated(&self, network: NetworkId) { - self.transport.stop_network(network); - self.inner.on_network_deactivated(network); - } - fn on_network_forgotten(&self, network: NetworkId) { - self.inner.on_network_forgotten(network); - } - fn shutdown(&self) -> BoxFuture<'_, ()> { - Box::pin(async move { - self.transport.shutdown(); - self.inner.shutdown().await; - }) - } -} diff --git a/crates/tsunagi-awg/src/transport.rs b/crates/tsunagi-awg/src/transport.rs deleted file mode 100644 index 491e601..0000000 --- a/crates/tsunagi-awg/src/transport.rs +++ /dev/null @@ -1,720 +0,0 @@ -//! Private pair negotiation. No secret profiles in capabilities or durable state. -use crate::{ - AWG_PROTOCOL, AWG_WIRE_VERSION, - profile::PairProfile, - session::{self, PairLink}, -}; -use boringtun::{ - noise::Tunn, - x25519::{PublicKey, StaticSecret}, -}; -use iroh::EndpointId; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::net::{Ipv4Addr, SocketAddr}; -use std::sync::{Arc, Mutex, OnceLock, Weak}; -use std::time::Duration; -use tokio::net::UdpSocket; -use tokio::sync::{mpsc, oneshot, watch}; -use tsunagi::dataplane::transport::TransportPayload; -use tsunagi::dataplane::transport::{ - ControlSender, InboundLink, PacketLink, PacketTransport, SharedLink, TransportError, -}; -use tsunagi::{BoxFuture, NetworkId}; - -type Pair = (NetworkId, EndpointId); -#[derive(Serialize, Deserialize)] -enum Negotiation { - Offer { - version: u16, - profile: PairProfile, - key: [u8; 32], - #[serde(deserialize_with = "bounded_addresses")] - addresses: Vec, - }, - Answer { - attempt: [u8; 16], - key: [u8; 32], - #[serde(deserialize_with = "bounded_addresses")] - addresses: Vec, - }, -} -fn bounded_addresses<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - struct Visitor; - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = Vec; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("at most 32 UDP candidates") - } - fn visit_seq>( - self, - mut sequence: A, - ) -> Result { - if sequence.size_hint().is_some_and(|n| n > 32) { - return Err(serde::de::Error::custom("too many UDP candidates")); - } - let mut addresses = Vec::with_capacity(sequence.size_hint().unwrap_or(0).min(32)); - while let Some(address) = sequence.next_element()? { - if addresses.len() == 32 { - return Err(serde::de::Error::custom("too many UDP candidates")); - } - addresses.push(address); - } - Ok(addresses) - } - } - deserializer.deserialize_seq(Visitor) -} -struct Pending { - profile: PairProfile, - key: StaticSecret, - socket: UdpSocket, - reply: oneshot::Sender>, -} -struct Inner { - stopped: watch::Sender, - local: EndpointId, - bind: Ipv4Addr, - stun: Option, - busy: Mutex>, - control: OnceLock>, - allowed: Mutex>, - pending: Mutex>, - links: Mutex>>, - inbound: mpsc::Sender, - port_by_pair: Mutex>, -} -/// Public diagnostics for one pair. No key or masking parameters are exposed. -#[derive(Debug, Clone)] -pub struct PairOverview { - /// Other endpoint's authenticated identity. - pub peer: EndpointId, - /// Fresh connection-attempt identifier. - pub attempt: [u8; 16], - /// Last known UDP port; None if diagnostics race with pair retirement. - pub local_port: Option, - /// Actual transport path, if established. - pub path: String, - /// Whether this attempt has been retired. - pub closed: bool, -} -/// A bounded set of independently configured pair engines. -pub struct AwgTransport { - inner: Arc, - commands: mpsc::Sender<(NetworkId, EndpointId, Negotiation)>, - incoming: tokio::sync::Mutex>, -} -impl std::fmt::Debug for AwgTransport { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AwgTransport") - .field("local", &self.inner.local) - .finish_non_exhaustive() - } -} -fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { - match mutex.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - } -} -fn error(text: &str) -> TransportError { - TransportError::Other(text.into()) -} -async fn addresses(inner: &Inner, socket: &UdpSocket) -> Result, TransportError> { - let local = socket - .local_addr() - .map_err(|_| error("AWG socket address unavailable"))?; - let mut candidates = if inner.bind.is_loopback() { - vec![local] - } else { - tsunagi_wireguard::local_ip_candidates() - .into_iter() - .filter(|ip| ip.is_ipv4()) - .take(31) - .map(|ip| SocketAddr::new(ip, local.port())) - .collect() - }; - if let Some(server) = &inner.stun - && let Some(mapped) = crate::stun::mapped(socket, server).await - && !candidates.contains(&mapped) - { - candidates.push(mapped); - } - if candidates.is_empty() { - return Err(error("AWG has no UDP address candidates")); - } - Ok(candidates) -} -fn validate_endpoint(key: &[u8; 32], addresses: &[SocketAddr]) -> Result<(), TransportError> { - if *key == [0; 32] - || addresses.is_empty() - || addresses.len() > 32 - || addresses.iter().any(|address| { - address.port() == 0 - || !address.is_ipv4() - || address.ip().is_unspecified() - || address.ip().is_multicast() - || matches!(address.ip(), std::net::IpAddr::V4(ip) if ip.is_broadcast()) - }) - { - return Err(error("invalid AWG endpoint")); - } - Ok(()) -} - -impl AwgTransport { - /// Starts the coordinator. Production binds wildcard IPv4; tests use loopback. - pub fn new(local: EndpointId, bind: Ipv4Addr) -> Arc { - Self::new_with_stun(local, bind, None) - } - /// Enables an optional STUN server (host:port) for this pair's own socket. - /// Local-only callers pass None; failure still permits local UDP candidates. - pub fn new_with_stun(local: EndpointId, bind: Ipv4Addr, stun: Option) -> Arc { - let (commands, mut rx) = mpsc::channel(64); - let (inbound, incoming) = mpsc::channel(64); - let (stopped, _) = watch::channel(false); - let inner = Arc::new(Inner { - stopped, - local, - bind, - stun, - busy: Mutex::default(), - control: OnceLock::new(), - allowed: Mutex::default(), - pending: Mutex::default(), - links: Mutex::default(), - inbound, - port_by_pair: Mutex::default(), - }); - let weak = Arc::downgrade(&inner); - let mut stopped = inner.stopped.subscribe(); - tokio::spawn(async move { - while let Some((network, peer, message)) = - tokio::select! { biased; _=stopped.changed()=>None, message=rx.recv()=>message } - { - let Some(inner) = weak.upgrade() else { - break; - }; - if !lock(&inner.allowed).contains(&(network, peer)) - || !lock(&inner.busy).insert((network, peer)) - { - continue; - } - tokio::spawn(async move { - let mut stop = inner.stopped.subscribe(); - tokio::select! {biased; _=stop.changed()=>{}, _=async { - if let Err(err)=receive(&inner,(network,peer),message).await {tracing::debug!(%err,"AWG pair negotiation rejected");} - }=>{}} - lock(&inner.busy).remove(&(network, peer)); - }); - } - }); - Arc::new(Self { - inner, - commands, - incoming: tokio::sync::Mutex::new(incoming), - }) - } - /// Registers a peer after its authenticated capability has been validated. - pub fn allow_peer(&self, network: NetworkId, peer: EndpointId) { - let mut allowed = lock(&self.inner.allowed); - if !*self.inner.stopped.borrow() && allowed.len() < 128 { - allowed.insert((network, peer)); - } - } - /// Retires a pair when its network membership session goes away. - pub fn forget_peer(&self, network: NetworkId, peer: EndpointId) { - let pair = (network, peer); - lock(&self.inner.allowed).remove(&pair); - lock(&self.inner.pending).remove(&pair); - lock(&self.inner.port_by_pair).remove(&pair); - if let Some(link) = lock(&self.inner.links).remove(&pair) { - link.close(); - } - } - /// Stops all pair engines in this network. - pub fn stop_network(&self, network: NetworkId) { - let peers: Vec<_> = lock(&self.inner.allowed) - .iter() - .filter(|(n, _)| *n == network) - .map(|(_, p)| *p) - .collect(); - for peer in peers { - self.forget_peer(network, peer); - } - } - /// Stops all sockets and coordinator work without waiting for remote peers. - pub fn shutdown(&self) { - self.inner.stopped.send_replace(true); - lock(&self.inner.allowed).clear(); - lock(&self.inner.pending).clear(); - for (_, link) in lock(&self.inner.links).drain() { - link.close(); - } - lock(&self.inner.port_by_pair).clear(); - } - /// Receives a successfully negotiated and authenticated inbound pair link. - pub async fn next_inbound(&self) -> Option { - let mut stopped = self.inner.stopped.subscribe(); - if *stopped.borrow() { - return None; - } - let mut incoming = self.incoming.lock().await; - tokio::select! {biased; _=stopped.changed()=>None, link=incoming.recv()=>link} - } - /// Takes a public snapshot of this network's pair attempts. - pub fn overview(&self, network: NetworkId) -> Vec { - let links = lock(&self.inner.links); - let ports = lock(&self.inner.port_by_pair); - links - .iter() - .filter(|((n, _), _)| *n == network) - .map(|((_, peer), link)| PairOverview { - peer: *peer, - attempt: link.attempt, - local_port: ports.get(&(network, *peer)).copied(), - path: link.path_description(), - closed: link.is_closed(), - }) - .collect() - } -} -impl PacketTransport for AwgTransport { - fn name(&self) -> &str { - AWG_PROTOCOL - } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn attach_control(&self, control: Arc) { - let _ = self.inner.control.set(control); - } - fn on_control( - &self, - network: NetworkId, - peer: EndpointId, - protocol: &str, - payload: &[u8], - ) -> Result<(), TransportError> { - if protocol != AWG_PROTOCOL || !lock(&self.inner.allowed).contains(&(network, peer)) { - return Err(TransportError::Declined(protocol.into())); - } - if payload.len() > 4096 { - return Err(error("AWG negotiation too large")); - } - let message: Negotiation = - postcard::from_bytes(payload).map_err(|_| error("malformed AWG negotiation"))?; - match &message { - Negotiation::Offer { - version, - profile, - key, - addresses, - } => { - if *version != AWG_WIRE_VERSION || peer.as_bytes() >= self.inner.local.as_bytes() { - return Err(error("invalid AWG offer role/version")); - } - profile.validate().map_err(error)?; - validate_endpoint(key, addresses)?; - } - Negotiation::Answer { key, addresses, .. } => validate_endpoint(key, addresses)?, - } - self.commands - .try_send((network, peer, message)) - .map_err(|_| error("AWG negotiation queue full")) - } - fn open<'a>( - &'a self, - network: NetworkId, - peer: EndpointId, - protocol: &'a str, - ) -> BoxFuture<'a, Result> { - Box::pin(async move { - let pair = (network, peer); - if protocol != AWG_PROTOCOL - || self.inner.local.as_bytes() >= peer.as_bytes() - || !lock(&self.inner.allowed).contains(&pair) - { - return Err(TransportError::Declined(protocol.into())); - } - let control = self - .inner - .control - .get() - .ok_or_else(|| error("AWG control plane unavailable"))?; - let socket = UdpSocket::bind((self.inner.bind, 0)) - .await - .map_err(|_| error("AWG UDP bind failed"))?; - let profile = PairProfile::generate(); - let attempt = profile.id(); - let key = StaticSecret::from(rand::random::<[u8; 32]>()); - let public = PublicKey::from(&key).to_bytes(); - let message = Negotiation::Offer { - version: AWG_WIRE_VERSION, - profile: profile.clone(), - key: public, - addresses: addresses(&self.inner, &socket).await?, - }; - let (reply, rx) = oneshot::channel(); - if lock(&self.inner.pending).len() >= 128 { - return Err(error("too many AWG pair negotiations")); - } - lock(&self.inner.pending).insert( - pair, - Pending { - profile, - key, - socket, - reply, - }, - ); - // A cancellation guard also retires partially started attempts. - let mut guard = AttemptGuard { - inner: Arc::downgrade(&self.inner), - pair, - attempt, - completed: false, - }; - let bytes = - postcard::to_stdvec(&message).map_err(|_| error("AWG offer encoding failed"))?; - control - .send(network, peer, AWG_PROTOCOL, TransportPayload::new(bytes)) - .await?; - let result = tokio::time::timeout(Duration::from_secs(15), rx) - .await - .map_err(|_| error("AWG negotiation/handshake timed out"))? - .map_err(|_| error("AWG attempt cancelled"))?; - guard.completed = result.is_ok(); - result - }) - } -} -struct AttemptGuard { - inner: Weak, - pair: Pair, - attempt: [u8; 16], - completed: bool, -} -impl Drop for AttemptGuard { - fn drop(&mut self) { - if let Some(inner) = self.inner.upgrade() { - let mut pending = lock(&inner.pending); - if pending - .get(&self.pair) - .is_some_and(|p| p.profile.id() == self.attempt) - { - pending.remove(&self.pair); - } - drop(pending); - if !self.completed - && let Some(link) = lock(&inner.links).get(&self.pair) - && link.attempt == self.attempt - { - link.close(); - } - } - } -} -async fn receive( - inner: &Arc, - pair: Pair, - message: Negotiation, -) -> Result<(), TransportError> { - if !lock(&inner.allowed).contains(&pair) { - return Err(error("AWG peer no longer authenticated")); - } - match message { - Negotiation::Offer { - profile, - key, - addresses: candidates, - .. - } => { - if let Some(link) = lock(&inner.links).get(&pair) { - if link.attempt == profile.id() { - return Err(error("duplicate AWG offer")); - } - link.close(); - } - let socket = UdpSocket::bind((inner.bind, 0)) - .await - .map_err(|_| error("AWG UDP bind failed"))?; - let local_port = socket - .local_addr() - .map_err(|_| error("AWG socket address unavailable"))? - .port(); - let local_key = StaticSecret::from(rand::random::<[u8; 32]>()); - let public = PublicKey::from(&local_key).to_bytes(); - let answer = Negotiation::Answer { - attempt: profile.id(), - key: public, - addresses: addresses(inner, &socket).await?, - }; - let tunn = Tunn::new( - local_key, - PublicKey::from(key), - Some(profile.psk), - Some(1), - rand::random(), - None, - ); - - let link = session::start( - pair.0, - pair.1, - socket, - tunn, - profile, - candidates, - inner.stopped.subscribe(), - ); - if !lock(&inner.allowed).contains(&pair) { - link.close(); - return Err(error("AWG membership ended")); - } - lock(&inner.links).insert(pair, link.clone()); - lock(&inner.port_by_pair).insert(pair, local_port); - let control = inner - .control - .get() - .ok_or_else(|| error("AWG control unavailable"))?; - if let Err(err) = control - .send( - pair.0, - pair.1, - AWG_PROTOCOL, - TransportPayload::new( - postcard::to_stdvec(&answer) - .map_err(|_| error("AWG answer encoding failed"))?, - ), - ) - .await - { - link.close(); - return Err(err); - } - publish_when_up(Arc::downgrade(inner), pair, link, None); - } - Negotiation::Answer { - attempt, - key, - addresses: candidates, - } => { - if !lock(&inner.pending) - .get(&pair) - .is_some_and(|p| p.profile.id() == attempt) - { - return Err(error("stale AWG answer")); - } - let pending = lock(&inner.pending) - .remove(&pair) - .ok_or_else(|| error("AWG attempt cancelled"))?; - let local_port = pending - .socket - .local_addr() - .map_err(|_| error("AWG socket address unavailable"))? - .port(); - let tunn = Tunn::new( - pending.key, - PublicKey::from(key), - Some(pending.profile.psk), - Some(1), - rand::random(), - None, - ); - - let link = session::start( - pair.0, - pair.1, - pending.socket, - tunn, - pending.profile, - candidates, - inner.stopped.subscribe(), - ); - if !lock(&inner.allowed).contains(&pair) { - link.close(); - return Err(error("AWG membership ended")); - } - lock(&inner.links).insert(pair, link.clone()); - lock(&inner.port_by_pair).insert(pair, local_port); - publish_when_up(Arc::downgrade(inner), pair, link, Some(pending.reply)); - } - } - Ok(()) -} -fn publish_when_up( - inner: Weak, - pair: Pair, - link: Arc, - reply: Option>>, -) { - tokio::spawn(async move { - let mut reply = reply; - let result = tokio::time::timeout(Duration::from_secs(14), async { - loop { - if link.is_closed() || reply.as_ref().is_some_and(oneshot::Sender::is_closed) { - return Err(error("AWG pair closed during handshake")); - } - if link.has_authenticated_path() { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await; - if !matches!(result, Ok(Ok(()))) { - link.close(); - if let Some(reply) = reply.take() { - let _ = reply.send(Err(error("AWG authenticated data path timed out"))); - } - return; - } - let Some(inner) = inner.upgrade() else { - link.close(); - return; - }; - if !lock(&inner.allowed).contains(&pair) { - link.close(); - return; - } - if link.is_closed() - || !lock(&inner.links) - .get(&pair) - .is_some_and(|current| current.attempt == link.attempt) - { - link.close(); - return; - } - let shared: SharedLink = link.clone(); - if let Some(reply) = reply { - if reply.send(Ok(shared)).is_err() { - link.close(); - } - } else if inner - .inbound - .try_send(InboundLink { - network: pair.0, - peer: pair.1, - protocol: AWG_PROTOCOL.into(), - link: shared, - }) - .is_err() - { - link.close(); - } - }); -} -impl Drop for AwgTransport { - fn drop(&mut self) { - self.shutdown(); - } -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - use super::*; - #[tokio::test] - async fn concurrent_admission_is_bounded_across_networks_and_retirement_releases_slots() { - let identity = tsunagi::identity::DeviceIdentity::generate(); - let transport = AwgTransport::new(identity.endpoint_id(), Ipv4Addr::LOCALHOST); - let peer = tsunagi::identity::DeviceIdentity::generate().endpoint_id(); - std::thread::scope(|scope| { - for thread in 0..8u8 { - let transport = &transport; - scope.spawn(move || { - for item in 0..32u8 { - let mut id = [0; 32]; - id[0] = thread; - id[1] = item; - transport.allow_peer(NetworkId::from_bytes(id), peer); - } - }); - } - }); - assert_eq!(lock(&transport.inner.allowed).len(), 128); - let network = lock(&transport.inner.allowed).iter().next().unwrap().0; - transport.stop_network(network); - let new = NetworkId::from_bytes([255; 32]); - transport.allow_peer(new, peer); - assert!(lock(&transport.inner.allowed).contains(&(new, peer))); - assert_eq!(lock(&transport.inner.allowed).len(), 128); - transport.shutdown(); - assert!(lock(&transport.inner.allowed).is_empty()); - } - #[tokio::test] - async fn private_negotiation_bounds_roles_and_membership_before_starting_engines() { - let mut ids = [ - tsunagi::identity::DeviceIdentity::generate().endpoint_id(), - tsunagi::identity::DeviceIdentity::generate().endpoint_id(), - ]; - ids.sort_by_key(|id| *id.as_bytes()); - let peer = ids[0]; - let local = ids[1]; - let network = NetworkId::from_bytes([7; 32]); - let transport = AwgTransport::new(local, Ipv4Addr::LOCALHOST); - let offer = |version, key, addresses| Negotiation::Offer { - version, - profile: PairProfile::generate(), - key, - addresses, - }; - let address = SocketAddr::from((Ipv4Addr::LOCALHOST, 12345)); - let encode = |message: &Negotiation| postcard::to_stdvec(message).unwrap(); - let valid = encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![address])); - assert!( - transport - .on_control(network, peer, AWG_PROTOCOL, &valid) - .is_err() - ); - transport.allow_peer(network, peer); - assert!( - transport - .on_control(network, peer, "wrong-protocol", &valid) - .is_err() - ); - assert!( - transport - .on_control(network, peer, AWG_PROTOCOL, &[0; 4097]) - .is_err() - ); - for bytes in [ - encode(&offer(99, [1; 32], vec![address])), - encode(&offer(AWG_WIRE_VERSION, [0; 32], vec![address])), - encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![])), - encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![address; 33])), - ] { - assert!( - transport - .on_control(network, peer, AWG_PROTOCOL, &bytes) - .is_err() - ); - } - for len in 0..valid.len() { - assert!( - transport - .on_control(network, peer, AWG_PROTOCOL, &valid[..len]) - .is_err() - ); - } - let wrong_role = AwgTransport::new(peer, Ipv4Addr::LOCALHOST); - wrong_role.allow_peer(network, local); - assert!( - wrong_role - .on_control(network, local, AWG_PROTOCOL, &valid) - .is_err() - ); - transport.forget_peer(network, peer); - assert!( - transport - .on_control(network, peer, AWG_PROTOCOL, &valid) - .is_err() - ); - assert!(transport.overview(network).is_empty()); - transport.shutdown(); - wrong_role.shutdown(); - } -} diff --git a/crates/tsunagi-awg/tests/common/tcp.rs b/crates/tsunagi-awg/tests/common/tcp.rs new file mode 100644 index 0000000..b5d862c --- /dev/null +++ b/crates/tsunagi-awg/tests/common/tcp.rs @@ -0,0 +1,221 @@ +//! A real userspace TCP stack attached to each memory TUN, without host routes. +use bytes::Bytes; +use smoltcp::{ + iface::{Config, Interface, SocketHandle, SocketSet}, + phy::{Device, DeviceCapabilities, Medium, RxToken, TxToken}, + socket::tcp, + time::Instant as TcpInstant, + wire::{HardwareAddress, IpAddress, IpCidr}, +}; +use std::{ + collections::VecDeque, + net::Ipv4Addr, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; +use tsunagi::overlay::MemoryTun; + +struct TunDevice { + input: Arc>>>, + tun: Arc, +} +struct Rx(Vec); +impl RxToken for Rx { + fn consume R>(self, f: F) -> R { + f(&self.0) + } +} +struct Tx(Arc); +impl TxToken for Tx { + fn consume R>(self, len: usize, f: F) -> R { + let mut packet = vec![0; len]; + let result = f(&mut packet); + self.0.push_from_os(Bytes::from(packet)); + result + } +} +impl Device for TunDevice { + type RxToken<'a> + = Rx + where + Self: 'a; + type TxToken<'a> + = Tx + where + Self: 'a; + fn receive(&mut self, _: TcpInstant) -> Option<(Rx, Tx)> { + self.input + .lock() + .unwrap() + .pop_front() + .map(|packet| (Rx(packet), Tx(self.tun.clone()))) + } + fn transmit(&mut self, _: TcpInstant) -> Option { + Some(Tx(self.tun.clone())) + } + fn capabilities(&self) -> DeviceCapabilities { + let mut caps = DeviceCapabilities::default(); + caps.medium = Medium::Ip; + caps.max_transmission_unit = 1280; + caps + } +} +struct Stack { + iface: Interface, + sockets: SocketSet<'static>, + socket: SocketHandle, + device: TunDevice, + reader: tokio::task::JoinHandle<()>, +} +impl Stack { + fn new(tun: Arc, address: Ipv4Addr) -> Self { + let input = Arc::new(Mutex::new(VecDeque::new())); + let queue = input.clone(); + let received = tun.clone(); + let reader = tokio::spawn(async move { + while let Some(packet) = received.pop_to_os().await { + let mut queue = queue.lock().unwrap(); + assert!(queue.len() < 256); + queue.push_back(packet.to_vec()); + } + }); + let mut device = TunDevice { input, tun }; + let mut config = Config::new(HardwareAddress::Ip); + config.random_seed = rand::random(); + let mut iface = Interface::new(config, &mut device, TcpInstant::from_millis(0)); + let [a, b, c, d] = address.octets(); + iface.update_ip_addrs(|addresses| { + addresses + .push(IpCidr::new(IpAddress::v4(a, b, c, d), 24)) + .unwrap() + }); + let mut sockets = SocketSet::new(Vec::new()); + let socket = sockets.add(tcp::Socket::new( + tcp::SocketBuffer::new(vec![0; 65536]), + tcp::SocketBuffer::new(vec![0; 65536]), + )); + Self { + iface, + sockets, + socket, + device, + reader, + } + } + fn tcp(&mut self) -> &mut tcp::Socket<'static> { + self.sockets.get_mut(self.socket) + } + fn poll(&mut self, now: TcpInstant) { + self.iface.poll(now, &mut self.device, &mut self.sockets); + } +} +impl Drop for Stack { + fn drop(&mut self) { + self.reader.abort(); + } +} + +pub(super) struct TcpPair { + client: Stack, + server: Stack, + started: Instant, +} +impl TcpPair { + pub(super) async fn connect( + a: Arc, + a_ip: Ipv4Addr, + b: Arc, + b_ip: Ipv4Addr, + ) -> Self { + let mut pair = Self { + client: Stack::new(a, a_ip), + server: Stack::new(b, b_ip), + started: Instant::now(), + }; + pair.server.tcp().listen(22).unwrap(); + let [a, b, c, d] = b_ip.octets(); + pair.client + .sockets + .get_mut::(pair.client.socket) + .connect( + pair.client.iface.context(), + (IpAddress::v4(a, b, c, d), 22), + 40000, + ) + .unwrap(); + pair.until(|p| { + p.client.tcp().state() == tcp::State::Established + && p.server.tcp().state() == tcp::State::Established + }) + .await; + pair + } + fn poll(&mut self) { + let now = TcpInstant::from_millis(self.started.elapsed().as_millis() as i64); + self.client.poll(now); + self.server.poll(now); + } + async fn until(&mut self, mut ready: impl FnMut(&mut Self) -> bool) { + tokio::time::timeout(Duration::from_secs(10), async { + let mut ticks = tokio::time::interval(Duration::from_millis(2)); + loop { + ticks.tick().await; + self.poll(); + if ready(self) { + break; + } + } + }) + .await + .expect("same TCP connection must progress across carrier/route changes"); + } + pub(super) fn queue_while_unreachable(&mut self, payload: &[u8]) { + assert_eq!( + self.client.tcp().send_slice(payload).unwrap(), + payload.len() + ); + self.poll(); + } + pub(super) async fn receive_queued(&mut self, payload: &[u8]) { + let mut received = Vec::new(); + self.until(|p| { + if p.server.tcp().can_recv() { + p.server + .tcp() + .recv(|bytes| { + received.extend_from_slice(bytes); + (bytes.len(), ()) + }) + .unwrap(); + } + received.len() >= payload.len() + }) + .await; + assert_eq!(received, payload); + assert_eq!(self.client.tcp().state(), tcp::State::Established); + assert_eq!(self.server.tcp().state(), tcp::State::Established); + } + pub(super) async fn exchange(&mut self, payload: &[u8]) { + self.queue_while_unreachable(payload); + self.receive_queued(payload).await; + assert_eq!( + self.server.tcp().send_slice(payload).unwrap(), + payload.len() + ); + let mut received = Vec::new(); + self.until(|p| { + if p.client.tcp().can_recv() { + p.client + .tcp() + .recv(|bytes| { + received.extend_from_slice(bytes); + (bytes.len(), ()) + }) + .unwrap(); + } + received.len() >= payload.len() + }) + .await; + assert_eq!(received, payload); + } +} diff --git a/crates/tsunagi-awg/tests/pairs.rs b/crates/tsunagi-awg/tests/pairs.rs index 73573b1..3669ed7 100644 --- a/crates/tsunagi-awg/tests/pairs.rs +++ b/crates/tsunagi-awg/tests/pairs.rs @@ -1,5 +1,7 @@ //! Real membership, UDP sockets, pair negotiation and nested WireGuard encryption. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +#[path = "common/tcp.rs"] +mod tcp; use bytes::Bytes; use std::{ net::Ipv4Addr, @@ -24,8 +26,7 @@ use tsunagi_wg_quic::WireguardConfig; struct Node { agent: Agent, - plugin: Arc, - quic: Option>, + plugin: Arc, transport: Arc, tuns: MemoryTunFactory, loss: Arc, @@ -65,11 +66,10 @@ impl Node { .load_or_create_device_identity() .unwrap(); let transport = AwgTransport::new(identity.endpoint_id(), Ipv4Addr::LOCALHOST); - let plugin = AwgPlugin::open( + let plugin = tsunagi_wg_quic::WireguardPlugin::open( WireguardConfig::new(root.join("awg")) .with_mtu(mtu) .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), - transport.clone(), ) .await .unwrap(); @@ -81,19 +81,13 @@ impl Node { let tuns = MemoryTunFactory::new(); let mut config = config_with(root, discovery) .with_interface(Arc::new(tuns.clone()), interface, mtu) - .with_plugin(plugin.clone() as Arc) + .with_tunnel(plugin.clone() as Arc) + .with_carrier(AwgPlugin::new(transport.clone())) .with_custom_transport(AWG_PROTOCOL, faults); - let mut quic_plugin = None; if fallback { - let quic = tsunagi_wg_quic::WireguardPlugin::open( - WireguardConfig::new(root.join("wg-quic")) - .with_mtu(mtu) - .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), - ) - .await - .unwrap(); - config = config.with_plugin(quic.clone()); - quic_plugin = Some(quic); + config = config.with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )); } config.default_protocols = default_protocols; let agent = Agent::spawn(config).await.unwrap(); @@ -109,7 +103,6 @@ impl Node { Self { agent, plugin, - quic: quic_plugin, transport, tuns, loss, @@ -252,7 +245,7 @@ async fn a_control_only_agent_enables_real_data_without_restart_and_cancels_stal } #[tokio::test] -async fn runtime_protocol_selection_replaces_real_tunnels_and_survives_restart() { +async fn runtime_protocol_selection_preserves_real_tunnels_and_survives_restart() { let discovery = SharedMemoryDiscovery::new(); let roots = [TempDir::new().unwrap(), TempDir::new().unwrap()]; let a = Node::open_with_fallback(roots[0].path(), &discovery, "runtime-a", true).await; @@ -263,6 +256,12 @@ async fn runtime_protocol_selection_replaces_real_tunnels_and_survives_restart() a.ready(id, 1).await; b.ready(id, 1).await; exchange(&a, &b, id).await; + let original_key = a.plugin.overview(id).unwrap().public_key; + let original_stats = a.plugin.overview(id).unwrap().peers[0] + .tunnel + .as_ref() + .unwrap() + .stats; let (other_name, other_secret, other_id) = loop { let (name, secret) = network("unchanged-runtime-network"); @@ -338,13 +337,25 @@ async fn runtime_protocol_selection_replaces_real_tunnels_and_survives_restart() .then_some(()) }) .await; - // Get addresses from the system allocation: the disabled AWG plugin no - // longer has a live view of this network. + assert_eq!(a.plugin.overview(id).unwrap().public_key, original_key); + let stats = a.plugin.overview(id).unwrap().peers[0] + .tunnel + .as_ref() + .unwrap() + .stats; + assert_eq!(stats.counter_epoch, original_stats.counter_epoch); + assert!(stats.tx_bytes >= original_stats.tx_bytes); + assert!(stats.rx_bytes >= original_stats.rx_bytes); + assert!(a.plugin.overview(id).unwrap().peers.iter().any(|peer| { + peer.tunnel + .as_ref() + .is_some_and(|tunnel| tunnel.health.is_up()) + })); + // The same internal plugin supplies addresses and counters on the new carrier. for (source, dest) in [(&a, &b), (&b, &a)] { wait_until("QUIC end-to-end handshake", || async { source - .quic - .as_ref()? + .plugin .overview(id)? .peers .iter() @@ -352,15 +363,7 @@ async fn runtime_protocol_selection_replaces_real_tunnels_and_survives_restart() .then_some(()) }) .await; - let addr = |n: &Node| { - n.quic - .as_ref() - .unwrap() - .overview(id) - .unwrap() - .overlay_address_v4 - .unwrap() - }; + let addr = |n: &Node| n.plugin.overview(id).unwrap().overlay_address_v4.unwrap(); let bytes = packet(addr(source), addr(dest), 1280); source.tun().push_from_os(bytes.clone()); assert_eq!( @@ -604,18 +607,13 @@ async fn persistent_runtime_loss_retires_the_pair_and_negotiates_fresh_parameter } else { &b }; - let received = initiator.plugin.overview(id).unwrap().peers[0] - .tunnel - .as_ref() - .unwrap() - .stats - .rx_packets; let attempt = initiator.transport.overview(id)[0].attempt; + let received = initiator.transport.overview(id)[0].delivery_checks; wait_until( - "fresh pair delivery checks answered after installation", + "fresh physical delivery checks answered after installation", || async { - let view = initiator.plugin.overview(id)?; - (view.peers.first()?.tunnel.as_ref()?.stats.rx_packets >= received + 3).then_some(()) + (initiator.transport.overview(id).first()?.delivery_checks >= received + 3) + .then_some(()) }, ) .await; @@ -749,11 +747,11 @@ async fn encrypted_health_checks_fit_a_smaller_peers_configured_interface_mtu() bytes ); wait_until( - "encrypted delivery checks answered with a small MTU", + "physical delivery checks do not depend on overlay MTU", || async { - let view = a.plugin.overview(id)?; - let peer = view.peers.first()?.tunnel.as_ref()?; - (peer.stats.rx_packets >= 3).then_some(()) + (a.transport.overview(id).first()?.delivery_checks >= 4 + && b.transport.overview(id).first()?.delivery_checks >= 4) + .then_some(()) }, ) .await; @@ -771,3 +769,179 @@ async fn encrypted_health_checks_fit_a_smaller_peers_configured_interface_mtu() a.agent.shutdown().await; b.agent.shutdown().await; } + +#[tokio::test] +async fn disjoint_external_policies_relay_the_same_internal_tunnel_and_keep_it_when_routes_change() +{ + let discovery = SharedMemoryDiscovery::new(); + let roots = [ + TempDir::new().unwrap(), + TempDir::new().unwrap(), + TempDir::new().unwrap(), + ]; + let a = Node::open_selected( + roots[0].path(), + &discovery, + "only-awg", + true, + 1280, + Some(vec!["awg".into()]), + ) + .await; + let b = Node::open_with_fallback(roots[1].path(), &discovery, "mixed-relay", true).await; + let c = Node::open_selected( + roots[2].path(), + &discovery, + "only-quic", + true, + 1280, + Some(vec!["wg-quic".into()]), + ) + .await; + let (name, secret) = network("independent-internal-encryption"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + c.agent.join_network(&name, &secret).await.unwrap(); + for node in [&a, &b, &c] { + wait_for_peers(&node.agent, id, 2).await; + } + wait_until("AWG-only to QUIC-only through an authenticated mixed relay", || async { + let status = a.agent.network_status(id).await.ok()?; + let peer = status.peers.iter().find(|p| p.endpoint_id == c.agent.endpoint_id())?; + (matches!(peer.data_path, tsunagi::agent::DataPath::Relayed { via, .. } if via == b.agent.endpoint_id()) && a.plugin.overview(id)?.established_peers() == 2 && c.plugin.overview(id)?.established_peers() == 2).then_some(()) + }).await; + exchange(&a, &c, id).await; + exchange(&c, &a, id).await; + let mut tcp = tcp::TcpPair::connect(a.tun(), a.addr(id), c.tun(), c.addr(id)).await; + tcp.exchange(b"SSH-like TCP stream over the initial relay") + .await; + let key = a.plugin.overview(id).unwrap().public_key; + let before = a + .plugin + .overview(id) + .unwrap() + .peers + .iter() + .find(|p| p.endpoint_id == c.agent.endpoint_id()) + .unwrap() + .tunnel + .as_ref() + .unwrap() + .stats + .tx_packets; + // A's additional permission creates a direct QUIC route without replacing its IP tunnel. + a.agent + .set_protocols(id, vec!["wg-quic".into(), "awg".into()]) + .await + .unwrap(); + wait_until("direct carrier replaces the relay", || async { + let status = a.agent.network_status(id).await.ok()?; + matches!( + status + .peers + .iter() + .find(|p| p.endpoint_id == c.agent.endpoint_id())? + .data_path, + tsunagi::agent::DataPath::Direct { .. } + ) + .then_some(()) + }) + .await; + tcp.exchange(b"same TCP socket after changing the carrier and next hop") + .await; + assert_eq!(a.plugin.overview(id).unwrap().public_key, key); + assert!( + a.plugin + .overview(id) + .unwrap() + .peers + .iter() + .find(|p| p.endpoint_id == c.agent.endpoint_id()) + .unwrap() + .tunnel + .as_ref() + .unwrap() + .stats + .tx_packets + > before + ); + a.agent.set_protocols(id, Vec::new()).await.unwrap(); + // Poll the actual TCP stack while the route is absent: its first transmission is lost. + let pending = b"TCP retransmission survives the temporary absence of every data carrier"; + tcp.queue_while_unreachable(pending); + a.agent.set_protocols(id, vec!["awg".into()]).await.unwrap(); + wait_until("forbidden QUIC removed and relay restored", || async { + let status = a.agent.network_status(id).await.ok()?; + matches!( + status + .peers + .iter() + .find(|p| p.endpoint_id == c.agent.endpoint_id())? + .data_path, + tsunagi::agent::DataPath::Relayed { .. } + ) + .then_some(()) + }) + .await; + assert_eq!(a.plugin.overview(id).unwrap().public_key, key); + tcp.receive_queued(pending).await; + tcp.exchange(b"same TCP connection after returning to AWG and the relay") + .await; + drop(tcp); + // Transit ciphertext must never be delivered to the middle agent's IP interface. + assert!( + tokio::time::timeout(Duration::from_millis(150), b.tun().pop_to_os()) + .await + .is_err() + ); + for node in [&a, &b, &c] { + node.agent.shutdown().await; + } +} + +#[tokio::test] +async fn no_common_carrier_is_an_actionable_policy_error_without_disabling_control() { + let discovery = SharedMemoryDiscovery::new(); + let roots = [TempDir::new().unwrap(), TempDir::new().unwrap()]; + let a = Node::open_selected( + roots[0].path(), + &discovery, + "awg-policy", + true, + 1280, + Some(vec!["awg".into()]), + ) + .await; + let b = Node::open_selected( + roots[1].path(), + &discovery, + "quic-policy", + true, + 1280, + Some(vec!["wg-quic".into()]), + ) + .await; + let (name, secret) = network("no-common-carrier"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_until("explain independent external policies", || async { + let status = a.agent.network_status(id).await.ok()?; + let peer = status.peers.first()?; + peer.data_error + .as_ref()? + .contains("no common external transport") + .then_some(()) + }) + .await; + assert_eq!( + a.agent + .network_status(id) + .await + .unwrap() + .metrics + .data_links_established, + 0 + ); + a.agent.shutdown().await; + b.agent.shutdown().await; +} diff --git a/crates/tsunagi-core/src/agent/mod.rs b/crates/tsunagi-core/src/agent/mod.rs index 130a26c..901de0f 100644 --- a/crates/tsunagi-core/src/agent/mod.rs +++ b/crates/tsunagi-core/src/agent/mod.rs @@ -135,6 +135,10 @@ impl Inner { .await { Ok(interface) => { + self.adapter.exclude_interface(interface.name()); + for carrier in &self.config.carriers { + carrier.on_interface_ready(interface.name()); + } let _ = self.interface.set(Arc::new(interface)); } Err(err) => { @@ -188,14 +192,13 @@ impl PacketCarrier for Carrier { let Some(inner) = self.0.upgrade() else { return false; }; - // Offered to each protocol in turn. With one configured this is a - // single call; the shape is what allows several to be live at once, - // each carrying the peers it has a link to. + // Every routed IP packet enters the same internal cipher, whatever + // physical carrier currently supplies the logical peer link. inner .config - .plugins - .iter() - .any(|plugin| plugin.carry(route.network, route.peer, packet.clone())) + .tunnel + .as_ref() + .is_some_and(|tunnel| tunnel.carry(route.network, route.peer, packet)) } } @@ -213,21 +216,6 @@ impl PacketSink for Sink { let Some(inner) = self.0.upgrade() else { return; }; - if let Some(payload) = crate::dataplane::transport::health::ip_payload(&packet) { - if let Some(header) = crate::overlay::IpHeader::parse(&packet) - && let (std::net::IpAddr::V4(source), std::net::IpAddr::V4(destination)) = - (header.source(), header.destination()) - && inner.routes.may_send_from(network, peer, source) - && inner.routes.is_local_destination(network, destination) - && let Some(handle) = inner.networks.read().await.get(&network) - { - let _ = handle.commands.try_send(NetCommand::HealthProbe { - peer, - packet: bytes::Bytes::copy_from_slice(payload), - }); - } - return; - } let Some(interface) = inner.interface.get() else { return; }; @@ -301,7 +289,7 @@ impl TransportContext for TransportCtx { } inner .config - .plugins + .carriers .iter() .any(|plugin| plugin.protocol_id() == protocol) }) @@ -359,20 +347,23 @@ impl Agent { .unwrap_or_else(|| { self.inner .config - .plugins + .carriers .iter() .map(|p| p.protocol_id().to_owned()) .collect() }) } - fn selected_plugins(&self, protocols: &[String]) -> Vec { + fn selected_carriers( + &self, + protocols: &[String], + ) -> Vec { protocols .iter() .filter_map(|name| { self.inner .config - .plugins + .carriers .iter() .find(|p| p.protocol_id() == name) .cloned() @@ -388,7 +379,7 @@ impl Agent { if !self .inner .config - .plugins + .carriers .iter() .any(|p| p.protocol_id() == name) { @@ -429,7 +420,7 @@ impl Agent { if let Some(protocols) = &config.default_protocols { crate::dataplane::validate_protocols(protocols, config.limits.max_capabilities)?; for name in protocols { - if !config.plugins.iter().any(|p| p.protocol_id() == name) { + if !config.carriers.iter().any(|p| p.protocol_id() == name) { return Err(Error::InvalidProtocols(format!( "this agent has no `{name}` plugin" ))); @@ -508,7 +499,7 @@ impl Agent { // Plugins get a handle to ask for re-announcements and report errors. // A bounded queue keeps a noisy plugin from growing memory without // bound; overflow drops the request rather than stalling the plugin. - if !inner.config.plugins.is_empty() { + if !inner.config.carriers.is_empty() || inner.config.tunnel.is_some() { let (plugin_tx, plugin_rx) = mpsc::channel(64); let sink = inner .config @@ -516,8 +507,8 @@ impl Agent { .as_ref() .map(|_| Arc::new(Sink(Arc::downgrade(&inner))) as Arc); let context = PluginContext::new(plugin_tx, inner.identity.endpoint_id(), sink); - for plugin in &inner.config.plugins { - plugin.attach(context.clone()); + if let Some(tunnel) = &inner.config.tunnel { + tunnel.attach(context); } let task = tokio::spawn(plugin_request_loop(Arc::downgrade(&inner), plugin_rx)); if let Ok(mut guard) = inner.plugin_task.lock() { @@ -1008,7 +999,7 @@ impl Agent { let broadcast = stored.as_ref().is_none_or(|stored| stored.broadcast); let protocols = self.effective_protocols(stored.as_ref().and_then(|s| s.protocols.as_ref())); - let plugins = self.selected_plugins(&protocols); + let plugins = self.selected_carriers(&protocols); if !plugins.is_empty() { self.inner.ensure_interface().await; } @@ -1032,8 +1023,9 @@ impl Agent { discovery, discovery_interval: self.inner.config.discovery_interval, discovery_policy: self.inner.config.discovery_policy.clone(), - available_plugins: self.inner.config.plugins.clone(), - plugins: plugins.clone(), + available_carriers: self.inner.config.carriers.clone(), + tunnel: self.inner.config.tunnel.clone(), + carriers: plugins.clone(), routes: Arc::clone(&self.inner.routes), interface: self.inner.interface.get().cloned(), hostname: self.inner.read_hostname(), @@ -1050,6 +1042,9 @@ impl Agent { for plugin in &plugins { plugin.on_network_activated(network_id); } + if let Some(tunnel) = &self.inner.config.tunnel { + tunnel.on_network_activated(network_id); + } let _ = self.inner.events.send(Event::NetworkActivated { network: network_id, @@ -1094,9 +1089,12 @@ impl Agent { .ok_or(Error::NetworkNotActive(network_id))? }; handle.stop().await; - for plugin in &self.inner.config.plugins { + for plugin in &self.inner.config.carriers { plugin.on_network_deactivated(network_id); } + if let Some(tunnel) = &self.inner.config.tunnel { + tunnel.on_network_deactivated(network_id); + } // The network's addresses come off the interface; the interface // itself stays, because the agent owns it and other networks may // still be using it. @@ -1129,9 +1127,12 @@ impl Agent { self.stop_network_locked(network_id).await?; } self.inner.storage.remove_network(network_id).await?; - for plugin in &self.inner.config.plugins { + for plugin in &self.inner.config.carriers { plugin.on_network_forgotten(network_id); } + if let Some(tunnel) = &self.inner.config.tunnel { + tunnel.on_network_forgotten(network_id).await; + } self.resume_networks_locked().await } @@ -1423,7 +1424,7 @@ impl Agent { // Plugins remove whatever system objects they created. A plugin that // misbehaves here must not hold up the agent, so this is bounded. - for plugin in &self.inner.config.plugins { + for plugin in &self.inner.config.carriers { if tokio::time::timeout(PLUGIN_SHUTDOWN_GRACE, plugin.shutdown()) .await .is_err() @@ -1435,6 +1436,14 @@ impl Agent { } } + if let Some(tunnel) = &self.inner.config.tunnel + && tokio::time::timeout(PLUGIN_SHUTDOWN_GRACE, tunnel.shutdown()) + .await + .is_err() + { + tracing::warn!("data encryption engine did not shut down in time"); + } + // Release the directory so another instance can claim it right away. self.inner.storage.release_ownership_lock(); } diff --git a/crates/tsunagi-core/src/agent/network.rs b/crates/tsunagi-core/src/agent/network.rs index 7b7b15a..f4102a6 100644 --- a/crates/tsunagi-core/src/agent/network.rs +++ b/crates/tsunagi-core/src/agent/network.rs @@ -15,7 +15,7 @@ use tokio::sync::{broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use crate::config::{DiscoveryPolicy, Limits, ReconnectPolicy}; -use crate::dataplane::transport::health::{HealthLink, ProbeRoute, Settings as HealthSettings}; +use crate::dataplane::transport::health::{HealthLink, Settings as HealthSettings}; use crate::dataplane::transport::{InboundLink, PacketLink, PacketTransport, SharedLink}; use crate::dataplane::{PluginCapability, SharedPlugin}; use crate::discovery::worker::DiscoveryWorker; @@ -73,10 +73,6 @@ pub(crate) enum NetCommand { }, /// A peer opened a data plane link towards us. InboundLink(Box), - HealthProbe { - peer: EndpointId, - packet: bytes::Bytes, - }, Status { reply: oneshot::Sender>, }, @@ -149,7 +145,6 @@ impl std::fmt::Debug for NetCommand { NetCommand::SetHostname(_) => f.write_str("SetHostname"), NetCommand::Release { .. } => f.write_str("Release"), NetCommand::PluginError { protocol, .. } => write!(f, "PluginError({protocol})"), - NetCommand::HealthProbe { peer, .. } => write!(f, "HealthProbe({})", peer.fmt_short()), NetCommand::ProtocolFailure { peer, protocol, .. } => { write!(f, "ProtocolFailure({}, {protocol})", peer.fmt_short()) } @@ -193,8 +188,9 @@ pub(crate) struct RuntimeParams { pub(crate) discovery: Option>, pub(crate) discovery_interval: Duration, pub(crate) discovery_policy: DiscoveryPolicy, - pub(crate) plugins: Vec, - pub(crate) available_plugins: Vec, + pub(crate) carriers: Vec, + pub(crate) available_carriers: Vec, + pub(crate) tunnel: Option, /// Who holds which overlay address, shared with every other network. pub(crate) routes: Arc, /// The one interface, when the agent has one. @@ -398,6 +394,8 @@ struct Runtime { /// The last hostname each peer announced, kept across restarts so a /// member that is away can still be called by name. known_hostnames: HashMap, + data_errors: HashMap, + tunnel_errors: HashMap, } impl Runtime { @@ -418,7 +416,7 @@ impl Runtime { // preferred when a neighbour can be reached over more than one. hub.set_priority( params - .plugins + .carriers .iter() .map(|plugin| plugin.protocol_id().to_string()) .collect(), @@ -478,6 +476,8 @@ impl Runtime { exit_via_was_online: false, announced_exit: false, known_hostnames: HashMap::new(), + data_errors: HashMap::new(), + tunnel_errors: HashMap::new(), } } @@ -629,7 +629,7 @@ impl Runtime { self.params.interface = interface; let current: Vec<_> = self .params - .plugins + .carriers .iter() .map(|p| p.protocol_id().to_owned()) .collect(); @@ -638,22 +638,22 @@ impl Runtime { for peer in &peers { self.reset_peer_data(*peer, "local protocol selection changed"); } - for plugin in &self.params.plugins { + for plugin in &self.params.carriers { if !protocols.iter().any(|name| name == plugin.protocol_id()) { plugin.on_network_deactivated(self.network_id); } } - self.params.plugins = protocols + self.params.carriers = protocols .iter() .filter_map(|name| { self.params - .available_plugins + .available_carriers .iter() .find(|p| p.protocol_id() == name) .cloned() }) .collect(); - for plugin in &self.params.plugins { + for plugin in &self.params.carriers { if !current.iter().any(|name| name == plugin.protocol_id()) { plugin.on_network_activated(self.network_id); } @@ -753,13 +753,6 @@ impl Runtime { reason, }); } - NetCommand::HealthProbe { peer, packet } => { - for ((p, _), link) in &self.links { - if *p == peer { - link.receive_probe(packet.clone()); - } - } - } NetCommand::ProtocolFailure { peer, protocol, @@ -932,7 +925,8 @@ impl Runtime { .await; } - fn add_candidate(&mut self, candidate: Candidate) { + fn add_candidate(&mut self, mut candidate: Candidate) { + candidate.addr = self.underlay_addr(candidate.addr); let peer = candidate.endpoint_id(); if peer == self.local_id { return; @@ -1891,8 +1885,8 @@ impl Runtime { return; } self.update_vips(); - for plugin in &self.params.plugins { - plugin.on_address_allocation(self.network_id, range, &allocations); + if let Some(tunnel) = &self.params.tunnel { + tunnel.on_address_allocation(self.network_id, range, &allocations); } if self.published_range != Some(range) { self.published_range = Some(range); @@ -1984,7 +1978,7 @@ impl Runtime { /// The protocols this agent speaks, with the version it speaks them at. fn served_protocols(&self) -> Vec<(String, u16)> { self.params - .plugins + .carriers .iter() .map(|plugin| (plugin.protocol_id().to_string(), plugin.protocol_version())) .collect() @@ -2045,6 +2039,8 @@ impl Runtime { // reachable through somebody is decided below; the protocol's // link stays either way. self.hub.clear_direct(peer, &protocol); + self.data_errors + .insert(peer, format!("{protocol}: {reason}")); self.record_protocol_failure(peer, &protocol); self.emit(Event::DataLinkDown { network: self.network_id, @@ -2169,6 +2165,7 @@ impl Runtime { } } + let addr = self.underlay_addr(addr); let fresh = !self.candidate_addrs.contains_key(&peer); self.candidate_addrs .entry(peer) @@ -2184,6 +2181,10 @@ impl Runtime { learned } + fn underlay_addr(&self, addr: iroh::EndpointAddr) -> iroh::EndpointAddr { + without_overlay_addresses(addr, &self.params.routes.overlay_ranges()) + } + /// Tells the peers it is talking to about the peers it knows. /// /// A device told about one member ends up talking to all of them: a @@ -2230,14 +2231,14 @@ impl Runtime { // other side's own discovery may be able to resolve it. hints.push(crate::proto::message::PeerHint { endpoint: *peer.as_bytes(), - addrs: hint_addrs(addr), + addrs: hint_addrs(&self.underlay_addr(addr.clone())), }); } // This agent, too: it is the one member the others cannot be told // about by anybody else. hints.push(crate::proto::message::PeerHint { endpoint: *self.local_id.as_bytes(), - addrs: hint_addrs(&self.params.adapter.addr()), + addrs: hint_addrs(&self.underlay_addr(self.params.adapter.addr())), }); hints.truncate(self.params.limits.max_state_records); if hints.is_empty() { @@ -2282,34 +2283,18 @@ impl Runtime { } } - /// The protocol this agent and a member run their tunnel in, if they have - /// one in common. - /// - /// Both ends must name the same one without asking each other, and what - /// each side was configured to prefer can differ. So the member with the - /// lower id decides, by the order it announces its protocols in, and the - /// other side reads that order from the announcement it already has. + /// Matches the single internal encryption capability, independently of carrier policy. fn tunnel_protocol(&self, session: &Session) -> Option { - let usable = |ours: &[PluginCapability], theirs: &[PluginCapability], name: &str| { - ours.iter().any(|capability| { - capability.enabled - && capability.protocol == name - && theirs.iter().any(|other| { - other.enabled - && other.protocol == name - && other.version == capability.version - }) - }) - }; - let (decides, other) = if self.local_id.as_bytes() < session.peer.as_bytes() { - (&self.local_capabilities, &session.capabilities) - } else { - (&session.capabilities, &self.local_capabilities) - }; - decides + let tunnel = self.params.tunnel.as_ref()?; + session + .capabilities .iter() - .find(|capability| usable(decides, other, &capability.protocol)) - .map(|capability| capability.protocol.clone()) + .any(|cap| { + cap.enabled + && cap.protocol == tunnel.protocol_id() + && cap.version == tunnel.protocol_version() + }) + .then(|| tunnel.protocol_id().to_owned()) } /// Builds the graph off the packet path. Every remote row comes from that @@ -2351,9 +2336,9 @@ impl Runtime { } let Some(plugin) = self .params - .plugins - .iter() - .find(|plugin| plugin.protocol_id() == protocol) + .tunnel + .as_ref() + .filter(|plugin| plugin.protocol_id() == protocol) .cloned() else { continue; @@ -2398,6 +2383,8 @@ impl Runtime { self.announce_reach(false); } Err(reason) => { + self.data_errors + .insert(outcome.peer, format!("{}: {reason}", outcome.protocol)); // A data plane that cannot be set up is reported, never fatal. self.metrics.data_link_failures += 1; self.record_protocol_failure(outcome.peer, &outcome.protocol); @@ -2456,7 +2443,7 @@ impl Runtime { link.close(); return; } - if !self.params.plugins.iter().any(|plugin| { + if !self.params.carriers.iter().any(|plugin| { plugin.protocol_id() == protocol && self.sessions.get(&peer).is_some_and(|s| { s.capabilities.iter().any(|c| { @@ -2470,6 +2457,7 @@ impl Runtime { return; } + self.data_errors.remove(&peer); let key = (peer, protocol.clone()); let settings = if self.local_id.as_bytes() < peer.as_bytes() { let settings = HealthSettings::fresh(); @@ -2486,21 +2474,7 @@ impl Runtime { } else { self.pending_health.remove(&key) }; - let route = self - .sessions - .get(&peer) - .and_then(|s| self.tunnel_protocol(s)) - .and_then(|name| self.params.plugins.iter().find(|p| p.protocol_id() == name)) - .map(|plugin| ProbeRoute { - plugin: plugin.clone(), - routes: self.params.routes.clone(), - network: self.network_id, - peer, - max_probe_size: self.params.interface.as_ref().map_or(540, |interface| { - (interface.mtu() as usize).saturating_sub(36) - }), - }); - let link = HealthLink::new(link, self.local_id, settings, route); + let link = HealthLink::new(link, self.local_id, settings); let path = link.path_description(); let max_datagram = link.max_datagram_size(); if let Some(old) = self.links.insert(key, Arc::clone(&link)) { @@ -2524,15 +2498,21 @@ impl Runtime { /// Drops every link to a peer. fn drop_links_for(&mut self, peer: EndpointId) { + self.tunnel_errors.remove(&peer); self.retirement_cursor.remove(&peer); self.vip_cursor.remove(&peer); self.reset_peer_data(peer, "peer session ended"); + if let Some(tunnel) = &self.params.tunnel { + tunnel.on_peer_gone(self.network_id, peer); + } + self.hub.clear_peer_links(peer); self.hub.remove_peer(peer); self.reachable.remove(&peer); } /// Retire only the data plane; authenticated membership and topology remain. fn reset_peer_data(&mut self, peer: EndpointId, reason: &str) { + self.data_errors.remove(&peer); self.link_attempts.retain(|(id, _), _| *id != peer); self.opening.retain(|(id, _)| *id != peer); self.link_tasks.retain(|(id, _), task| { @@ -2543,7 +2523,6 @@ impl Runtime { true } }); - self.hub.clear_peer_links(peer); let keys: Vec<_> = self .links .keys() @@ -2554,6 +2533,7 @@ impl Runtime { if let Some(link) = self.links.remove(&key) { link.close(); } + self.hub.clear_direct(peer, &key.1); self.emit(Event::DataLinkDown { network: self.network_id, peer, @@ -2564,7 +2544,7 @@ impl Runtime { self.protocol_dial_states.retain(|(id, _), _| *id != peer); self.last_failed_protocol.remove(&peer); self.pending_health.retain(|(id, _), _| *id != peer); - for plugin in &self.params.plugins { + for plugin in &self.params.carriers { plugin.on_peer_gone(self.network_id, peer); } } @@ -2650,11 +2630,27 @@ impl Runtime { fn local_announcement(&mut self) -> Announcement { let mut capabilities = Vec::new(); let mut errors = Vec::new(); - for plugin in &self.params.plugins { - match plugin.local_capability(self.network_id) { - Ok(Some(capability)) => capabilities.push(capability), + let sources = self + .params + .carriers + .iter() + .map(|carrier| { + ( + carrier.protocol_id().to_owned(), + carrier.local_capability(self.network_id), + ) + }) + .chain(self.params.tunnel.iter().map(|tunnel| { + ( + tunnel.protocol_id().to_owned(), + tunnel.local_capability(self.network_id), + ) + })); + for (protocol, result) in sources { + match result { + Ok(Some(cap)) => capabilities.push(cap), Ok(None) => {} - Err(err) => errors.push((plugin.protocol_id().to_string(), err.to_string())), + Err(err) => errors.push((protocol, err.to_string())), } } for (protocol, reason) in errors { @@ -2922,34 +2918,57 @@ impl Runtime { fn dispatch_capabilities(&mut self, peer: EndpointId, capabilities: &[PluginCapability]) { let mut errors = Vec::new(); - for capability in capabilities { - for plugin in &self.params.plugins { - if plugin.protocol_id() != capability.protocol { - continue; - } - if plugin.protocol_version() != capability.version { - // Said once per peer and protocol. A version that will - // not match this build will not match it on the next - // announcement either, and repeating it every round - // would bury everything else. - let key = (peer, capability.protocol.clone()); - if self.reported_mismatch.insert(key) { - errors.push(( - capability.protocol.clone(), - format!( - "peer speaks {} version {} and this build speaks {}, so there \ - is no data plane with it; the control plane is unaffected", - capability.protocol, - capability.version, - plugin.protocol_version() - ), - )); + for cap in capabilities { + let carriers = self + .params + .carriers + .iter() + .filter(|carrier| carrier.protocol_id() == cap.protocol) + .map(|carrier| { + ( + carrier.protocol_version(), + if carrier.protocol_version() == cap.version { + carrier.on_peer_capability(self.network_id, peer, cap) + } else { + Ok(()) + }, + ) + }); + let tunnel = self + .params + .tunnel + .iter() + .filter(|tunnel| tunnel.protocol_id() == cap.protocol) + .map(|tunnel| { + ( + tunnel.protocol_version(), + if tunnel.protocol_version() == cap.version { + tunnel.on_peer_capability(self.network_id, peer, cap) + } else { + Ok(()) + }, + ) + }); + for (version, result) in carriers.chain(tunnel) { + if version != cap.version { + if self.reported_mismatch.insert((peer, cap.protocol.clone())) { + errors.push((cap.protocol.clone(), format!("peer speaks {} version {} and this build speaks {version}; update both agents; control plane is unaffected", cap.protocol, cap.version))); + } + } else { + let is_tunnel = self + .params + .tunnel + .as_ref() + .is_some_and(|t| t.protocol_id() == cap.protocol); + if let Err(err) = result { + if is_tunnel { + self.tunnel_errors + .insert(peer, format!("encrypted data channel: {err}")); + } + errors.push((cap.protocol.clone(), err.to_string())); + } else if is_tunnel { + self.tunnel_errors.remove(&peer); } - continue; - } - // The core hands the opaque payload over without interpreting it. - if let Err(err) = plugin.on_peer_capability(self.network_id, peer, capability) { - errors.push((plugin.protocol_id().to_string(), err.to_string())); } } } @@ -3028,6 +3047,58 @@ impl Runtime { }) } + fn data_error(&self, session: &Session) -> Option { + if self.params.carriers.is_empty() { + return Some("data transports disabled locally".into()); + } + if self.params.tunnel.is_some() && self.tunnel_protocol(session).is_none() { + return Some( + "peer has no compatible encrypted data channel; update both agents".into(), + ); + } + if let Some(reason) = self.tunnel_errors.get(&session.peer) { + return Some(reason.clone()); + } + if !matches!(self.data_path(session.peer), DataPath::None) { + return None; + } + let served = self.served_protocols(); + if !served.iter().any(|(name, version)| { + session + .capabilities + .iter() + .any(|cap| cap.enabled && cap.protocol == *name && cap.version == *version) + }) { + let local = served + .iter() + .map(|(name, _)| name.as_str()) + .collect::>() + .join(","); + let remote = session + .capabilities + .iter() + .filter(|cap| { + cap.enabled + && self + .params + .tunnel + .as_ref() + .is_none_or(|t| t.protocol_id() != cap.protocol) + }) + .map(|cap| cap.protocol.as_str()) + .collect::>() + .join(","); + return Some(format!( + "no common external transport (local: {local}; peer: {})", + if remote.is_empty() { "none" } else { &remote } + )); + } + self.data_errors + .get(&session.peer) + .cloned() + .or_else(|| Some("establishing external data transport".into())) + } + fn status(&self) -> NetworkStatus { let served = self.served_protocols(); let mut peers: Vec = self @@ -3055,6 +3126,7 @@ impl Runtime { }) .map(|capability| capability.protocol.clone()) .collect(), + data_error: self.data_error(session), capabilities: session.capabilities.clone(), connected_for: session.established.elapsed(), paths: snapshot.paths, @@ -3117,7 +3189,7 @@ impl Runtime { vips: self.params.routes.vips.statuses(self.network_id), protocols: self .params - .plugins + .carriers .iter() .map(|p| p.protocol_id().to_owned()) .collect(), @@ -3265,10 +3337,43 @@ fn merge_addr(existing: &mut iroh::EndpointAddr, incoming: &iroh::EndpointAddr) } } +/// Introductions and discovery candidates are underlay addresses. An overlay +/// address here would send the tunnel's own transport recursively through itself. +fn without_overlay_addresses( + mut addr: iroh::EndpointAddr, + ranges: &[Ipv4Range], +) -> iroh::EndpointAddr { + addr.addrs.retain(|address| match address { + iroh::TransportAddr::Ip(socket) => match socket.ip() { + std::net::IpAddr::V4(ip) => !ranges.iter().any(|range| range.contains(ip)), + std::net::IpAddr::V6(_) => true, + }, + _ => true, + }); + addr +} + #[cfg(test)] mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::*; + #[test] + fn underlay_hints_exclude_overlay_subnets_but_preserve_lan_and_relay_candidates() { + let peer = crate::identity::DeviceIdentity::generate().endpoint_id(); + let overlay = "10.13.37.99:1234".parse().unwrap(); + let lan = "192.168.1.5:1234".parse().unwrap(); + let relay: iroh::RelayUrl = "https://relay.example.com".parse().unwrap(); + let addr = iroh::EndpointAddr::new(peer) + .with_ip_addr(overlay) + .with_ip_addr(lan) + .with_relay_url(relay.clone()); + let filtered = without_overlay_addresses(addr, &[crate::state::DEFAULT_IPV4_RANGE]); + assert_eq!(filtered.ip_addrs().copied().collect::>(), vec![lan]); + assert!(filtered.addrs.contains(&iroh::TransportAddr::Relay(relay))); + assert_eq!(filtered.id, peer); + } + #[test] fn test_data_link_backoff_progression() { assert_eq!(data_link_backoff(0), Duration::ZERO); diff --git a/crates/tsunagi-core/src/agent/status.rs b/crates/tsunagi-core/src/agent/status.rs index aa6b73c..b3c1b22 100644 --- a/crates/tsunagi-core/src/agent/status.rs +++ b/crates/tsunagi-core/src/agent/status.rs @@ -158,6 +158,8 @@ pub struct PeerStatus { pub protocols: Vec, /// Capabilities the peer announced. Payloads stay opaque. pub capabilities: Vec, + /// Why no data route is available, when known. + pub data_error: Option, /// How long the session has been up. pub connected_for: Duration, /// Verified paths of the underlying connection. diff --git a/crates/tsunagi-core/src/config.rs b/crates/tsunagi-core/src/config.rs index ee43ae7..7981593 100644 --- a/crates/tsunagi-core/src/config.rs +++ b/crates/tsunagi-core/src/config.rs @@ -353,10 +353,12 @@ pub struct AgentConfig { pub limits: Limits, /// Reconnect backoff policy. pub reconnect: ReconnectPolicy, - /// IP plugins whose capabilities are announced and dispatched. - pub plugins: Vec, + /// External data carriers whose capabilities are announced and dispatched. + pub carriers: Vec, + /// The single internal end-to-end engine, independent of carrier selection. + pub tunnel: Option, /// Initial ordered selection for networks without a saved override. - /// `None` enables all registered plugins; an empty list is control-only. + /// `None` enables all registered carriers; an empty list is control-only. pub default_protocols: Option>, /// Custom packet transports registered per protocol. pub custom_transports: @@ -405,7 +407,8 @@ impl AgentConfig { discovery_interval: Duration::from_secs(5), limits: Limits::default(), reconnect: ReconnectPolicy::default(), - plugins: Vec::new(), + carriers: Vec::new(), + tunnel: None, default_protocols: None, custom_transports: std::collections::HashMap::new(), tun_factory: None, @@ -489,9 +492,15 @@ impl AgentConfig { self } - /// Registers an IP plugin. - pub fn with_plugin(mut self, plugin: SharedPlugin) -> Self { - self.plugins.push(plugin); + /// Registers an external data carrier. + pub fn with_carrier(mut self, plugin: crate::dataplane::transport::SharedCarrier) -> Self { + self.carriers.push(plugin); + self + } + + /// Installs the internal encryption engine. It is never a selectable carrier. + pub fn with_tunnel(mut self, tunnel: SharedPlugin) -> Self { + self.tunnel = Some(tunnel); self } @@ -545,9 +554,9 @@ impl std::fmt::Debug for AgentConfig { .field("limits", &self.limits) .field("reconnect", &self.reconnect) .field( - "plugins", + "carriers", &self - .plugins + .carriers .iter() .map(|p| p.protocol_id().to_string()) .collect::>(), diff --git a/crates/tsunagi-core/src/dataplane/mod.rs b/crates/tsunagi-core/src/dataplane/mod.rs index 79daabd..3e6b9fe 100644 --- a/crates/tsunagi-core/src/dataplane/mod.rs +++ b/crates/tsunagi-core/src/dataplane/mod.rs @@ -1,28 +1,19 @@ -//! Boundary between the control plane core and future IP plugins. +//! Boundaries between the agent, one IP encryption tunnel, and external carriers. //! -//! The data plane is where actual IP connectivity is created. WireGuard is the -//! first planned plugin; none is implemented here. +//! The core moves bounded opaque capabilities without parsing cipher payloads. +//! An [`IpPlugin`] sees routed IP packets and a logical [`PacketLink`] per peer; +//! it never learns reachability or the network secret. The agent configures one +//! such plugin. Built-in agents use the same internal WireGuard cipher for every +//! peer, regardless of which external transports carry the encrypted bytes. //! -//! Two rules shape this module: -//! -//! 1. **The core never parses plugin payloads.** A [`PluginCapability`] carries -//! a protocol id, a version, an enabled flag and a bounded opaque blob. The -//! core transports the blob and hands it to the matching plugin. It does not -//! know what a WireGuard configuration looks like. -//! 2. **Plugin keys and lifecycle are separate from iroh identity and from the -//! network secret.** A plugin owns its own keys and its own system objects. -//! -//! An iroh address is *not* automatically a WireGuard address. A future plugin -//! is expected to gather its own reachability information and ship it through -//! the control plane as its announcement payload. -//! -//! A data plane failure never stops the daemon: errors returned here are -//! recorded and surfaced, the control plane keeps running. +//! [`transport::DataCarrier`] owns external capability negotiation and lifecycle. +//! It sees reachability but never plaintext overlay packets. A relay forwards +//! the encrypted inner bytes without entering an IP plugin or the TUN. +//! Data-plane failure never stops control sessions or discovery. pub mod relay; pub mod routing; pub mod transport; -pub mod tunnel; use std::sync::Arc; @@ -35,7 +26,6 @@ use crate::identity::NetworkId; pub use transport::{ MultiPacketTransport, PacketLink, PacketTransport, SharedLink, TransportError, }; -pub use tunnel::{GenericTunnelPlugin, TunnelCodec}; /// Maximum length of a plugin protocol identifier. pub const MAX_PROTOCOL_ID_LEN: usize = 32; @@ -438,7 +428,9 @@ pub trait IpPlugin: Send + Sync + std::fmt::Debug + 'static { /// Always preceded by [`IpPlugin::on_network_deactivated`], so this is /// only about what outlives a session. Defaulted to nothing, for a /// plugin that stores nothing. - fn on_network_forgotten(&self, _network: NetworkId) {} + fn on_network_forgotten(&self, _network: NetworkId) -> BoxFuture<'_, ()> { + Box::pin(async {}) + } /// Called once when the agent shuts down. /// diff --git a/crates/tsunagi-core/src/dataplane/transport/carrier.rs b/crates/tsunagi-core/src/dataplane/transport/carrier.rs new file mode 100644 index 0000000..d2f0a01 --- /dev/null +++ b/crates/tsunagi-core/src/dataplane/transport/carrier.rs @@ -0,0 +1,44 @@ +//! External link negotiation, deliberately separate from the IP encryption plugin. +use crate::dataplane::{PluginCapability, PluginError, ProtocolOption}; +use crate::{BoxFuture, NetworkId}; +use iroh::EndpointId; +use std::sync::Arc; + +/// Capability and lifecycle of one external data carrier. +/// Only this layer sees reachability. It never receives plaintext overlay IP packets. +pub trait DataCarrier: Send + Sync + std::fmt::Debug + 'static { + /// External protocol identifier used in local policy and peer negotiation. + fn protocol_id(&self) -> &str; + /// Wire version, independent of software releases and the internal tunnel. + fn protocol_version(&self) -> u16; + /// Supported carrier settings. + fn options(&self) -> &'static [ProtocolOption] { + &[] + } + /// Bounded opaque reachability/negotiation capability for one network. + fn local_capability(&self, network: NetworkId) + -> Result, PluginError>; + /// Installs a capability received through an authenticated membership session. + fn on_peer_capability( + &self, + network: NetworkId, + peer: EndpointId, + capability: &PluginCapability, + ) -> Result<(), PluginError>; + /// Supplies the actual owned interface name, including kernel-assigned names. + fn on_interface_ready(&self, _name: &str) {} + /// Activates this carrier for one local network. + fn on_network_activated(&self, _network: NetworkId) {} + /// Retires all negotiation and links for this member. + fn on_peer_gone(&self, network: NetworkId, peer: EndpointId); + /// Retires this network's sockets and negotiations. + fn on_network_deactivated(&self, network: NetworkId); + /// Forgets durable carrier settings, if any. + fn on_network_forgotten(&self, _network: NetworkId) {} + /// Stops carrier tasks, with the agent enforcing a shutdown deadline. + fn shutdown(&self) -> BoxFuture<'_, ()> { + Box::pin(async {}) + } +} +/// One shared carrier adapter. +pub type SharedCarrier = Arc; diff --git a/crates/tsunagi-core/src/dataplane/transport/health.rs b/crates/tsunagi-core/src/dataplane/transport/health.rs index e2a63ef..788a9ae 100644 --- a/crates/tsunagi-core/src/dataplane/transport/health.rs +++ b/crates/tsunagi-core/src/dataplane/transport/health.rs @@ -1,8 +1,6 @@ //! Authenticated probes of the actual data link, independent of control RTT. //! Keys are negotiated privately for each physical link. No fixed wire magic. use super::{PacketLink, SharedLink, TransportError}; -use crate::dataplane::SharedPlugin; -use crate::overlay::RoutingTable; use crate::{BoxFuture, NetworkId}; use bytes::Bytes; use hmac::{Hmac, KeyInit, Mac}; @@ -70,52 +68,8 @@ impl Samples { && self.recent.iter().filter(|ok| !**ok).count() * 100 >= self.recent.len() * 60 } } -// This prefix is inside end-to-end encryption. It never appears on the UDP wire. -const IP_MAGIC: &[u8; 16] = b"tsunagi-health01"; -pub(crate) fn ip_payload(packet: &[u8]) -> Option<&[u8]> { - (packet.len() >= 36 && packet[0] == 0x45 && packet[9] == 253 && &packet[20..36] == IP_MAGIC) - .then(|| &packet[36..]) -} -/// Inject probes through the same encrypted tunnel as application IP packets. -/// This matters for pure WG: raw UDP probes could pass DPI while WG is blocked. -pub(crate) struct ProbeRoute { - pub(crate) plugin: SharedPlugin, - pub(crate) routes: Arc, - pub(crate) network: NetworkId, - pub(crate) peer: EndpointId, - pub(crate) max_probe_size: usize, -} -impl ProbeRoute { - fn send(&self, payload: Bytes) -> bool { - let Some((source, destination)) = self.routes.probe_addresses(self.network, self.peer) - else { - return false; - }; - let mut packet = vec![0; 36]; - packet[0] = 0x45; - packet[8] = 64; - packet[9] = 253; - packet[12..16].copy_from_slice(&source.octets()); - packet[16..20].copy_from_slice(&destination.octets()); - packet[20..36].copy_from_slice(IP_MAGIC); - let length = (packet.len() + payload.len()) as u16; - packet[2..4].copy_from_slice(&length.to_be_bytes()); - let sum = packet[..20] - .as_chunks::<2>() - .0 - .iter() - .map(|b| u32::from(u16::from_be_bytes(*b))) - .sum::(); - let sum = (sum & 0xffff) + (sum >> 16); - packet[10..12].copy_from_slice(&(!(sum as u16)).to_be_bytes()); - packet.extend_from_slice(&payload); - self.plugin - .carry(self.network, self.peer, Bytes::from(packet)) - } -} /// A monitored physical link. It forwards application bytes unchanged. pub(crate) struct HealthLink { - probes: mpsc::Sender, raw: SharedLink, settings: Mutex>, rx: tokio::sync::Mutex>, @@ -136,17 +90,10 @@ fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { } } impl HealthLink { - pub(crate) fn new( - raw: SharedLink, - local: EndpointId, - settings: Option, - route: Option, - ) -> Arc { + pub(crate) fn new(raw: SharedLink, local: EndpointId, settings: Option) -> Arc { let (tx, rx) = mpsc::channel(64); - let (probe_tx, mut probe_rx) = mpsc::channel(64); let (stop, _) = watch::channel(false); let link = Arc::new(Self { - probes: probe_tx, raw: raw.clone(), settings: Mutex::new(settings), rx: tokio::sync::Mutex::new(rx), @@ -174,29 +121,21 @@ impl HealthLink { *lock(&link.reason)=Some("data delivery checks: at least 60% loss over the last 20 probes".into());break; } sequence=sequence.wrapping_add(1); - let size=route.as_ref().map_or(raw.max_datagram_size(),|route|route.max_probe_size); + let size=raw.max_datagram_size(); let packet=probe(&settings,local,1,sequence,size.min(1000)); - if let Some(route)=&route { if route.send(packet) {samples.sent(sequence,now);} } - else { let _=raw.send(packet); samples.sent(sequence,now); } + samples.sent(sequence,now); + let _=raw.send(packet); } packet=raw.recv()=>{ let Some(packet)=packet else{break;};let Some(link)=weak.upgrade()else{break;}; let settings=lock(&link.settings).clone(); - if route.is_none() && let Some(settings)=settings && packet.starts_with(&settings.marker) { + if let Some(settings)=settings && packet.starts_with(&settings.marker) { if let Some((kind,seq))=verify(&settings,raw.peer(),&packet) { if kind==1 {let _=raw.send(probe(&settings,local,2,seq,packet.len()));} else if kind==2 {samples.acknowledge(seq);} } } else {let _=tx.try_send(packet);} } - packet=probe_rx.recv(),if route.is_some()=>{ - let Some(packet)=packet else {break;};let Some(link)=weak.upgrade()else{break;}; - let Some(settings)=lock(&link.settings).clone()else{continue;}; - if let Some((kind,seq))=verify(&settings,raw.peer(),&packet) { - if kind==1 { if let Some(route)=&route { route.send(probe(&settings,local,2,seq,packet.len().min(route.max_probe_size))); } } - else if kind==2 {samples.acknowledge(seq);} - } - } } } @@ -207,9 +146,6 @@ impl HealthLink { }); link } - pub(crate) fn receive_probe(&self, packet: Bytes) { - let _ = self.probes.try_send(packet); - } pub(crate) fn configure(&self, settings: Settings) { *lock(&self.settings) = Some(settings); } diff --git a/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs b/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs index 4cbd90f..7362772 100644 --- a/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs +++ b/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs @@ -40,6 +40,49 @@ use crate::proto::{read_frame, write_frame}; use super::fragments::{self, Reassembler}; use super::{InboundLink, PacketLink, PacketTransport, SharedLink, TransportError}; +/// QUIC carrier capability. Encryption of end-to-end IP packets is independent. +#[derive(Debug, Default)] +pub struct IrohCarrier; + +impl super::DataCarrier for IrohCarrier { + fn protocol_id(&self) -> &str { + "wg-quic" + } + fn protocol_version(&self) -> u16 { + 1 + } + fn local_capability( + &self, + _: NetworkId, + ) -> Result, crate::dataplane::PluginError> { + Ok(Some(crate::dataplane::PluginCapability { + protocol: self.protocol_id().into(), + version: self.protocol_version(), + enabled: true, + data: Vec::new(), + })) + } + fn on_peer_capability( + &self, + _: NetworkId, + _: EndpointId, + capability: &crate::dataplane::PluginCapability, + ) -> Result<(), crate::dataplane::PluginError> { + if capability.protocol != self.protocol_id() + || capability.version != self.protocol_version() + || !capability.enabled + || !capability.data.is_empty() + { + return Err(crate::dataplane::PluginError::Rejected( + "unsupported QUIC carrier capability".into(), + )); + } + Ok(()) + } + fn on_peer_gone(&self, _: NetworkId, _: EndpointId) {} + fn on_network_deactivated(&self, _: NetworkId) {} +} + /// What the iroh transport needs from the agent. /// /// Implemented by the agent, which is the only thing that knows which networks @@ -190,6 +233,10 @@ impl PacketLink for IrohLink { self.conn.close_reason().is_some() } + fn failure_reason(&self) -> Option { + self.conn.close_reason().map(|reason| reason.to_string()) + } + fn path_description(&self) -> String { // Report what iroh actually knows, never a guess. let snapshot = crate::net::snapshot_connection(&self.conn); @@ -264,7 +311,8 @@ impl IrohTransport { )); } - let serves = self.lookup.serves(outcome.network_id, &open.protocol).await; + let serves = open.protocol == "wg-quic" + && self.lookup.serves(outcome.network_id, &open.protocol).await; if open.max_datagram == 0 || conn.max_datagram_size().is_none() { return Err(TransportError::Other( "data channel has no datagram support".into(), @@ -325,6 +373,12 @@ impl PacketTransport for IrohTransport { "data channel protocol id is out of bounds".into(), )); } + // This default must never impersonate a missing native carrier. + if protocol != "wg-quic" { + return Err(TransportError::Declined(format!( + "no native `{protocol}` data transport registered" + ))); + } let keys = self .lookup .snapshot() diff --git a/crates/tsunagi-core/src/dataplane/transport/mod.rs b/crates/tsunagi-core/src/dataplane/transport/mod.rs index 708fd30..c6b346d 100644 --- a/crates/tsunagi-core/src/dataplane/transport/mod.rs +++ b/crates/tsunagi-core/src/dataplane/transport/mod.rs @@ -33,6 +33,9 @@ mod fragments; pub(crate) mod health; pub mod iroh_link; +pub mod carrier; +pub use carrier::{DataCarrier, SharedCarrier}; + use std::sync::Arc; use bytes::Bytes; diff --git a/crates/tsunagi-core/src/dataplane/tunnel.rs b/crates/tsunagi-core/src/dataplane/tunnel.rs deleted file mode 100644 index 41a5597..0000000 --- a/crates/tsunagi-core/src/dataplane/tunnel.rs +++ /dev/null @@ -1,380 +0,0 @@ -//! Generic harness for IP tunnel plugins. -//! -//! Implementing [`super::IpPlugin`] directly requires managing per-peer background -//! tasks, synchronization, packet queues, and graceful shutdown. -//! -//! [`TunnelCodec`] provides a simplified interface where a plugin author only -//! defines cryptographic transformation and handshake payloads. -//! [`GenericTunnelPlugin`] adapts any [`TunnelCodec`] into a full [`super::IpPlugin`]. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; - -use bytes::Bytes; -use iroh::EndpointId; -use tokio::task::JoinHandle; - -use super::{IpPlugin, PluginCapability, PluginContext, PluginError, ProtocolOption, SharedLink}; -use crate::BoxFuture; -use crate::identity::NetworkId; - -/// The simplified contract for an IP tunnel protocol implementation. -/// -/// Implementors only handle key announcements, link lifecycle events, -/// and packet encrypt/decrypt. Concurrency and task management are handled -/// by [`GenericTunnelPlugin`]. -pub trait TunnelCodec: Send + Sync + 'static { - /// Protocol identifier, e.g. `tcp-tls`. - fn protocol_id(&self) -> &str; - - /// Protocol wire version. - fn protocol_version(&self) -> u16; - - /// Options accepted by the protocol. - fn options(&self) -> &'static [ProtocolOption] { - &[] - } - - /// Generates local capability announcement data for a network. - fn local_capability_data(&self, network: NetworkId) -> Result, PluginError>; - - /// Handles a peer's announced capability payload. - fn on_peer_capability_data( - &self, - network: NetworkId, - peer: EndpointId, - data: &[u8], - ) -> Result<(), PluginError>; - - /// Called when an authenticated link to a peer is established. - fn on_peer_link_up(&self, network: NetworkId, peer: EndpointId, link: &SharedLink) { - let _ = (network, peer, link); - } - - /// Called when a peer goes away. - fn on_peer_down(&self, network: NetworkId, peer: EndpointId) { - let _ = (network, peer); - } - - /// Encrypts an outbound IP packet into a payload to send over the link. - fn encrypt( - &self, - network: NetworkId, - peer: EndpointId, - packet: &[u8], - ) -> Result; - - /// Decrypts an inbound payload from the link into an IP packet. - fn decrypt( - &self, - network: NetworkId, - peer: EndpointId, - payload: &[u8], - ) -> Result; - - /// Called during shutdown. - fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> { - Box::pin(async {}) - } -} - -struct TunnelPeerState { - link: SharedLink, - tx_packets: AtomicU64, - rx_packets: AtomicU64, - tx_bytes: AtomicU64, - rx_bytes: AtomicU64, - dropped: AtomicU64, - established_at: std::time::Instant, -} - -/// Statistics and status for one active peer in [`GenericTunnelPlugin`]. -#[derive(Debug, Clone)] -pub struct TunnelPeerReport { - /// Peer endpoint identity. - pub peer: EndpointId, - /// Protocol identifier. - pub protocol: String, - /// Human-readable path description. - pub path: String, - /// Packets successfully encrypted and sent. - pub tx_packets: u64, - /// Packets successfully received and decrypted. - pub rx_packets: u64, - /// Plaintext bytes successfully encrypted and sent. - pub tx_bytes: u64, - /// Plaintext bytes successfully received and decrypted. - pub rx_bytes: u64, - /// Packets dropped due to encryption/decryption/send failures. - pub dropped: u64, - /// Uptime in seconds since the data link was established. - pub uptime_secs: u64, -} - -/// Generic wrapper that turns a [`TunnelCodec`] into a full [`IpPlugin`]. -pub struct GenericTunnelPlugin { - codec: Arc, - context: Mutex>, - links: RwLock>>, - tasks: Mutex>>, -} - -impl GenericTunnelPlugin { - /// Wraps `codec` into an [`IpPlugin`]. - pub fn new(codec: C) -> Self { - Self { - codec: Arc::new(codec), - context: Mutex::new(None), - links: RwLock::new(HashMap::new()), - tasks: Mutex::new(HashMap::new()), - } - } - - /// Returns a reference to the underlying codec. - pub fn codec(&self) -> &C { - &self.codec - } - - /// Returns the active peers with established data links for a network. - pub fn active_peers(&self, network: NetworkId) -> Vec { - let guard = match self.links.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard - .iter() - .filter(|((net, _), _)| *net == network) - .map(|((_, peer), state)| TunnelPeerReport { - peer: *peer, - protocol: self.codec.protocol_id().to_string(), - path: state.link.path_description(), - tx_packets: state.tx_packets.load(Ordering::Relaxed), - rx_packets: state.rx_packets.load(Ordering::Relaxed), - tx_bytes: state.tx_bytes.load(Ordering::Relaxed), - rx_bytes: state.rx_bytes.load(Ordering::Relaxed), - dropped: state.dropped.load(Ordering::Relaxed), - uptime_secs: state.established_at.elapsed().as_secs(), - }) - .collect() - } -} - -impl std::fmt::Debug for GenericTunnelPlugin { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("GenericTunnelPlugin") - .field("protocol_id", &self.codec.protocol_id()) - .field("protocol_version", &self.codec.protocol_version()) - .finish() - } -} - -impl IpPlugin for GenericTunnelPlugin { - fn protocol_id(&self) -> &str { - self.codec.protocol_id() - } - - fn protocol_version(&self) -> u16 { - self.codec.protocol_version() - } - - fn options(&self) -> &'static [ProtocolOption] { - self.codec.options() - } - - fn attach(&self, context: PluginContext) { - if let Ok(mut guard) = self.context.lock() { - *guard = Some(context); - } - } - - fn local_capability( - &self, - network: NetworkId, - ) -> Result, PluginError> { - let data = self.codec.local_capability_data(network)?; - Ok(Some(PluginCapability { - protocol: self.codec.protocol_id().to_string(), - version: self.codec.protocol_version(), - enabled: true, - data, - })) - } - - fn on_peer_capability( - &self, - network: NetworkId, - peer: EndpointId, - capability: &PluginCapability, - ) -> Result<(), PluginError> { - if capability.protocol != self.codec.protocol_id() { - return Ok(()); - } - self.codec - .on_peer_capability_data(network, peer, &capability.data) - } - - fn carry(&self, network: NetworkId, peer: EndpointId, packet: bytes::Bytes) -> bool { - let state = { - let guard = match self.links.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.get(&(network, peer)).cloned() - }; - - let Some(state) = state else { - return false; - }; - - match self.codec.encrypt(network, peer, &packet) { - Ok(ciphertext) => { - if state.link.send(ciphertext).is_ok() { - state.tx_packets.fetch_add(1, Ordering::Relaxed); - state - .tx_bytes - .fetch_add(packet.len() as u64, Ordering::Relaxed); - true - } else { - state.dropped.fetch_add(1, Ordering::Relaxed); - false - } - } - Err(err) => { - state.dropped.fetch_add(1, Ordering::Relaxed); - tracing::debug!(%peer, %err, "tunnel encrypt failed"); - false - } - } - } - - fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) { - let state = Arc::new(TunnelPeerState { - link: Arc::clone(&link), - tx_packets: AtomicU64::new(0), - rx_packets: AtomicU64::new(0), - tx_bytes: AtomicU64::new(0), - rx_bytes: AtomicU64::new(0), - dropped: AtomicU64::new(0), - established_at: std::time::Instant::now(), - }); - { - let mut guard = match self.links.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.insert((network, peer), Arc::clone(&state)); - } - - self.codec.on_peer_link_up(network, peer, &link); - - let codec = Arc::clone(&self.codec); - let sink = match self.context.lock() { - Ok(g) => g.as_ref().map(|ctx| ctx.packet_sink()), - Err(p) => p.into_inner().as_ref().map(|ctx| ctx.packet_sink()), - }; - - let rx_link = Arc::clone(&link); - let rx_state = Arc::clone(&state); - let handle = tokio::spawn(async move { - while let Some(payload) = rx_link.recv().await { - match codec.decrypt(network, peer, &payload) { - Ok(packet) => { - rx_state.rx_packets.fetch_add(1, Ordering::Relaxed); - rx_state - .rx_bytes - .fetch_add(packet.len() as u64, Ordering::Relaxed); - if let Some(ref sink) = sink { - sink.deliver(network, peer, packet).await; - } - } - Err(err) => { - rx_state.dropped.fetch_add(1, Ordering::Relaxed); - tracing::debug!(%peer, %err, "tunnel decrypt failed"); - } - } - } - }); - - let mut guard = match self.tasks.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(old) = guard.insert((network, peer), handle) { - old.abort(); - } - } - - fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) { - { - let mut guard = match self.tasks.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(task) = guard.remove(&(network, peer)) { - task.abort(); - } - } - { - let mut guard = match self.links.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.remove(&(network, peer)); - } - self.codec.on_peer_down(network, peer); - } - - fn on_network_deactivated(&self, network: NetworkId) { - let keys_to_remove: Vec<(NetworkId, EndpointId)> = { - let guard = match self.tasks.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard - .keys() - .filter(|(net, _)| *net == network) - .copied() - .collect() - }; - - { - let mut guard = match self.tasks.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - for key in &keys_to_remove { - if let Some(task) = guard.remove(key) { - task.abort(); - } - } - } - - { - let mut guard = match self.links.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.retain(|(net, _), _| *net != network); - } - } - - fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> { - Box::pin(async move { - let mut tasks = Vec::new(); - { - let mut guard = match self.tasks.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - for (_, task) in guard.drain() { - tasks.push(task); - } - } - for task in tasks { - task.abort(); - } - self.codec.shutdown().await; - }) - } -} diff --git a/crates/tsunagi-core/src/ipc/mod.rs b/crates/tsunagi-core/src/ipc/mod.rs index ddbd058..3cdfc7c 100644 --- a/crates/tsunagi-core/src/ipc/mod.rs +++ b/crates/tsunagi-core/src/ipc/mod.rs @@ -828,6 +828,8 @@ pub struct MemberReport { /// One control plane peer. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct PeerReport { + /// Actionable carrier failure or policy mismatch; absent for a usable route. + pub data_error: Option, /// The peer's endpoint id. pub endpoint_id: String, /// Hostname it announced, if any. @@ -917,10 +919,12 @@ pub struct HostRulesReport { /// One overlay peer. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OverlayPeerReport { + /// Opaque counter lifetime. A new tunnel or agent restart starts a new epoch. + pub counter_epoch: u64, /// The peer's control plane identity, so a tunnel can be matched to the /// session and the member it belongs to. pub endpoint_id: String, - /// Protocol carrying overlay traffic for this peer (e.g. "tcp-tls", "wg-quic"). + /// Reserved legacy field; the internal encryption protocol is not user selectable. pub protocol: String, /// The peer's WireGuard public key (if applicable). pub public_key: String, @@ -1131,7 +1135,7 @@ pub const EXCHANGE_TIMEOUT: Duration = Duration::from_secs(5); /// /// Bump it whenever [`Request`], [`Response`] or anything they contain /// changes shape. -pub const CONTROL_PROTOCOL: u32 = u32::from_be_bytes([b'T', b'S', b'N', 22]); +pub const CONTROL_PROTOCOL: u32 = u32::from_be_bytes([b'T', b'S', b'N', 23]); /// Reads one request off an accepted stream, answers it, writes the response. /// diff --git a/crates/tsunagi-core/src/net.rs b/crates/tsunagi-core/src/net.rs index 410c899..3b56b76 100644 --- a/crates/tsunagi-core/src/net.rs +++ b/crates/tsunagi-core/src/net.rs @@ -22,6 +22,7 @@ //! own reachability data. use std::net::SocketAddr; +use std::sync::Arc; use std::time::Duration; use iroh::endpoint::{ @@ -221,9 +222,18 @@ fn local_addr_string(addr: &iroh::endpoint::LocalTransportAddr) -> Option>, } impl EndpointAdapter { + /// Excludes the real name after interface creation (macOS assigns utunN). + pub(crate) fn exclude_interface(&self, name: &str) { + *self + .owned_interface + .write() + .unwrap_or_else(|p| p.into_inner()) = name.to_lowercase(); + } + /// Binds an endpoint according to `config`, reusing the persistent device key. pub async fn bind(config: &AgentConfig, identity: &DeviceIdentity) -> Result { let mut builder = Endpoint::builder(presets::Minimal) @@ -240,11 +250,15 @@ impl EndpointAdapter { TransportPolicy::N0Defaults => builder.preset(presets::N0), }; - if matches!( - config.transport, - TransportPolicy::N0Defaults | TransportPolicy::DirectOnly - ) { - let filter = iroh::address_lookup::AddrFilter::new(|addrs| { + let owned_interface = + Arc::new(std::sync::RwLock::new(config.interface_name.to_lowercase())); + { + let interface_filter = Arc::clone(&owned_interface); + let filter = iroh::address_lookup::AddrFilter::new(move |addrs| { + let owned_interface = interface_filter + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone(); let mut overlay_ips = std::collections::HashSet::new(); for iface in netdev::get_interfaces() { let name = iface.name.to_lowercase(); @@ -258,7 +272,9 @@ impl EndpointAdapter { .as_deref() .map(str::to_lowercase) .unwrap_or_default(); - if name.starts_with("tsun") + if name == owned_interface + || friendly == owned_interface + || name.starts_with("tsun") || name.contains("tsunagi") || name.contains("wintun") || friendly.starts_with("tsun") @@ -288,7 +304,9 @@ impl EndpointAdapter { .collect(); std::borrow::Cow::Owned(filtered) }); - builder = builder.addr_filter(filter); + builder = builder + .direct_addr_filter(filter.clone()) + .addr_filter(filter); } #[cfg(feature = "testing")] @@ -395,7 +413,10 @@ impl EndpointAdapter { } }); - Ok(Self { endpoint }) + Ok(Self { + endpoint, + owned_interface, + }) } /// The underlying iroh endpoint. diff --git a/crates/tsunagi-core/src/overlay/router.rs b/crates/tsunagi-core/src/overlay/router.rs index 9dc098b..e17fe10 100644 --- a/crates/tsunagi-core/src/overlay/router.rs +++ b/crates/tsunagi-core/src/overlay/router.rs @@ -386,24 +386,6 @@ impl RoutingTable { conflicts } - /// Current signed allocations used for encrypted data delivery checks. - pub(crate) fn probe_addresses( - &self, - network: NetworkId, - peer: EndpointId, - ) -> Option<(Ipv4Addr, Ipv4Addr)> { - let table = self.read(); - let routes = table.get(&network)?; - Some(( - routes.local?, - routes - .peers - .iter() - .find(|(_, holder)| *holder == peer) - .map(|(address, _)| *address)?, - )) - } - /// How many networks the table covers. pub fn len(&self) -> usize { self.read().len() diff --git a/crates/tsunagi-core/src/proto/message.rs b/crates/tsunagi-core/src/proto/message.rs index f9b0a2a..9cd983c 100644 --- a/crates/tsunagi-core/src/proto/message.rs +++ b/crates/tsunagi-core/src/proto/message.rs @@ -33,8 +33,10 @@ pub const ALPN: &[u8] = b"tsunagi/ctrl/8"; /// transport fragmentation. Version 5 adds the end-to-end protocol to that /// envelope, so a tunnel no longer depends on the kind of link under it, and /// a first-claimed time to a signed hostname claim. All members must upgrade -/// together. Persistent identities and network configuration do not change. -pub const DATA_ALPN: &[u8] = b"tsunagi/data/5"; +/// together. Version 6 uses one fixed inner WireGuard channel and independent +/// external carrier capabilities. Control authentication, persistent identities +/// and network configuration do not change. +pub const DATA_ALPN: &[u8] = b"tsunagi/data/6"; /// Largest plugin protocol identifier accepted when opening a data channel. pub const MAX_DATA_PROTOCOL_LEN: usize = 32; diff --git a/crates/tsunagi-core/tests/iroh_candidates.rs b/crates/tsunagi-core/tests/iroh_candidates.rs new file mode 100644 index 0000000..602977f --- /dev/null +++ b/crates/tsunagi-core/tests/iroh_candidates.rs @@ -0,0 +1,135 @@ +//! The local candidate filter must affect actual hole-punch addresses, not only publications. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use iroh::{ + Endpoint, RelayMode, TransportAddr, + endpoint::{PortmapperConfig, presets}, +}; +use std::borrow::Cow; + +#[tokio::test] +async fn direct_candidate_filter_changes_the_address_watch_used_by_connections() { + let observed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let called = observed.clone(); + let endpoint = Endpoint::builder(presets::Minimal) + .relay_mode(RelayMode::Disabled) + .clear_address_lookup() + .portmapper_config(PortmapperConfig::Disabled) + .net_report_config(iroh::NetReportConfig::minimal()) + .clear_ip_transports() + .bind_addr("127.0.0.1:0") + .unwrap() + // Stand in for addresses on the agent's own overlay interface. + .direct_addr_filter(iroh::address_lookup::AddrFilter::new(move |addrs| { + if addrs + .iter() + .any(|addr| matches!(addr, TransportAddr::Ip(_))) + { + called.store(true, std::sync::atomic::Ordering::Relaxed); + } + Cow::Owned( + addrs + .iter() + .filter(|addr| !matches!(addr, TransportAddr::Ip(_))) + .cloned() + .collect(), + ) + })) + .bind() + .await + .unwrap(); + tsunagi::testing::wait_until("filter applied to actual bound candidates", || async { + observed + .load(std::sync::atomic::Ordering::Relaxed) + .then_some(()) + }) + .await; + assert!( + !endpoint.bound_sockets().is_empty(), + "real UDP sockets are still bound" + ); + assert!( + endpoint.addr().ip_addrs().next().is_none(), + "excluded local paths never reach iroh's connection candidates" + ); + endpoint.close().await; +} + +#[tokio::test] +async fn carriers_exclude_the_kernel_assigned_interface_name_instead_of_the_requested_alias() { + use std::sync::{Arc, Mutex}; + use tsunagi::dataplane::transport::DataCarrier; + use tsunagi::dataplane::{PluginCapability, PluginError}; + use tsunagi::overlay::{MemoryTunFactory, OverlayError, TunDevice, TunFactory, TunRequest}; + use tsunagi::{BoxFuture, NetworkId}; + + #[derive(Debug)] + struct RenamingFactory(MemoryTunFactory); + impl TunFactory for RenamingFactory { + fn name(&self) -> &str { + "kernel-name-test" + } + fn on_host(&self) -> bool { + false + } + fn create<'a>( + &'a self, + mut request: TunRequest, + ) -> BoxFuture<'a, Result, OverlayError>> { + request.name = "kernel-assigned".into(); + self.0.create(request) + } + fn reconfigure<'a>( + &'a self, + request: TunRequest, + ) -> BoxFuture<'a, Result<(), OverlayError>> { + self.0.reconfigure(request) + } + fn destroy<'a>(&'a self, name: &'a str) -> BoxFuture<'a, ()> { + self.0.destroy(name) + } + } + #[derive(Debug, Default)] + struct Carrier(Mutex>); + impl DataCarrier for Carrier { + fn protocol_id(&self) -> &str { + "candidate-test" + } + fn protocol_version(&self) -> u16 { + 1 + } + fn on_interface_ready(&self, name: &str) { + *self.0.lock().unwrap() = Some(name.into()); + } + fn local_capability(&self, _: NetworkId) -> Result, PluginError> { + Ok(None) + } + fn on_peer_capability( + &self, + _: NetworkId, + _: iroh::EndpointId, + _: &PluginCapability, + ) -> Result<(), PluginError> { + Ok(()) + } + fn on_peer_gone(&self, _: NetworkId, _: iroh::EndpointId) {} + fn on_network_deactivated(&self, _: NetworkId) {} + } + let root = tempfile::TempDir::new().unwrap(); + let discovery = tsunagi::discovery::SharedMemoryDiscovery::new(); + let carrier = Arc::new(Carrier::default()); + let config = tsunagi::testing::config_with(root.path(), &discovery) + .with_interface( + Arc::new(RenamingFactory(MemoryTunFactory::new())), + "requested-alias", + 1280, + ) + .with_carrier(carrier.clone()); + let agent = tsunagi::Agent::spawn(config).await.unwrap(); + assert_eq!(agent.overlay().unwrap().interface, "kernel-assigned"); + assert_eq!( + carrier.0.lock().unwrap().as_deref(), + Some("kernel-assigned") + ); + agent.shutdown().await; +} diff --git a/crates/tsunagi-core/tests/mixed_transports.rs b/crates/tsunagi-core/tests/mixed_transports.rs index dd2f29d..5e7d279 100644 --- a/crates/tsunagi-core/tests/mixed_transports.rs +++ b/crates/tsunagi-core/tests/mixed_transports.rs @@ -1,220 +1,9 @@ -//! Connectivity does not depend on what kind of link each hop happens to be, -//! and a member is online however far away it is. - +//! Signed hostname ownership remains independent of connectivity and identity order. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex}; - -use bytes::Bytes; -use iroh::EndpointId; -use tsunagi::NetworkId; -use tsunagi::agent::DataPath; -use tsunagi::dataplane::transport::SharedLink; -use tsunagi::dataplane::{IpPlugin, PluginCapability, PluginError}; -use tsunagi::discovery::SharedMemoryDiscovery; -use tsunagi::testing::{TestAgent, network, wait_for_peers, wait_until}; - -/// A protocol that moves bytes between two members unchanged and keeps the -/// link it was handed, so a test can use the link directly. -#[derive(Debug)] -struct Carrier { - protocol: String, - links: Mutex>, -} - -impl Carrier { - fn new(protocol: &str) -> Arc { - Arc::new(Self { - protocol: protocol.into(), - links: Mutex::default(), - }) - } - - fn link(&self, peer: EndpointId) -> Option { - self.links.lock().unwrap().get(&peer).cloned() - } -} - -impl IpPlugin for Carrier { - fn protocol_id(&self) -> &str { - &self.protocol - } - - fn protocol_version(&self) -> u16 { - 1 - } - - fn local_capability( - &self, - _network: NetworkId, - ) -> Result, PluginError> { - Ok(Some(PluginCapability { - protocol: self.protocol.clone(), - version: 1, - enabled: true, - data: Vec::new(), - })) - } - - fn on_peer_capability( - &self, - _network: NetworkId, - _peer: EndpointId, - _capability: &PluginCapability, - ) -> Result<(), PluginError> { - Ok(()) - } - - fn on_peer_link(&self, _network: NetworkId, peer: EndpointId, link: SharedLink) { - self.links.lock().unwrap().insert(peer, link); - } - - fn on_peer_gone(&self, _network: NetworkId, peer: EndpointId) { - self.links.lock().unwrap().remove(&peer); - } - - fn on_network_deactivated(&self, _network: NetworkId) {} -} - -type Blocked = Arc>>; - -async fn member( - discovery: &SharedMemoryDiscovery, - plugins: &[Arc], - blocked: &Blocked, -) -> TestAgent { - let plugins: Vec<_> = plugins.to_vec(); - let blocked = Arc::clone(blocked); - TestAgent::spawn_with( - move |config| { - let config = config.with_unreachable_data_peers(blocked); - plugins - .into_iter() - .fold(config, |config, plugin| config.with_plugin(plugin)) - }, - discovery, - ) - .await - .unwrap() -} - -async fn exchange(from: &SharedLink, to: &SharedLink, payload: &'static [u8]) { - from.send(Bytes::from_static(payload)).unwrap(); - let got = tokio::time::timeout(tsunagi::testing::DEADLINE, to.recv()) - .await - .expect("the datagram should arrive") - .expect("the link stays open"); - assert_eq!(got, Bytes::from_static(payload)); -} - -#[tokio::test] -async fn a_tunnel_crosses_a_member_that_shares_no_protocol_with_either_end_and_moves_to_a_direct_link_by_itself() - { - // A and B share one protocol and nothing else. C shares a different - // one with each, so every link here is of a different kind and the - // middle has never heard of what A and B speak. A and B cannot reach - // each other directly, which is the case a blocked UDP path produces. - let discovery = SharedMemoryDiscovery::new(); - let (name, secret) = network("mixed-transports"); - - let blocked_a: Blocked = Arc::default(); - let blocked_b: Blocked = Arc::default(); - let a_shared = Carrier::new("shared"); - let b_shared = Carrier::new("shared"); - let a = member( - &discovery, - &[Carrier::new("a-and-c"), Arc::clone(&a_shared)], - &blocked_a, - ) - .await; - let b = member( - &discovery, - &[Carrier::new("c-and-b"), Arc::clone(&b_shared)], - &blocked_b, - ) - .await; - let c = member( - &discovery, - &[Carrier::new("a-and-c"), Carrier::new("c-and-b")], - &Blocked::default(), - ) - .await; - let (a_id, b_id, c_id) = ( - a.agent.endpoint_id(), - b.agent.endpoint_id(), - c.agent.endpoint_id(), - ); - blocked_a.lock().unwrap().insert(b_id); - blocked_b.lock().unwrap().insert(a_id); - - let network_id = a.agent.join_network(&name, &secret).await.unwrap(); - b.agent.join_network(&name, &secret).await.unwrap(); - c.agent.join_network(&name, &secret).await.unwrap(); - for agent in [&a, &b, &c] { - wait_for_peers(&agent.agent, network_id, 2).await; - } - - // The tunnel comes up through C, and data goes both ways. - let to_b = wait_until("a has a link to b", || async { a_shared.link(b_id) }).await; - let to_a = wait_until("b has a link to a", || async { b_shared.link(a_id) }).await; - exchange(&to_b, &to_a, b"a to b, sealed").await; - exchange(&to_a, &to_b, b"b to a, sealed").await; - - // The status says what is true: online, and through whom. - let status = a.agent.network_status(network_id).await.unwrap(); - let b_peer = status.peers.iter().find(|p| p.endpoint_id == b_id).unwrap(); - assert_eq!(b_peer.data_path, DataPath::Relayed { hops: 2, via: c_id }); - let c_peer = status.peers.iter().find(|p| p.endpoint_id == c_id).unwrap(); - assert!(matches!(&c_peer.data_path, DataPath::Direct { transport } if transport == "a-and-c")); - let b_member = wait_until("a lists b as a member", || async { - let status = a.agent.network_status(network_id).await.ok()?; - status.members.into_iter().find(|m| m.endpoint_id == b_id) - }) - .await; - assert!(b_member.online); - let carried = c.agent.network_status(network_id).await.unwrap().relay; - assert!(carried.forwarded >= 2, "{carried:?}"); - - // The direct path becomes possible: the same link carries on, now - // without C, and nobody has to do anything about it. - blocked_a.lock().unwrap().clear(); - blocked_b.lock().unwrap().clear(); - wait_until("a reaches b directly", || async { - let status = a.agent.network_status(network_id).await.ok()?; - let peer = status.peers.iter().find(|p| p.endpoint_id == b_id)?; - matches!(&peer.data_path, DataPath::Direct { transport } if transport == "shared") - .then_some(()) - }) - .await; - let forwarded = c - .agent - .network_status(network_id) - .await - .unwrap() - .relay - .forwarded; - exchange(&to_b, &to_a, b"direct now").await; - assert!( - Arc::ptr_eq(&to_b, &a_shared.link(b_id).unwrap()), - "the tunnel's link is the one it always had" - ); - assert_eq!( - c.agent - .network_status(network_id) - .await - .unwrap() - .relay - .forwarded, - forwarded, - "nothing goes through C any more" - ); - - a.stop().await; - b.stop().await; - c.stop().await; -} - +use tsunagi::{ + discovery::SharedMemoryDiscovery, + testing::{TestAgent, network, wait_for_peers, wait_until}, +}; #[tokio::test] async fn a_name_belongs_to_the_member_that_claimed_it_first_on_every_member() { let discovery = SharedMemoryDiscovery::new(); diff --git a/crates/tsunagi-core/tests/multi_peer.rs b/crates/tsunagi-core/tests/multi_peer.rs index f7f5231..3175b23 100644 --- a/crates/tsunagi-core/tests/multi_peer.rs +++ b/crates/tsunagi-core/tests/multi_peer.rs @@ -132,7 +132,7 @@ async fn opaque_plugin_capabilities_cross_the_control_plane() { let a = TestAgent::spawn_with( { let plugin = Arc::clone(&plugin_a); - move |cfg| cfg.with_plugin(plugin) + move |cfg| cfg.with_tunnel(plugin) }, &discovery, ) @@ -141,7 +141,7 @@ async fn opaque_plugin_capabilities_cross_the_control_plane() { let b = TestAgent::spawn_with( { let plugin = Arc::clone(&plugin_b); - move |cfg| cfg.with_plugin(plugin) + move |cfg| cfg.with_tunnel(plugin) }, &discovery, ) diff --git a/crates/tsunagi-tcp-tls/Cargo.toml b/crates/tsunagi-tcp-tls/Cargo.toml index bcc6539..3c5cc32 100644 --- a/crates/tsunagi-tcp-tls/Cargo.toml +++ b/crates/tsunagi-tcp-tls/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tsunagi-tcp-tls" version = "0.1.0" -description = "TCP TLS 443 transport and E2EE tunnel protocol for tsunagi mesh networks." +description = "TCP TLS carrier for independently encrypted tsunagi mesh traffic." edition.workspace = true rust-version.workspace = true license.workspace = true @@ -14,19 +14,10 @@ tokio = { workspace = true, features = ["net", "io-util", "time", "sync", "macro rustls = "0.23" tokio-rustls = "0.26" rcgen = { version = "0.14", default-features = false, features = ["ring"] } -x25519-dalek = { version = "2.0", features = ["static_secrets"] } -chacha20poly1305 = "0.10" -sha2 = "0.11" -hkdf = "0.13" -rand = "0.10" -subtle = "2.6" -zeroize = { version = "1.9", features = ["derive"] } bytes.workspace = true serde.workspace = true postcard.workspace = true tracing.workspace = true -thiserror.workspace = true -hex.workspace = true netdev = "0.46" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/crates/tsunagi-tcp-tls/src/crypto.rs b/crates/tsunagi-tcp-tls/src/crypto.rs deleted file mode 100644 index 5ca6cab..0000000 --- a/crates/tsunagi-tcp-tls/src/crypto.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! End-to-end packet encryption for `tcp-tls`. -//! -//! Uses X25519 Diffie-Hellman key exchange and ChaCha20-Poly1305 AEAD. -//! This ensures that even when traffic passes through intermediate relay nodes, -//! the intermediate nodes cannot inspect or tamper with the payload. - -use std::sync::atomic::{AtomicU64, Ordering}; - -use chacha20poly1305::aead::{Aead, KeyInit}; -use chacha20poly1305::{ChaCha20Poly1305, Nonce}; -use hkdf::Hkdf; -use iroh::EndpointId; -use sha2::Sha256; -use tsunagi::dataplane::PluginError; -use x25519_dalek::{PublicKey, StaticSecret}; - -/// Length of the X25519 public key. -pub const KEY_LEN: usize = 32; - -/// Domain separator for session key derivation. -const HKDF_SALT: &[u8] = b"tsunagi/e2ee/noise/v1"; - -/// A session cipher for communicating with one peer. -pub struct PeerCipher { - tx_cipher: ChaCha20Poly1305, - rx_cipher: ChaCha20Poly1305, - tx_nonce: AtomicU64, -} - -impl PeerCipher { - /// Establishes a new cipher between `local_id` and `remote_id` given - /// our local secret and their public key. - pub fn new( - local_id: EndpointId, - remote_id: EndpointId, - local_secret: &StaticSecret, - remote_public: &PublicKey, - ) -> Self { - let shared = local_secret.diffie_hellman(remote_public); - let hk = Hkdf::::new(Some(HKDF_SALT), shared.as_bytes()); - - let mut key_a_to_b = [0u8; 32]; - let mut key_b_to_a = [0u8; 32]; - let _ = hk.expand(b"tsunagi-key-a-to-b", &mut key_a_to_b); - let _ = hk.expand(b"tsunagi-key-b-to-a", &mut key_b_to_a); - - // Deterministic role: smaller EndpointId is party A. - let is_party_a = local_id.as_bytes() < remote_id.as_bytes(); - let (tx_key, rx_key) = if is_party_a { - (key_a_to_b, key_b_to_a) - } else { - (key_b_to_a, key_a_to_b) - }; - - let tx_cipher = ChaCha20Poly1305::new(&tx_key.into()); - let rx_cipher = ChaCha20Poly1305::new(&rx_key.into()); - - Self { - tx_cipher, - rx_cipher, - tx_nonce: AtomicU64::new(0), - } - } - - /// Encrypts an outbound packet. Prefixes the ciphertext with the 8-byte nonce. - pub fn encrypt(&self, packet: &[u8]) -> Result { - let counter = self.tx_nonce.fetch_add(1, Ordering::Relaxed); - let mut nonce_bytes = [0u8; 12]; - nonce_bytes[4..12].copy_from_slice(&counter.to_be_bytes()); - let nonce = Nonce::from_slice(&nonce_bytes); - - let ciphertext = self - .tx_cipher - .encrypt(nonce, packet) - .map_err(|err| PluginError::Other(format!("encryption failed: {err}")))?; - - let mut out = Vec::with_capacity(8 + ciphertext.len()); - out.extend_from_slice(&counter.to_be_bytes()); - out.extend_from_slice(&ciphertext); - Ok(bytes::Bytes::from(out)) - } - - /// Decrypts an inbound packet prefixed with the 8-byte nonce. - pub fn decrypt(&self, payload: &[u8]) -> Result { - if payload.len() < 8 + 16 { - return Err(PluginError::Other("payload too short".into())); - } - - let counter_bytes: [u8; 8] = match payload[..8].try_into() { - Ok(b) => b, - Err(_) => return Err(PluginError::Other("invalid nonce".into())), - }; - let mut nonce_bytes = [0u8; 12]; - nonce_bytes[4..12].copy_from_slice(&counter_bytes); - let nonce = Nonce::from_slice(&nonce_bytes); - - let plaintext = self - .rx_cipher - .decrypt(nonce, &payload[8..]) - .map_err(|err| PluginError::Other(format!("decryption failed: {err}")))?; - - Ok(bytes::Bytes::from(plaintext)) - } -} diff --git a/crates/tsunagi-tcp-tls/src/lib.rs b/crates/tsunagi-tcp-tls/src/lib.rs index c72bec9..979c07e 100644 --- a/crates/tsunagi-tcp-tls/src/lib.rs +++ b/crates/tsunagi-tcp-tls/src/lib.rs @@ -1,11 +1,8 @@ -//! The `tcp-tls` protocol: TCP with TLS 1.3 on port 443 with Noise E2EE. - +//! TCP TLS 1.3 carrier for opaque, independently encrypted mesh datagrams. pub mod cert; -pub mod crypto; pub mod plugin; pub mod transport; - pub use plugin::{ - TCP_TLS_PROTOCOL, TCP_TLS_VERSION, TcpTlsCodec, TcpTlsConfig, TcpTlsPlugin, local_ip_candidates, + TCP_TLS_PROTOCOL, TCP_TLS_VERSION, TcpTlsCarrier, TcpTlsConfig, local_ip_candidates, }; pub use transport::TcpTlsTransport; diff --git a/crates/tsunagi-tcp-tls/src/plugin.rs b/crates/tsunagi-tcp-tls/src/plugin.rs index cc7cf61..d611f04 100644 --- a/crates/tsunagi-tcp-tls/src/plugin.rs +++ b/crates/tsunagi-tcp-tls/src/plugin.rs @@ -1,26 +1,14 @@ -//! The `tcp-tls` IP plugin implementation. - -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use bytes::Bytes; -use iroh::EndpointId; -use serde::{Deserialize, Serialize}; -use x25519_dalek::{PublicKey, StaticSecret}; - -use tsunagi::dataplane::tunnel::{GenericTunnelPlugin, TunnelCodec}; -use tsunagi::dataplane::{PluginError, ProtocolOption}; -use tsunagi::identity::NetworkId; - -use crate::crypto::PeerCipher; +//! TLS carrier options and bounded reachability announcements. No IP cipher lives here. use crate::transport::TcpTlsTransport; - -/// The wire protocol identifier for `tcp-tls`. +use iroh::EndpointId; +use serde::Deserialize; +use std::sync::Arc; +use tsunagi::NetworkId; +use tsunagi::dataplane::{PluginError, ProtocolOption}; +/// External TLS carrier identifier. pub const TCP_TLS_PROTOCOL: &str = "tcp-tls"; - -/// Wire version for `tcp-tls`. -pub const TCP_TLS_VERSION: u16 = 1; - +/// Carrier announcement wire version. +pub const TCP_TLS_VERSION: u16 = 2; /// Configuration for the `tcp-tls` protocol plugin. #[derive(Debug, Clone, Default)] pub struct TcpTlsConfig { @@ -49,23 +37,22 @@ impl TcpTlsConfig { } } -/// Announcement payload exchanged between peers for `tcp-tls`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TcpTlsAnnouncement { - /// X25519 public key for E2EE noise encryption. - pub noise_public: [u8; 32], - /// TCP port the peer is listening on. - pub port: u16, - /// Candidate IP addresses where this peer can be reached on `port`. - #[serde(default)] - pub addrs: Vec, -} - /// Discovers local network interface IPs to announce to peers. pub fn local_ip_candidates() -> Vec { + local_ip_candidates_excluding(None) +} + +fn local_ip_candidates_excluding(excluded_interface: Option<&str>) -> Vec { let mut candidates = Vec::new(); for iface in netdev::get_interfaces() { - if !is_candidate_interface(&iface) { + if excluded_interface.is_some_and(|name| { + iface.name.eq_ignore_ascii_case(name) + || iface + .friendly_name + .as_deref() + .is_some_and(|friendly| friendly.eq_ignore_ascii_case(name)) + }) || !is_candidate_interface(&iface) + { continue; } for ip in &iface.ipv4 { @@ -203,26 +190,7 @@ pub(crate) fn ip_priority(iface: &netdev::Interface, ip: std::net::IpAddr) -> u3 } } } - -/// The codec implementing cryptographic transformation and announcements for `tcp-tls`. -pub struct TcpTlsCodec { - local_id: EndpointId, - static_secret: StaticSecret, - public_key: PublicKey, - transport: Arc, - ciphers: RwLock>>, -} - -impl std::fmt::Debug for TcpTlsCodec { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TcpTlsCodec") - .field("local_id", &self.local_id.fmt_short().to_string()) - .field("port", &self.transport.bound_port()) - .finish() - } -} - -impl TcpTlsCodec { +impl TcpTlsConfig { /// Supported options for `tcp-tls`. pub const OPTIONS: &'static [ProtocolOption] = &[ ProtocolOption { @@ -239,22 +207,6 @@ impl TcpTlsCodec { }, ]; - /// Creates a new `TcpTlsCodec`. - pub fn new(local_id: EndpointId, transport: Arc) -> Self { - let mut secret_bytes = [0u8; 32]; - rand::fill(&mut secret_bytes); - let static_secret = StaticSecret::from(secret_bytes); - let public_key = PublicKey::from(&static_secret); - - Self { - local_id, - static_secret, - public_key, - transport, - ciphers: RwLock::new(HashMap::new()), - } - } - /// Configures settings from parsed `-o` options. pub fn configure( mut config: TcpTlsConfig, @@ -281,250 +233,169 @@ impl TcpTlsCodec { Ok(config) } } - -impl TunnelCodec for TcpTlsCodec { +/// TLS carrier announcements and lifecycle, independent of IP encryption. +#[derive(Debug)] +pub struct TcpTlsCarrier { + transport: Arc, + excluded_interface: std::sync::Mutex>, + peers: std::sync::Mutex< + std::collections::HashMap<(NetworkId, EndpointId), Vec>, + >, +} +impl TcpTlsCarrier { + /// Shares the transport's listener; creates no end-to-end cipher. + pub fn new(transport: Arc) -> Arc { + Arc::new(Self { + transport, + excluded_interface: std::sync::Mutex::new(None), + peers: Default::default(), + }) + } + /// Registers a carrier while excluding the agent's own, possibly custom-named interface. + pub fn with_interface(transport: Arc, interface: String) -> Arc { + Arc::new(Self { + transport, + excluded_interface: std::sync::Mutex::new(Some(interface)), + peers: Default::default(), + }) + } + fn update_peer( + &self, + network: NetworkId, + peer: EndpointId, + addrs: Option>, + ) { + let mut peers = self.peers.lock().unwrap_or_else(|p| p.into_inner()); + match addrs { + Some(addrs) => { + peers.insert((network, peer), addrs); + } + None => { + peers.remove(&(network, peer)); + } + } + let mut combined = peers + .iter() + .filter(|((_, id), _)| *id == peer) + .flat_map(|(_, addrs)| addrs.iter().copied()) + .collect::>(); + combined.sort_unstable(); + combined.dedup(); + combined.truncate(32); + self.transport.replace_peer_addrs(peer, combined); + } +} +impl tsunagi::dataplane::transport::DataCarrier for TcpTlsCarrier { fn protocol_id(&self) -> &str { TCP_TLS_PROTOCOL } - fn protocol_version(&self) -> u16 { - TCP_TLS_VERSION + 2 } - - fn options(&self) -> &'static [ProtocolOption] { - Self::OPTIONS + fn on_interface_ready(&self, name: &str) { + *self + .excluded_interface + .lock() + .unwrap_or_else(|p| p.into_inner()) = Some(name.into()); } - - fn local_capability_data(&self, _network: NetworkId) -> Result, PluginError> { - let announcement = TcpTlsAnnouncement { - noise_public: *self.public_key.as_bytes(), - port: self.transport.bound_port(), - addrs: local_ip_candidates(), - }; - postcard::to_allocvec(&announcement) - .map_err(|err| PluginError::Other(format!("encode announcement failed: {err}"))) + fn local_capability( + &self, + _: NetworkId, + ) -> Result, PluginError> { + let data = postcard::to_allocvec(&( + self.transport.bound_port(), + local_ip_candidates_excluding( + self.excluded_interface + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_deref(), + ) + .into_iter() + .take(32) + .collect::>(), + )) + .map_err(|_| PluginError::Other("cannot encode TLS addresses".into()))?; + Ok(Some(tsunagi::dataplane::PluginCapability { + protocol: TCP_TLS_PROTOCOL.into(), + version: 2, + enabled: true, + data, + })) } - - fn on_peer_capability_data( + fn on_peer_capability( &self, network: NetworkId, peer: EndpointId, - data: &[u8], + cap: &tsunagi::dataplane::PluginCapability, ) -> Result<(), PluginError> { - let announcement: TcpTlsAnnouncement = postcard::from_bytes(data) - .map_err(|err| PluginError::Rejected(format!("invalid announcement: {err}")))?; - - for ip in &announcement.addrs { - if is_candidate_ip(*ip) { - self.transport - .set_peer_addr(peer, std::net::SocketAddr::new(*ip, announcement.port)); - } + if !cap.enabled || cap.protocol != TCP_TLS_PROTOCOL || cap.version != 2 { + return Err(PluginError::Rejected("unsupported TLS carrier".into())); } - - let peer_noise_pubkey = PublicKey::from(announcement.noise_public); - let cipher = Arc::new(PeerCipher::new( - self.local_id, + // Decode a fixed bounded array through the existing common announcement limit. + #[derive(Deserialize)] + struct Addresses( + u16, + #[serde(deserialize_with = "bounded_ips")] Vec, + ); + let Addresses(port, ips) = postcard::from_bytes(&cap.data) + .map_err(|_| PluginError::Rejected("invalid TLS carrier addresses".into()))?; + if port == 0 { + return Err(PluginError::Rejected("invalid TLS carrier port".into())); + } + self.update_peer( + network, peer, - &self.static_secret, - &peer_noise_pubkey, - )); - - let mut guard = match self.ciphers.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.insert((network, peer), cipher); + Some( + ips.into_iter() + .filter(|ip| is_candidate_ip(*ip)) + .map(|ip| std::net::SocketAddr::new(ip, port)) + .collect(), + ), + ); Ok(()) } - - fn on_peer_down(&self, network: NetworkId, peer: EndpointId) { - let mut guard = match self.ciphers.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.remove(&(network, peer)); + fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) { + self.update_peer(network, peer, None); } - - fn encrypt( - &self, - network: NetworkId, - peer: EndpointId, - packet: &[u8], - ) -> Result { - let cipher = { - let guard = match self.ciphers.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.get(&(network, peer)).cloned() - }; - - let Some(cipher) = cipher else { - return Err(PluginError::Unavailable("no cipher for peer".into())); - }; - - cipher.encrypt(packet) - } - - fn decrypt( - &self, - network: NetworkId, - peer: EndpointId, - payload: &[u8], - ) -> Result { - let cipher = { - let guard = match self.ciphers.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.get(&(network, peer)).cloned() - }; - - let Some(cipher) = cipher else { - return Err(PluginError::Unavailable("no cipher for peer".into())); - }; - - cipher.decrypt(payload) + fn on_network_deactivated(&self, network: NetworkId) { + let peers: Vec<_> = self + .peers + .lock() + .unwrap_or_else(|p| p.into_inner()) + .keys() + .filter(|(id, _)| *id == network) + .map(|(_, peer)| *peer) + .collect(); + for peer in peers { + self.update_peer(network, peer, None); + } } } - -/// The complete `tcp-tls` plugin. -pub type TcpTlsPlugin = GenericTunnelPlugin; - -#[cfg(test)] -mod tests { - use super::*; - use std::net::{Ipv4Addr, Ipv6Addr}; - - #[test] - fn test_is_candidate_interface_filters_overlay_and_bridges() { - let mut iface = netdev::Interface::dummy(); - iface.flags = 1; // UP - - // Standard Ethernet - should pass - iface.name = "eth0".to_string(); - assert!(is_candidate_interface(&iface)); - - // Windows Ethernet with GUID name - should pass - iface.name = "{B6149D2C-1234-5678-9ABC-DEF012345678}".to_string(); - iface.friendly_name = Some("Ethernet".to_string()); - iface.description = Some("Intel(R) Ethernet Connection".to_string()); - assert!(is_candidate_interface(&iface)); - - // Linux tsun0 overlay - must be filtered - iface.name = "tsun0".to_string(); - iface.friendly_name = None; - iface.description = None; - assert!(!is_candidate_interface(&iface)); - - // Windows Wintun overlay (GUID name with tsun0 friendly name) - must be filtered! - iface.name = "{72D85D76-F1A5-42AC-BC49-2169527CDDDF}".to_string(); - iface.friendly_name = Some("tsun0".to_string()); - iface.description = Some("Wintun Userspace Tunnel".to_string()); - assert!(!is_candidate_interface(&iface)); - - // Windows Wintun overlay with description containing Wintun - must be filtered! - iface.friendly_name = Some("Local Area Connection 2".to_string()); - iface.description = Some("Wintun Userspace Tunnel".to_string()); - assert!(!is_candidate_interface(&iface)); - - // Docker bridge - must be filtered - iface.name = "docker0".to_string(); - iface.friendly_name = None; - iface.description = None; - assert!(!is_candidate_interface(&iface)); - - // Docker network bridge br-* - must be filtered - iface.name = "br-4f6d3a9b1c2e".to_string(); - assert!(!is_candidate_interface(&iface)); - - // Virtual ethernet veth* - must be filtered - iface.name = "veth12345".to_string(); - assert!(!is_candidate_interface(&iface)); - - // Libvirt NAT bridge virbr* - must be filtered - iface.name = "virbr0".to_string(); - assert!(!is_candidate_interface(&iface)); - - // CNI bridge - must be filtered - iface.name = "cni0".to_string(); - assert!(!is_candidate_interface(&iface)); - - // Hyper-V / WSL vEthernet switch on Windows - must be filtered - iface.name = "{A1B2C3D4-E5F6-7890-1234-567890ABCDEF}".to_string(); - iface.friendly_name = Some("vEthernet (WSL)".to_string()); - iface.description = Some("Hyper-V Virtual Ethernet Adapter".to_string()); - assert!(!is_candidate_interface(&iface)); - - // Down interface - must be filtered - let mut down_iface = netdev::Interface::dummy(); - down_iface.flags = 0; // Not UP - down_iface.name = "eth0".to_string(); - assert!(!is_candidate_interface(&down_iface)); - } - - #[test] - fn test_is_candidate_ip() { - // Valid LAN IP - assert!(is_candidate_ip(std::net::IpAddr::V4(Ipv4Addr::new( - 192, 168, 1, 117 - )))); - // Valid Tailscale IP - assert!(is_candidate_ip(std::net::IpAddr::V4(Ipv4Addr::new( - 100, 77, 155, 120 - )))); - // Valid Public IP - assert!(is_candidate_ip(std::net::IpAddr::V4(Ipv4Addr::new( - 1, 1, 1, 1 - )))); - // Valid IPv6 GUA - assert!(is_candidate_ip(std::net::IpAddr::V6(Ipv6Addr::new( - 0x2a01, 0x4b00, 0xb8e3, 0x4e00, 0, 0, 0, 1 - )))); - - // Loopback - must be rejected - assert!(!is_candidate_ip(std::net::IpAddr::V4(Ipv4Addr::new( - 127, 0, 0, 1 - )))); - assert!(!is_candidate_ip(std::net::IpAddr::V6(Ipv6Addr::LOCALHOST))); - - // Unspecified - must be rejected - assert!(!is_candidate_ip(std::net::IpAddr::V4( - Ipv4Addr::UNSPECIFIED - ))); - assert!(!is_candidate_ip(std::net::IpAddr::V6( - Ipv6Addr::UNSPECIFIED - ))); - - // Link-local - must be rejected - assert!(!is_candidate_ip(std::net::IpAddr::V4(Ipv4Addr::new( - 169, 254, 1, 1 - )))); - assert!(!is_candidate_ip(std::net::IpAddr::V6(Ipv6Addr::new( - 0xfe80, 0, 0, 0, 0, 0x1ff, 0xfe00, 1 - )))); - - // Broadcast - must be rejected - assert!(!is_candidate_ip(std::net::IpAddr::V4(Ipv4Addr::BROADCAST))); - } - - #[test] - fn test_ip_priority_ordering() { - let mut iface_lan = netdev::Interface::dummy(); - iface_lan.flags = 1; - iface_lan.gateway = Some(netdev::net::device::NetworkDevice::new()); - - let mut iface_vpn = netdev::Interface::dummy(); - iface_vpn.flags = 1; - iface_vpn.name = "tailscale0".to_string(); - - let lan_ip = std::net::IpAddr::V4(Ipv4Addr::new(192, 168, 1, 117)); - let tailscale_ip = std::net::IpAddr::V4(Ipv4Addr::new(100, 77, 155, 120)); - - let p_lan = ip_priority(&iface_lan, lan_ip); - let p_ts = ip_priority(&iface_vpn, tailscale_ip); - - // LAN IP with gateway should score higher than VPN IP without gateway - assert!(p_lan > p_ts); +fn bounded_ips<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Vec; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("at most 32 IP candidates") + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + if seq.size_hint().is_some_and(|n| n > 32) { + return Err(serde::de::Error::custom("too many IP candidates")); + } + let mut ips = Vec::new(); + while let Some(ip) = seq.next_element()? { + if ips.len() == 32 { + return Err(serde::de::Error::custom("too many IP candidates")); + } + ips.push(ip); + } + Ok(ips) + } } + deserializer.deserialize_seq(Visitor) } diff --git a/crates/tsunagi-tcp-tls/src/transport.rs b/crates/tsunagi-tcp-tls/src/transport.rs index 0d469df..788a801 100644 --- a/crates/tsunagi-tcp-tls/src/transport.rs +++ b/crates/tsunagi-tcp-tls/src/transport.rs @@ -464,6 +464,17 @@ impl TcpTlsTransport { } } + /// Replaces a bounded candidate set learned through an authenticated session. + pub fn replace_peer_addrs(&self, peer: EndpointId, addresses: Vec) { + if let Ok(mut peers) = self.peer_addresses.write() { + if addresses.is_empty() { + peers.remove(&peer); + } else { + peers.insert(peer, addresses.into_iter().take(32).collect()); + } + } + } + /// Runs the accept loop for inbound TCP TLS connections. pub async fn accept_loop(&self, on_inbound: F) where diff --git a/crates/tsunagi-tcp-tls/tests/crypto_tests.rs b/crates/tsunagi-tcp-tls/tests/crypto_tests.rs deleted file mode 100644 index 118423f..0000000 --- a/crates/tsunagi-tcp-tls/tests/crypto_tests.rs +++ /dev/null @@ -1,73 +0,0 @@ -#![allow(missing_docs)] -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use tsunagi_tcp_tls::crypto::PeerCipher; -use x25519_dalek::{PublicKey, StaticSecret}; - -#[test] -fn test_peer_cipher_bidirectional_roundtrip() { - let peer_a_id = iroh::SecretKey::generate().public(); - let peer_b_id = iroh::SecretKey::generate().public(); - - let secret_a = StaticSecret::from([10u8; 32]); - let public_a = PublicKey::from(&secret_a); - - let secret_b = StaticSecret::from([20u8; 32]); - let public_b = PublicKey::from(&secret_b); - - let cipher_a = PeerCipher::new(peer_a_id, peer_b_id, &secret_a, &public_b); - let cipher_b = PeerCipher::new(peer_b_id, peer_a_id, &secret_b, &public_a); - - // A -> B - let msg_from_a = b"hello from node A to node B over Noise E2EE"; - let encrypted_by_a = cipher_a - .encrypt(msg_from_a) - .expect("encryption should succeed"); - let decrypted_by_b = cipher_b - .decrypt(&encrypted_by_a) - .expect("decryption should succeed"); - assert_eq!(&decrypted_by_b[..], msg_from_a); - - // B -> A - let msg_from_b = b"hello back from node B to node A"; - let encrypted_by_b = cipher_b - .encrypt(msg_from_b) - .expect("encryption should succeed"); - let decrypted_by_a = cipher_a - .decrypt(&encrypted_by_b) - .expect("decryption should succeed"); - assert_eq!(&decrypted_by_a[..], msg_from_b); - - // Sequence nonces increment - let msg2 = b"packet 2"; - let enc2 = cipher_a.encrypt(msg2).expect("encryption should succeed"); - assert_ne!(encrypted_by_a[..8], enc2[..8], "nonces should differ"); - let dec2 = cipher_b.decrypt(&enc2).expect("decryption should succeed"); - assert_eq!(&dec2[..], msg2); -} - -#[test] -fn test_tampered_payload_rejected() { - let peer_a_id = iroh::SecretKey::generate().public(); - let peer_b_id = iroh::SecretKey::generate().public(); - - let secret_a = StaticSecret::from([10u8; 32]); - let public_a = PublicKey::from(&secret_a); - - let secret_b = StaticSecret::from([20u8; 32]); - let public_b = PublicKey::from(&secret_b); - - let cipher_a = PeerCipher::new(peer_a_id, peer_b_id, &secret_a, &public_b); - let cipher_b = PeerCipher::new(peer_b_id, peer_a_id, &secret_b, &public_a); - - let encrypted = cipher_a.encrypt(b"secret payload").expect("encrypt"); - // Corrupt one byte of ciphertext - let len = encrypted.len(); - let mut modified = encrypted.to_vec(); - modified[len - 1] ^= 0xff; - - assert!( - cipher_b.decrypt(&modified).is_err(), - "corrupted ciphertext must fail AEAD verification" - ); -} diff --git a/crates/tsunagi-tcp-tls/tests/plugin_tests.rs b/crates/tsunagi-tcp-tls/tests/plugin_tests.rs index 1b98eb6..fe15ef3 100644 --- a/crates/tsunagi-tcp-tls/tests/plugin_tests.rs +++ b/crates/tsunagi-tcp-tls/tests/plugin_tests.rs @@ -1,149 +1,33 @@ -#![allow(missing_docs)] +//! Bounded TLS carrier announcements contain no end-to-end cipher selection. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::sync::Arc; -use std::time::Duration; - -use bytes::Bytes; -use tokio::sync::mpsc; -use tsunagi::dataplane::transport::{InboundLink, PacketTransport}; -use tsunagi::dataplane::{IpPlugin, PacketSink, PluginContext}; -use tsunagi::identity::{DeviceIdentity, NetworkId}; -use tsunagi_tcp_tls::{TcpTlsCodec, TcpTlsPlugin, TcpTlsTransport}; - -struct TestSink { - delivered: mpsc::Sender<(NetworkId, iroh::EndpointId, Bytes)>, -} - -impl PacketSink for TestSink { - fn deliver( - &self, - network: NetworkId, - from: iroh::EndpointId, - packet: Bytes, - ) -> tsunagi::BoxFuture<'_, ()> { - let delivered = self.delivered.clone(); - Box::pin(async move { - let _ = delivered.send((network, from, packet)).await; - }) - } -} +use std::{net::IpAddr, sync::Arc}; +use tsunagi::{ + dataplane::transport::DataCarrier, + identity::{DeviceIdentity, NetworkId}, +}; +use tsunagi_tcp_tls::{TcpTlsCarrier, TcpTlsTransport}; #[tokio::test] -async fn test_tcp_tls_plugin_end_to_end() { - let identity_a = DeviceIdentity::generate(); - let peer_a = identity_a.endpoint_id(); - - let identity_b = DeviceIdentity::generate(); - let peer_b = identity_b.endpoint_id(); - - let network = NetworkId::from_bytes([77u8; 32]); - - // Transports - let transport_b = Arc::new( - TcpTlsTransport::bind(&identity_b, Some(0), None) +async fn tls_capabilities_bound_addresses_and_reject_invalid_ports_and_versions() { + let identity = DeviceIdentity::generate(); + let peer = DeviceIdentity::generate().endpoint_id(); + let transport = Arc::new( + TcpTlsTransport::bind(&identity, Some(0), None) .await - .expect("bind B"), + .unwrap(), ); - let port_b = transport_b.bound_port(); - - let transport_a = Arc::new( - TcpTlsTransport::bind(&identity_a, Some(0), None) - .await - .expect("bind A"), - ); - - // Codecs and plugins - let codec_a = TcpTlsCodec::new(peer_a, Arc::clone(&transport_a)); - let plugin_a = Arc::new(TcpTlsPlugin::new(codec_a)); - - let codec_b = TcpTlsCodec::new(peer_b, Arc::clone(&transport_b)); - let plugin_b = Arc::new(TcpTlsPlugin::new(codec_b)); - - // Attach sinks - let (sink_a_tx, mut sink_a_rx) = mpsc::channel(10); - plugin_a.attach(PluginContext::with_sink(Arc::new(TestSink { - delivered: sink_a_tx, - }))); - - let (sink_b_tx, mut sink_b_rx) = mpsc::channel(10); - plugin_b.attach(PluginContext::with_sink(Arc::new(TestSink { - delivered: sink_b_tx, - }))); - - // Capability exchange - let cap_a = plugin_a - .local_capability(network) - .expect("cap A") - .expect("some cap A"); - let cap_b = plugin_b - .local_capability(network) - .expect("cap B") - .expect("some cap B"); - - plugin_a - .on_peer_capability(network, peer_b, &cap_b) - .expect("A on cap B"); - plugin_b - .on_peer_capability(network, peer_a, &cap_a) - .expect("B on cap A"); - - // Spawn accept loop on B - let (inbound_tx, mut inbound_rx) = mpsc::channel::(1); - let transport_b_clone = Arc::clone(&transport_b); - tokio::spawn(async move { - transport_b_clone - .accept_loop(move |inbound| { - let _ = inbound_tx.try_send(inbound); - }) - .await; - }); - - // Make sure A knows B's loopback address - transport_a.set_peer_addr(peer_b, format!("127.0.0.1:{port_b}").parse().unwrap()); - - // Connect link from A to B - let link_a = transport_a - .open(network, peer_b, "tcp-tls") - .await - .expect("open link from A"); - let inbound_b = tokio::time::timeout(Duration::from_secs(5), inbound_rx.recv()) - .await - .expect("recv timeout") - .expect("recv inbound"); - let link_b = inbound_b.link; - - // Install links into plugins - plugin_a.on_peer_link(network, peer_b, link_a); - plugin_b.on_peer_link(network, peer_a, link_b); - - // Carry packet from A to B - let ip_packet_a = Bytes::from_static(b"4500003c000100004011e0e8c0a80101c0a8010200000000"); - let carried_a = plugin_a.carry(network, peer_b, ip_packet_a.clone()); - assert!(carried_a, "plugin A should carry packet"); - - let (net_received, from_peer, delivered_pkt) = - tokio::time::timeout(Duration::from_secs(3), sink_b_rx.recv()) - .await - .expect("timeout waiting for delivery on B") - .expect("packet on B"); - - assert_eq!(net_received, network); - assert_eq!(from_peer, peer_a); - assert_eq!(delivered_pkt, ip_packet_a); - - // Carry reply packet from B to A - let ip_packet_b = Bytes::from_static(b"reply-packet-from-b"); - let carried_b = plugin_b.carry(network, peer_a, ip_packet_b.clone()); - assert!(carried_b, "plugin B should carry packet"); - - let (net_received, from_peer, delivered_pkt) = - tokio::time::timeout(Duration::from_secs(3), sink_a_rx.recv()) - .await - .expect("timeout waiting for delivery on A") - .expect("packet on A"); - - assert_eq!(net_received, network); - assert_eq!(from_peer, peer_b); - assert_eq!(delivered_pkt, ip_packet_b); + let carrier = TcpTlsCarrier::new(transport); + let network = NetworkId::from_bytes([7; 32]); + let mut cap = carrier.local_capability(network).unwrap().unwrap(); + assert_eq!(cap.protocol, "tcp-tls"); + let addresses: Vec = vec!["192.0.2.1".parse().unwrap(); 33]; + cap.data = postcard::to_allocvec(&(443u16, addresses)).unwrap(); + assert!(carrier.on_peer_capability(network, peer, &cap).is_err()); + cap.data = postcard::to_allocvec(&(0u16, Vec::::new())).unwrap(); + assert!(carrier.on_peer_capability(network, peer, &cap).is_err()); + cap.data = + postcard::to_allocvec(&(443u16, vec!["192.0.2.1".parse::().unwrap()])).unwrap(); + carrier.on_peer_capability(network, peer, &cap).unwrap(); + cap.version += 1; + assert!(carrier.on_peer_capability(network, peer, &cap).is_err()); } diff --git a/crates/tsunagi-tray/src/devices.rs b/crates/tsunagi-tray/src/devices.rs index dcb50bf..316bfef 100644 --- a/crates/tsunagi-tray/src/devices.rs +++ b/crates/tsunagi-tray/src/devices.rs @@ -45,10 +45,9 @@ struct Device<'a> { copy: Option, /// The full endpoint id, which clicking the short form copies. id: &'a str, - /// End-to-end tunnel protocol, independent of the path carrying it. - tunnel_protocol: &'a str, + /// Why the permitted external data channel is unavailable, when known. + data_error: Option<&'a str>, address: Option, - handshake_secs: Option, control_rtt_ms: Option, overlay: Option<&'a OverlayPeerReport>, /// It is connected and offers to be an exit node. @@ -431,11 +430,14 @@ fn devices<'a>(network: &'a NetworkReport, own_id: &str) -> Vec> { hostname, copy: hostname.map(|h| format!("{h}.{}", network.name)), id, - tunnel_protocol: overlay.map_or("—", |p| p.protocol.as_str()), + data_error: network + .peers + .iter() + .find(|p| p.endpoint_id == id) + .and_then(|p| p.data_error.as_deref()), address: overlay .and_then(|p| p.address.clone()) .or_else(|| member.and_then(|m| m.overlay_address_v4.clone())), - handshake_secs: overlay.and_then(|p| p.handshake_secs_ago), control_rtt_ms: network .peers .iter() @@ -463,8 +465,8 @@ fn devices<'a>(network: &'a NetworkReport, own_id: &str) -> Vec> { const COLUMN_HEADERS: [&str; 10] = [ "device", "exit", - "end-to-end", - "handshake / RTT", + "data", + "RTT", "address", "data path", "pkts", @@ -477,8 +479,8 @@ const COLUMN_HEADERS: [&str; 10] = [ const COLUMN_SAMPLES: [&str; 10] = [ "device", "exit", - "end-to-end", - "999d / 9999 ms", + "data", + "9999 ms", "255.255.255.255", "data path", "99999/99999", @@ -597,7 +599,8 @@ fn device_table( for (column, header) in COLUMN_HEADERS.into_iter().enumerate() { widths[column] = widths[column].max(measure(header.to_owned())); } - let (mut tx_packets, mut rx_packets, mut tx_bytes, mut rx_bytes) = (0u64, 0, 0, 0); + let (mut tx_packets, mut rx_packets, mut tx_bytes, mut rx_bytes): (u64, u64, u64, u64) = + (0, 0, 0, 0); for device in devices { let name = format!( "{} ({}){}", @@ -608,25 +611,18 @@ fn device_table( widths[0] = widths[0].max(measure(name) + 24.0); let (path, _) = data_path_text(device.path, device.via_name.as_deref(), device.online); for (column, text) in [ - (2, displayed_protocol(device.tunnel_protocol).to_owned()), - ( - 3, - handshake_rtt_text( - device.handshake_secs, - device.control_rtt_ms, - displayed_protocol(device.tunnel_protocol) != "—", - ), - ), + (2, "waiting".to_owned()), + (3, rtt_text(device.control_rtt_ms)), (4, device.address.clone().unwrap_or_default()), (5, path), ] { widths[column] = widths[column].max(measure(text)); } if let Some(peer) = device.overlay { - tx_packets += peer.tx_packets; - rx_packets += peer.rx_packets; - tx_bytes += peer.tx_bytes; - rx_bytes += peer.rx_bytes; + tx_packets = tx_packets.saturating_add(peer.tx_packets); + rx_packets = rx_packets.saturating_add(peer.rx_packets); + tx_bytes = tx_bytes.saturating_add(peer.tx_bytes); + rx_bytes = rx_bytes.saturating_add(peer.rx_bytes); for (column, text) in [ (6, format!("{}/{}", peer.tx_packets, peer.rx_packets)), ( @@ -669,13 +665,16 @@ fn device_table( let response = ui.weak(COLUMN_HEADERS[column]); match column { 2 => { - response.on_hover_text("End-to-end encryption."); + response.on_hover_text("Encrypted data channel readiness."); } 3 => { - response.on_hover_text("Handshake age / control-connection RTT."); + response.on_hover_text("Control-connection round-trip time."); + } + 7 => { + response.on_hover_text("Plaintext IP bytes accepted by the encrypted channel in this counter lifetime; excludes relay transit, transport overhead and delivery probes."); } 5 => { - response.on_hover_text("Transport carrying the encrypted tunnel."); + response.on_hover_text("External data transport or relay route."); } _ => {} } @@ -709,12 +708,8 @@ fn device_table( 1 => exit_cell(ui, device, network_id, confirm), 2 => tunnel_cell(ui, device), 3 => { - ui.label(handshake_rtt_text( - device.handshake_secs, - device.control_rtt_ms, - displayed_protocol(device.tunnel_protocol) != "—", - )) - .on_hover_text("Last handshake / control RTT."); + ui.label(rtt_text(device.control_rtt_ms)) + .on_hover_text("Control-connection round-trip time."); } 4 => match &device.address { Some(address) => format::copy_field(ui, address, address), @@ -776,43 +771,29 @@ fn device_table( ); } -/// A protocol name as it should appear in the table. -fn displayed_protocol(protocol: &str) -> &str { - if protocol.is_empty() { "—" } else { protocol } -} - -/// The end-to-end tunnel and its own health, without transport details. -fn handshake_rtt_text(handshake: Option, rtt: Option, has_tunnel: bool) -> String { - let age = handshake.map_or_else( - || { - if has_tunnel { - "waiting".into() - } else { - "—".into() - } - }, - format::age, - ); - let rtt = rtt.map_or_else(|| "—".into(), |value| format!("{value} ms")); - format!("{age} / {rtt}") -} - -fn tunnel_hint(protocol: &str) -> String { - let name = displayed_protocol(protocol); - if name == "tcp-tls" { - format!("{name}: Noise end-to-end encryption") - } else { - format!("{name}: WireGuard end-to-end encryption") - } +/// Control RTT is independent of the hidden internal cipher's handshake age. +fn rtt_text(rtt: Option) -> String { + rtt.map_or_else(|| "—".into(), |value| format!("{value} ms")) } fn tunnel_cell(ui: &mut egui::Ui, device: &Device<'_>) { - let protocol = displayed_protocol(device.tunnel_protocol); - if protocol == "—" { - ui.weak(protocol); - return; - } - ui.label(protocol).on_hover_text(tunnel_hint(protocol)); + let ready = device.overlay.is_some_and(|peer| peer.is_up()) + && !matches!(device.path, None | Some(DataPathReport::None)); + let text = if !device.online { + "—" + } else if ready { + "ready" + } else { + "waiting" + }; + let error = device.data_error.unwrap_or({ + if ready { + "Encrypted data channel ready" + } else { + "Encrypted data channel is not ready" + } + }); + ui.label(text).on_hover_text(error); } /// Compact route label and whether that route is the preferred direct form. @@ -941,8 +922,6 @@ mod tests { #[test] fn tunnel_and_direct_transport_are_named_as_different_layers() { - assert_eq!(displayed_protocol("wg"), "wg"); - assert_eq!(tunnel_hint("wg"), "wg: WireGuard end-to-end encryption"); assert_eq!( data_path_text( Some(&DataPathReport::Direct { @@ -953,15 +932,11 @@ mod tests { ), (format!("{} tcp-tls", icon::LINK), true) ); - assert_eq!( - tunnel_hint("tcp-tls"), - "tcp-tls: Noise end-to-end encryption" - ); } #[test] fn incomplete_and_longer_paths_stay_explicit() { - assert_eq!(handshake_rtt_text(None, Some(25), true), "waiting / 25 ms"); + assert_eq!(rtt_text(Some(25)), "25 ms"); assert_eq!( data_path_text( Some(&DataPathReport::Relayed { @@ -981,10 +956,11 @@ mod tests { } #[test] - fn handshake_rtt_has_no_invented_values_and_uses_control_measurements() { + fn rtt_has_no_invented_values_and_ignores_the_internal_handshake_age() { use tsunagi::ipc::{OverlayReport, PeerReport}; let network = NetworkReport { peers: vec![PeerReport { + data_error: None, endpoint_id: "peer".into(), rtt_ms: Some(27), ..Default::default() @@ -1001,13 +977,8 @@ mod tests { ..Default::default() }; let rows = devices(&network, "own"); - assert_eq!( - handshake_rtt_text(rows[0].handshake_secs, rows[0].control_rtt_ms, true), - "5s / 27 ms" - ); - assert_eq!(handshake_rtt_text(None, None, false), "— / —"); - assert_eq!(rows[0].tunnel_protocol, "wg"); - assert!(!tunnel_hint(rows[0].tunnel_protocol).contains("wg-quic")); + assert_eq!(rtt_text(rows[0].control_rtt_ms), "27 ms"); + assert_eq!(rtt_text(None), "—"); } #[test] @@ -1029,7 +1000,7 @@ mod tests { } #[test] - fn counters_headers_and_handshake_stay_on_one_line_without_overlapping() { + fn counters_headers_and_rtt_stay_on_one_line_without_overlapping() { use tsunagi::ipc::{OverlayReport, PeerReport}; fn collect_text<'a>( @@ -1058,6 +1029,7 @@ mod tests { peers: ["peer", "other"] .into_iter() .map(|id| PeerReport { + data_error: None, endpoint_id: id.into(), rtt_ms: Some(12_345), ..Default::default() @@ -1112,8 +1084,8 @@ mod tests { let mut texts = Vec::new(); collect_text(&shapes, &mut texts); for (label, next_column) in [ - ("handshake / RTT".to_string(), "address"), - (handshake_rtt_text(Some(age), Some(12_345), true), "address"), + ("RTT".to_string(), "address"), + (rtt_text(Some(12_345)), "address"), (format!("{packets}/{}", packets + 1), "bytes"), (format!("{}/{}", packets * 2, (packets + 1) * 2), "bytes"), ] { diff --git a/crates/tsunagi-tray/src/exit.rs b/crates/tsunagi-tray/src/exit.rs index 045c62c..45314ed 100644 --- a/crates/tsunagi-tray/src/exit.rs +++ b/crates/tsunagi-tray/src/exit.rs @@ -109,6 +109,7 @@ mod tests { fn network(exit: ExitReport) -> NetworkReport { NetworkReport { peers: vec![PeerReport { + data_error: None, endpoint_id: "abcdef0123456789".into(), hostname: Some("music".into()), exit_node: true, diff --git a/crates/tsunagi-tray/src/format.rs b/crates/tsunagi-tray/src/format.rs index 259fbd4..5b9b191 100644 --- a/crates/tsunagi-tray/src/format.rs +++ b/crates/tsunagi-tray/src/format.rs @@ -24,19 +24,6 @@ pub(crate) fn short(id: &str) -> String { } } -/// A duration in seconds as a compact age (`5s`, `12m`, `3h 20m`, `2d 4h`). -pub(crate) fn age(secs: u64) -> String { - if secs < 60 { - format!("{secs}s") - } else if secs < 3_600 { - format!("{}m", secs / 60) - } else if secs < 86_400 { - format!("{}h {}m", secs / 3_600, (secs % 3_600) / 60) - } else { - format!("{}d {}h", secs / 86_400, (secs % 86_400) / 3_600) - } -} - /// A byte count in binary units. pub(crate) fn bytes(n: f64) -> String { const UNITS: [&str; 5] = ["B", "kB", "MB", "GB", "TB"]; diff --git a/crates/tsunagi-tray/src/stats.rs b/crates/tsunagi-tray/src/stats.rs index 44c44ae..f9ce5c3 100644 --- a/crates/tsunagi-tray/src/stats.rs +++ b/crates/tsunagi-tray/src/stats.rs @@ -6,10 +6,10 @@ //! key: a network total, and each peer within it. Both packets and bytes are //! tracked so the UI can switch between them. -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::time::Instant; -use tsunagi::ipc::StatusReport; +use tsunagi::ipc::{OverlayPeerReport, StatusReport}; /// How many rate samples to keep for a sparkline. const HISTORY: usize = 60; @@ -26,7 +26,7 @@ pub(crate) enum Unit { /// One series' rolling packet and byte rates. #[derive(Default)] pub(crate) struct Series { - last: Option<(Instant, u64, u64)>, + last: Option<(Instant, Counters)>, packets_rate: f32, bytes_rate: f32, packets_history: VecDeque, @@ -34,17 +34,52 @@ pub(crate) struct Series { } impl Series { + /// Samples a single cumulative counter pair (VIP histories own their lifetime). pub(crate) fn sample(&mut self, now: Instant, packets: u64, bytes: u64) { - if let Some((then, prev_packets, prev_bytes)) = self.last { + self.sample_peer( + now, + Counters { + epoch: 0, + tx_packets: packets, + tx_bytes: bytes, + rx_packets: 0, + rx_bytes: 0, + }, + ); + } + + fn sample_peer(&mut self, now: Instant, counters: Counters) { + if let Some((then, previous)) = self.last { let dt = now.saturating_duration_since(then).as_secs_f32(); - if dt > 0.0 { - self.packets_rate = packets.saturating_sub(prev_packets) as f32 / dt; - self.bytes_rate = bytes.saturating_sub(prev_bytes) as f32 / dt; - push(&mut self.packets_history, self.packets_rate); - push(&mut self.bytes_history, self.bytes_rate); + // Ignore duplicate/out-of-order observations without changing the baseline. + if dt <= 0.0 { + return; + } + if counters.epoch == previous.epoch + && counters.tx_packets >= previous.tx_packets + && counters.rx_packets >= previous.rx_packets + && counters.tx_bytes >= previous.tx_bytes + && counters.rx_bytes >= previous.rx_bytes + { + self.record( + (counters.tx_packets - previous.tx_packets) as f32 / dt + + (counters.rx_packets - previous.rx_packets) as f32 / dt, + (counters.tx_bytes - previous.tx_bytes) as f32 / dt + + (counters.rx_bytes - previous.rx_bytes) as f32 / dt, + ); + } else { + // A counter reset is a new baseline, never traffic. + *self = Self::default(); } } - self.last = Some((now, packets, bytes)); + self.last = Some((now, counters)); + } + + fn record(&mut self, packets: f32, bytes: f32) { + self.packets_rate = packets; + self.bytes_rate = bytes; + push(&mut self.packets_history, packets); + push(&mut self.bytes_history, bytes); } /// The current rate in the chosen unit. @@ -71,6 +106,27 @@ fn push(history: &mut VecDeque, value: f32) { } } +#[derive(Clone, Copy)] +struct Counters { + epoch: u64, + tx_packets: u64, + rx_packets: u64, + tx_bytes: u64, + rx_bytes: u64, +} + +impl From<&OverlayPeerReport> for Counters { + fn from(peer: &OverlayPeerReport) -> Self { + Self { + epoch: peer.counter_epoch, + tx_packets: peer.tx_packets, + rx_packets: peer.rx_packets, + tx_bytes: peer.tx_bytes, + rx_bytes: peer.rx_bytes, + } + } +} + /// All traffic series, keyed by a stable string. #[derive(Default)] pub(crate) struct Traffic { @@ -80,26 +136,32 @@ pub(crate) struct Traffic { impl Traffic { /// Folds a fresh snapshot into the rates. Call once per new generation. pub(crate) fn observe(&mut self, report: &StatusReport, now: Instant) { + let mut active = HashSet::new(); for network in &report.networks { let Some(overlay) = &network.overlay else { continue; }; - let (mut net_packets, mut net_bytes) = (0u64, 0u64); + let (mut packets_rate, mut bytes_rate) = (0.0, 0.0); for peer in &overlay.peers { - let packets = peer.tx_packets.saturating_add(peer.rx_packets); - let bytes = peer.tx_bytes.saturating_add(peer.rx_bytes); - net_packets = net_packets.saturating_add(packets); - net_bytes = net_bytes.saturating_add(bytes); - self.series - .entry(peer_key(&network.network_id, &peer.public_key)) - .or_default() - .sample(now, packets, bytes); + let key = peer_key(&network.network_id, &peer.public_key); + // Duplicate rows are not additional traffic. + if !active.insert(key.clone()) { + continue; + } + let series = self.series.entry(key).or_default(); + series.sample_peer(now, peer.into()); + packets_rate += series.rate(Unit::Packets); + bytes_rate += series.rate(Unit::Bytes); } + // Sum rates, not cumulative counters of a changing set of peers. + let key = net_key(&network.network_id); + active.insert(key.clone()); self.series - .entry(net_key(&network.network_id)) + .entry(key) .or_default() - .sample(now, net_packets, net_bytes); + .record(packets_rate, bytes_rate); } + self.series.retain(|key, _| active.contains(key)); } /// The network-total series, if sampled. @@ -120,3 +182,112 @@ fn net_key(network_id: &str) -> String { fn peer_key(network_id: &str, public_key: &str) -> String { format!("peer:{network_id}:{public_key}") } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tsunagi::ipc::{NetworkReport, OverlayPeerReport, OverlayReport}; + + fn peer(key: &str, bytes: u64) -> OverlayPeerReport { + OverlayPeerReport { + public_key: key.into(), + tx_bytes: bytes, + ..Default::default() + } + } + + fn report(peers: Vec) -> StatusReport { + StatusReport { + networks: vec![NetworkReport { + network_id: "network".into(), + overlay: Some(OverlayReport { + peers, + ..Default::default() + }), + ..Default::default() + }], + ..Default::default() + } + } + + #[test] + fn joining_peer_does_not_add_its_past_traffic_to_the_network_rate() { + let mut traffic = Traffic::default(); + let now = Instant::now(); + traffic.observe(&report(vec![peer("a", 100)]), now); + traffic.observe( + &report(vec![peer("a", 120), peer("b", 100_000_000_000)]), + now + Duration::from_secs(1), + ); + assert_eq!( + traffic.network("network").map(|s| s.rate(Unit::Bytes)), + Some(20.0) + ); + traffic.observe( + &report(vec![peer("a", 150), peer("b", 100_000_000_040)]), + now + Duration::from_secs(2), + ); + assert_eq!( + traffic.network("network").map(|s| s.rate(Unit::Bytes)), + Some(70.0) + ); + } + + #[test] + fn departed_peers_and_removed_networks_do_not_keep_stale_rates() { + let mut traffic = Traffic::default(); + let now = Instant::now(); + traffic.observe(&report(vec![peer("a", 100)]), now); + traffic.observe(&report(vec![peer("a", 140)]), now + Duration::from_secs(1)); + traffic.observe(&report(vec![]), now + Duration::from_secs(2)); + assert!(traffic.peer("network", "a").is_none()); + assert_eq!( + traffic.network("network").map(|s| s.rate(Unit::Bytes)), + Some(0.0) + ); + traffic.observe(&StatusReport::default(), now + Duration::from_secs(3)); + assert!(traffic.network("network").is_none()); + } + + #[test] + fn restarted_counters_are_a_new_baseline_even_if_they_already_exceed_the_old_value() { + let mut traffic = Traffic::default(); + let now = Instant::now(); + let mut old = peer("a", 100); + old.counter_epoch = 1; + traffic.observe(&report(vec![old]), now); + let mut restarted = peer("a", 100_000_000_000); + restarted.counter_epoch = 2; + traffic.observe( + &report(vec![restarted.clone()]), + now + Duration::from_secs(1), + ); + assert_eq!( + traffic.peer("network", "a").map(|s| s.rate(Unit::Bytes)), + Some(0.0) + ); + restarted.tx_bytes += 80; + traffic.observe(&report(vec![restarted]), now + Duration::from_secs(2)); + assert_eq!( + traffic.network("network").map(|s| s.rate(Unit::Bytes)), + Some(80.0) + ); + } + + #[test] + fn reset_in_one_direction_cannot_be_hidden_by_growth_in_the_other() { + let mut traffic = Traffic::default(); + let now = Instant::now(); + let mut old = peer("a", 100); + old.rx_bytes = 1000; + traffic.observe(&report(vec![old]), now); + let mut reset = peer("a", 50); + reset.rx_bytes = 10_000; + traffic.observe(&report(vec![reset]), now + Duration::from_secs(1)); + assert_eq!( + traffic.network("network").map(|s| s.rate(Unit::Bytes)), + Some(0.0) + ); + } +} diff --git a/crates/tsunagi-tray/src/ui.rs b/crates/tsunagi-tray/src/ui.rs index 17d89de..bb90249 100644 --- a/crates/tsunagi-tray/src/ui.rs +++ b/crates/tsunagi-tray/src/ui.rs @@ -364,7 +364,7 @@ fn draw_protocols( } else { network.protocols.join(", ") }; - egui::CollapsingHeader::new(format!("protocols {summary}")) + egui::CollapsingHeader::new(format!("data transports {summary}")) .id_salt(("protocols", &network.network_id)) .show(ui, |ui| { ui.weak("Preference order; changes apply immediately and are saved."); diff --git a/crates/tsunagi-wg-quic/Cargo.toml b/crates/tsunagi-wg-quic/Cargo.toml index 460d268..ff29bea 100644 --- a/crates/tsunagi-wg-quic/Cargo.toml +++ b/crates/tsunagi-wg-quic/Cargo.toml @@ -4,7 +4,7 @@ name = "tsunagi-wg-quic" # negotiates: a release here does not stop it talking to a peer on an older # build, because what peers compare is `ANNOUNCEMENT_VERSION`. version = "0.1.0" -description = "The wg-quic protocol for tsunagi: WireGuard's cryptography carried in iroh's QUIC datagrams." +description = "The single internal WireGuard tunnel for tsunagi, independent of external carriers." edition.workspace = true rust-version.workspace = true license.workspace = true @@ -30,6 +30,8 @@ rand = "0.10" boringtun = { version = "0.7.1", default-features = false } [dev-dependencies] +tsunagi-wireguard = { path = "../tsunagi-wireguard" } +tsunagi-tcp-tls = { path = "../tsunagi-tcp-tls" } tsunagi-core = { path = "../tsunagi-core", features = ["testing"] } tokio = { version = "1.53", features = ["rt", "rt-multi-thread", "sync", "time", "macros"] } tempfile.workspace = true diff --git a/crates/tsunagi-wg-quic/src/device.rs b/crates/tsunagi-wg-quic/src/device.rs index 3e9416d..699bbc9 100644 --- a/crates/tsunagi-wg-quic/src/device.rs +++ b/crates/tsunagi-wg-quic/src/device.rs @@ -59,7 +59,7 @@ const SCRATCH: usize = 4096; /// All three operations share the same mutex as the cryptographic state. struct FlowTunnel { tunn: Tunn, - queued: VecDeque, + queued: VecDeque<(FlowId, usize)>, } fn is_data(result: &TunnResult<'_>) -> bool { @@ -79,7 +79,7 @@ impl FlowTunnel { } // MAX_QUEUE_DEPTH in boringtun 0.7: the newest packet is dropped at 256. if self.queued.len() < 256 { - self.queued.push_back(flow); + self.queued.push_back((flow, packet.len())); } (result, 0) } @@ -88,19 +88,22 @@ impl FlowTunnel { &mut self, packet: &[u8], scratch: &'a mut [u8], - ) -> (TunnResult<'a>, FlowId) { + ) -> (TunnResult<'a>, FlowId, Option) { let result = self.tunn.decapsulate(None, packet, scratch); - let mut flow = 0; + let (mut flow, mut plaintext_len) = (0, None); if packet.is_empty() { if is_data(&result) { - flow = self.queued.pop_front().unwrap_or(0); + if let Some((tag, len)) = self.queued.pop_front() { + flow = tag; + plaintext_len = Some(len); + } } else if let Some(pending) = self.queued.pop_front() { // Without a session, draining pops the oldest IP packet and // encapsulate queues it again at the back. self.queued.push_back(pending); } } - (result, flow) + (result, flow, plaintext_len) } fn update_timers<'a>(&mut self, scratch: &'a mut [u8]) -> TunnResult<'a> { @@ -118,6 +121,7 @@ impl FlowTunnel { /// Counters for one peer's tunnel. #[derive(Debug, Default)] struct PeerCounters { + epoch: u64, tx_packets: AtomicU64, tx_bytes: AtomicU64, rx_packets: AtomicU64, @@ -132,6 +136,8 @@ struct PeerCounters { /// A snapshot of one peer's tunnel counters. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct PeerStats { + /// Identifies this counter lifetime; unchanged by rekeying or route switches. + pub counter_epoch: u64, /// Plaintext packets encrypted and sent to this peer. pub tx_packets: u64, /// Plaintext bytes encrypted and sent to this peer. @@ -209,6 +215,7 @@ impl Drop for Peer { impl Peer { fn stats(&self) -> PeerStats { PeerStats { + counter_epoch: self.counters.epoch, tx_packets: self.counters.tx_packets.load(Ordering::Relaxed), tx_bytes: self.counters.tx_bytes.load(Ordering::Relaxed), rx_packets: self.counters.rx_packets.load(Ordering::Relaxed), @@ -328,26 +335,27 @@ impl WireguardDevice { }; let (result, selected_flow) = tunn.encapsulate(packet, &mut scratch, flow); match result { - TunnResult::WriteToNetwork(out) => Some((out.len(), selected_flow)), + TunnResult::WriteToNetwork(out) => { + Some((out.len(), selected_flow, out.starts_with(&[4, 0, 0, 0]))) + } // No session yet, so nothing to send. Counted as dropped // rather than reported: the handshake is in flight and the // next packet will go. _ => None, } }; - let Some((len, selected_flow)) = result else { + let Some((len, selected_flow, data)) = result else { peer.counters .dropped_no_session .fetch_add(1, Ordering::Relaxed); return false; }; - send_to_peer_flow(&peer, &scratch[..len], selected_flow); - peer.counters.tx_packets.fetch_add(1, Ordering::Relaxed); - peer.counters - .tx_bytes - .fetch_add(packet.len() as u64, Ordering::Relaxed); - true + let sent = send_to_peer_flow(&peer, &scratch[..len], selected_flow); + if sent && data { + count_sent(&peer, packet.len()); + } + sent } /// Adds or replaces a peer and starts its tunnel. @@ -388,7 +396,10 @@ impl WireguardDevice { queued: VecDeque::new(), }), link, - counters: Arc::new(PeerCounters::default()), + counters: Arc::new(PeerCounters { + epoch: rand::random(), + ..Default::default() + }), task: Mutex::new(None), }); @@ -501,9 +512,9 @@ fn send_to_peer(peer: &Peer, payload: &[u8]) { send_to_peer_flow(peer, payload, 0); } -fn send_to_peer_flow(peer: &Peer, payload: &[u8], flow: FlowId) { +fn send_to_peer_flow(peer: &Peer, payload: &[u8], flow: FlowId) -> bool { match peer.link.send_flow(Bytes::copy_from_slice(payload), flow) { - Ok(()) => {} + Ok(()) => return true, Err(TransportError::TooLarge { .. }) => { peer.counters .dropped_oversize @@ -514,6 +525,14 @@ fn send_to_peer_flow(peer: &Peer, payload: &[u8], flow: FlowId) { tracing::trace!(%err, "dropping a WireGuard packet the link refused"); } } + false +} + +fn count_sent(peer: &Peer, plaintext_len: usize) { + peer.counters.tx_packets.fetch_add(1, Ordering::Relaxed); + peer.counters + .tx_bytes + .fetch_add(plaintext_len as u64, Ordering::Relaxed); } async fn read_from_link(inner: Arc, peer: Arc) { @@ -532,9 +551,12 @@ async fn read_from_link(inner: Arc, peer: Arc) { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), }; - let (result, flow) = tunn.decapsulate(input.unwrap_or(&[]), &mut scratch); + let (result, flow, plaintext_len) = + tunn.decapsulate(input.unwrap_or(&[]), &mut scratch); match result { - TunnResult::WriteToNetwork(out) => Outcome::ToNetwork(out.len(), flow), + TunnResult::WriteToNetwork(out) => { + Outcome::ToNetwork(out.len(), flow, plaintext_len) + } // The source boringtun reports is not consulted here: // whether the peer may use it is checked where the // claims live. @@ -549,8 +571,12 @@ async fn read_from_link(inner: Arc, peer: Arc) { }; match outcome { - Outcome::ToNetwork(len, flow) => { - send_to_peer_flow(&peer, &scratch[..len], flow); + Outcome::ToNetwork(len, flow, plaintext_len) => { + if send_to_peer_flow(&peer, &scratch[..len], flow) + && let Some(len) = plaintext_len + { + count_sent(&peer, len); + } // Keep draining with an empty datagram, as boringtun asks. input = None; continue; @@ -584,7 +610,7 @@ async fn read_from_link(inner: Arc, peer: Arc) { } enum Outcome { - ToNetwork(usize, FlowId), + ToNetwork(usize, FlowId, Option), ToTunnel(usize), Done, Failed, @@ -671,7 +697,12 @@ mod flow_tests { let keepalive = wire(a.decapsulate(&response, &mut scratch).0); let _ = b.decapsulate(&keepalive, &mut scratch); for expected in 100..356 { - let (result, flow) = a.decapsulate(&[], &mut scratch); + let (result, flow, len) = a.decapsulate(&[], &mut scratch); + assert_eq!( + len, + Some(packet.len()), + "queued counters retain the exact unpadded IP length" + ); assert_eq!(flow, expected); let ciphertext = wire(result); match b.decapsulate(&ciphertext, &mut scratch).0 { diff --git a/crates/tsunagi-wg-quic/src/plugin.rs b/crates/tsunagi-wg-quic/src/plugin.rs index 3f6f695..7910ea0 100644 --- a/crates/tsunagi-wg-quic/src/plugin.rs +++ b/crates/tsunagi-wg-quic/src/plugin.rs @@ -47,13 +47,9 @@ use crate::store::WgKeyStore; use tsunagi::state::Ipv4Range; use tsunagi::task::{TASK_GRACE, wind_down}; -/// The protocol identifier this plugin announces. -/// The protocol id of this plugin. -/// -/// `wg-quic`, because that is what it is: WireGuard's cryptography carried -/// in QUIC datagrams. The name is on the wire, so it is a protocol name and -/// not a description of the implementation. -pub const WIREGUARD_PROTOCOL: &str = "wg-quic"; +/// Fixed internal encryption capability, independent of external carrier names. +/// The historical crate/storage name does not select a physical transport. +pub const WIREGUARD_PROTOCOL: &str = "wireguard-e2e"; /// Smallest interface MTU the overlay accepts. /// @@ -79,8 +75,8 @@ pub const WIREGUARD_OVERHEAD: u32 = 32; /// Configuration of the WireGuard plugin. #[derive(Debug, Clone)] pub struct WireguardConfig { - /// Diagnostic protocol id when this end-to-end engine is reused by another carrier. - pub protocol: &'static str, + /// Previous key stores to import before creating any network key. + pub legacy_key_stores: Vec, /// Directory for the plugin's own key store. Separate from agent state. pub state_dir: PathBuf, /// WireGuard keepalive, which keeps tunnels and their links warm. @@ -102,7 +98,7 @@ impl WireguardConfig { /// Creates a configuration rooted at `state_dir` with sensible defaults. pub fn new(state_dir: impl Into) -> Self { Self { - protocol: WIREGUARD_PROTOCOL, + legacy_key_stores: Vec::new(), state_dir: state_dir.into(), keepalive: Some(25), mtu: DEFAULT_MTU, @@ -302,9 +298,16 @@ impl WireguardPlugin { } let path = config.key_store_path(); - let store = tokio::task::spawn_blocking(move || WgKeyStore::open(path)) - .await - .map_err(|err| PluginError::Other(format!("key store task failed: {err}")))??; + let legacy = config.legacy_key_stores.clone(); + let store = tokio::task::spawn_blocking(move || { + let store = WgKeyStore::open(path)?; + for path in legacy { + store.import_legacy(&path)?; + } + Ok::<_, PluginError>(store) + }) + .await + .map_err(|err| PluginError::Other(format!("key store task failed: {err}")))??; let worker = Arc::new(Worker { config, @@ -397,7 +400,7 @@ impl Worker { fn report(&self, network: NetworkId, reason: impl std::fmt::Display) { tracing::warn!(network = %network.fmt_short(), %reason, "wireguard plugin error"); if let Some(context) = self.context.get() { - context.report_error(network, self.config.protocol, reason.to_string()); + context.report_error(network, WIREGUARD_PROTOCOL, reason.to_string()); } } @@ -650,7 +653,7 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { impl IpPlugin for WireguardPlugin { fn protocol_id(&self) -> &str { - self.worker.config.protocol + WIREGUARD_PROTOCOL } fn protocol_version(&self) -> u16 { @@ -689,7 +692,7 @@ impl IpPlugin for WireguardPlugin { // Identity only. Where to send packets is the transport's business. let announcement = WgAnnouncement::new(network, &state.key.public()); Ok(Some(PluginCapability { - protocol: self.worker.config.protocol.to_string(), + protocol: WIREGUARD_PROTOCOL.to_string(), version: crate::announcement::ANNOUNCEMENT_VERSION, enabled: true, data: announcement.encode()?, @@ -795,14 +798,26 @@ impl IpPlugin for WireguardPlugin { self.nudge(Command::Teardown(network)); } - fn on_network_forgotten(&self, network: NetworkId) { - // The key is this protocol's identity in that network and nothing - // else's. Keeping it after leaving would keep a secret for a - // network this agent is no longer in — and hand back the same - // overlay address on a rejoin that everyone else has moved past. - if let Err(err) = self.worker.store.forget(network) { - tracing::warn!(%err, "cannot remove the WireGuard key of a network we left"); - } + fn on_network_forgotten(&self, network: NetworkId) -> BoxFuture<'_, ()> { + let worker = Arc::clone(&self.worker); + Box::pin(async move { + // Remove old copies as well, otherwise the next startup's import + // would resurrect a key that the user explicitly forgot. + let result = tokio::task::spawn_blocking(move || { + for path in &worker.config.legacy_key_stores { + if path.exists() { + WgKeyStore::open(path)?.forget(network)?; + } + } + worker.store.forget(network) + }) + .await; + match result { + Ok(Ok(())) => {} + Ok(Err(err)) => tracing::warn!(%err, "cannot remove forgotten network data keys"), + Err(err) => tracing::warn!(%err, "data key cleanup task failed"), + } + }) } fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> { diff --git a/crates/tsunagi-wg-quic/src/store.rs b/crates/tsunagi-wg-quic/src/store.rs index d00f4dd..79e6007 100644 --- a/crates/tsunagi-wg-quic/src/store.rs +++ b/crates/tsunagi-wg-quic/src/store.rs @@ -106,6 +106,49 @@ impl WgKeyStore { } } + /// Imports missing network keys from an older carrier-specific store, atomically. + /// The source is read-only and existing canonical keys always win. + pub fn import_legacy(&self, path: &Path) -> Result<(), PluginError> { + if !path.exists() || path == self.path { + return Ok(()); + } + let fail = |err: rusqlite::Error| { + PluginError::Other(format!("cannot import legacy data keys: {err}")) + }; + let source = Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(fail)?; + let integrity: String = source + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .map_err(fail)?; + let schema: i64 = source + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(fail)?; + if integrity != "ok" || schema != SCHEMA_VERSION { + return Err(PluginError::Other( + "legacy data key store is damaged or has an unsupported schema".into(), + )); + } + let mut statement = source.prepare("SELECT network_id, secret, created_at, length(network_id), length(secret) FROM network_keys").map_err(fail)?; + let mut rows = statement.query([]).map_err(fail)?; + let mut target = self.lock(); + let transaction = target.transaction().map_err(fail)?; + while let Some(row) = rows.next().map_err(fail)? { + let id_len: i64 = row.get(3).map_err(fail)?; + let key_len: i64 = row.get(4).map_err(fail)?; + if id_len != 32 || key_len != KEY_LEN as i64 { + return Err(PluginError::Other( + "legacy data key has an invalid length".into(), + )); + } + let id: Vec = row.get(0).map_err(fail)?; + let key = zeroize::Zeroizing::new(row.get::<_, Vec>(1).map_err(fail)?); + let created: i64 = row.get(2).map_err(fail)?; + transaction.execute("INSERT OR IGNORE INTO network_keys(network_id, secret, created_at) VALUES (?1, ?2, ?3)", params![id, key.as_slice(), created]).map_err(fail)?; + } + transaction.commit().map_err(fail)?; + Ok(()) + } + /// Returns this agent's key for a network, creating it on first use. pub fn load_or_create(&self, network: NetworkId) -> Result { let conn = self.lock(); @@ -219,4 +262,70 @@ mod tests { "a damaged key store must not silently mint a new identity (was {original})" ); } + + #[tokio::test] + async fn forgetting_an_imported_key_cannot_resurrect_it_on_restart() { + use crate::{WireguardConfig, WireguardPlugin}; + use tsunagi::dataplane::IpPlugin; + let dir = tempfile::TempDir::new().unwrap(); + let id = network("forgotten-import"); + let legacy_path = dir.path().join("old.sqlite"); + let original = WgKeyStore::open(&legacy_path) + .unwrap() + .load_or_create(id) + .unwrap() + .public(); + let mut config = WireguardConfig::new(dir.path().join("canonical")); + config.legacy_key_stores.push(legacy_path); + let plugin = WireguardPlugin::open(config.clone()).await.unwrap(); + plugin.on_network_forgotten(id).await; + plugin.shutdown().await; + let restarted = WireguardPlugin::open(config.clone()).await.unwrap(); + let key = WgKeyStore::open(config.key_store_path()) + .unwrap() + .load_or_create(id) + .unwrap() + .public(); + assert_ne!(key, original); + restarted.shutdown().await; + } + + #[test] + fn legacy_import_preserves_keys_and_never_overwrites_the_canonical_identity() { + let dir = tempfile::TempDir::new().unwrap(); + let legacy = WgKeyStore::open(dir.path().join("legacy.sqlite")).unwrap(); + let canonical = WgKeyStore::open(dir.path().join("canonical.sqlite")).unwrap(); + let imported = network("imported"); + let existing = network("existing"); + let imported_key = legacy.load_or_create(imported).unwrap().public(); + legacy.load_or_create(existing).unwrap(); + let existing_key = canonical.load_or_create(existing).unwrap().public(); + canonical.import_legacy(legacy.path()).unwrap(); + canonical.import_legacy(legacy.path()).unwrap(); + assert_eq!( + canonical.load_or_create(imported).unwrap().public(), + imported_key + ); + assert_eq!( + canonical.load_or_create(existing).unwrap().public(), + existing_key + ); + assert_eq!( + legacy.load_or_create(imported).unwrap().public(), + imported_key + ); + } + + #[test] + fn a_damaged_legacy_store_aborts_import_without_regenerating_keys() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("legacy.sqlite"); + std::fs::write(&path, b"damaged SQLite").unwrap(); + let canonical = WgKeyStore::open(dir.path().join("canonical.sqlite")).unwrap(); + assert!(canonical.import_legacy(&path).is_err()); + canonical + .import_legacy(&dir.path().join("absent.sqlite")) + .unwrap(); + assert!(!dir.path().join("absent.sqlite").exists()); + } } diff --git a/crates/tsunagi-wg-quic/tests/carriers.rs b/crates/tsunagi-wg-quic/tests/carriers.rs new file mode 100644 index 0000000..7654be4 --- /dev/null +++ b/crates/tsunagi-wg-quic/tests/carriers.rs @@ -0,0 +1,285 @@ +//! Real membership and one internal cipher over each independent external carrier. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use bytes::Bytes; +use iroh::EndpointId; +use std::{ + net::{Ipv4Addr, SocketAddr}, + sync::Arc, + time::Duration, +}; +use tempfile::TempDir; +use tsunagi::dataplane::transport::{DataCarrier, PacketTransport, SharedLink, TransportError}; +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::overlay::{MemoryTun, MemoryTunFactory}; +use tsunagi::testing::{config_with, network, wait_for_peers, wait_until}; +use tsunagi::{Agent, BoxFuture, NetworkId}; +use tsunagi_tcp_tls::{TcpTlsCarrier, TcpTlsTransport}; +use tsunagi_wg_quic::{WireguardConfig, WireguardPlugin}; +use tsunagi_wireguard::{UdpCarrier, UdpTransport}; + +// Only address selection is constrained to loopback. TLS sockets, identity +// verification, framing, membership and both encryption layers remain real. +#[derive(Debug)] +struct LoopbackTls { + inner: Arc, + ports: Arc>>, +} +impl PacketTransport for LoopbackTls { + fn name(&self) -> &str { + "tcp-tls" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn open<'a>( + &'a self, + network: NetworkId, + peer: EndpointId, + protocol: &'a str, + ) -> BoxFuture<'a, Result> { + let port = self + .ports + .lock() + .unwrap() + .iter() + .find(|(id, _)| *id == peer) + .unwrap() + .1; + self.inner + .replace_peer_addrs(peer, vec![SocketAddr::from((Ipv4Addr::LOCALHOST, port))]); + self.inner.open(network, peer, protocol) + } +} + +struct Node { + _root: TempDir, + agent: Agent, + tunnel: Arc, + tuns: MemoryTunFactory, + accept: tokio::task::JoinHandle<()>, +} +impl Node { + async fn new( + discovery: &SharedMemoryDiscovery, + protocol: &str, + ports: Arc>>, + ) -> Self { + Self::configured(discovery, protocol, ports, true).await + } + async fn configured( + discovery: &SharedMemoryDiscovery, + protocol: &str, + ports: Arc>>, + register_transport: bool, + ) -> Self { + let root = TempDir::new().unwrap(); + let identity = tsunagi::storage::StateStore::open( + tsunagi::StoragePaths::under(root.path()).state_db(), + ) + .unwrap() + .load_or_create_device_identity() + .unwrap(); + let tunnel = WireguardPlugin::open( + WireguardConfig::new(root.path().join("cipher")) + .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), + ) + .await + .unwrap(); + let tuns = MemoryTunFactory::new(); + let config = config_with(root.path(), discovery) + .with_interface(Arc::new(tuns.clone()), "carrier-test", 1280) + .with_tunnel(tunnel.clone()); + let (agent, accept) = if protocol == "wg" { + let transport = + UdpTransport::wireguard(identity.endpoint_id(), Ipv4Addr::LOCALHOST, None); + let carrier: Arc = UdpCarrier::new(transport.clone()); + let mut config = config.with_carrier(carrier); + if register_transport { + config = config.with_custom_transport(protocol, transport.clone()); + } + let agent = Agent::spawn(config).await.unwrap(); + let incoming = agent.clone(); + let accept = tokio::spawn(async move { + while let Some(link) = transport.next_inbound().await { + let _ = incoming.install_inbound_link(link).await; + } + }); + (agent, accept) + } else { + let transport = Arc::new( + TcpTlsTransport::bind(&identity, Some(0), None) + .await + .unwrap(), + ); + ports + .lock() + .unwrap() + .push((identity.endpoint_id(), transport.bound_port())); + let carrier: Arc = TcpTlsCarrier::new(transport.clone()); + let custom = Arc::new(LoopbackTls { + inner: transport.clone(), + ports, + }); + let agent = Agent::spawn( + config + .with_carrier(carrier) + .with_custom_transport(protocol, custom), + ) + .await + .unwrap(); + let incoming = agent.clone(); + let accept = tokio::spawn(async move { + transport + .accept_loop(move |link| { + let incoming = incoming.clone(); + tokio::spawn(async move { + let _ = incoming.install_inbound_link(link).await; + }); + }) + .await; + }); + (agent, accept) + }; + Self { + _root: root, + agent, + tunnel, + tuns, + accept, + } + } + fn tun(&self) -> Arc { + self.tuns + .device(&self.agent.overlay().unwrap().interface) + .unwrap() + } + fn addr(&self, network: NetworkId) -> Ipv4Addr { + self.tunnel + .overview(network) + .unwrap() + .overlay_address_v4 + .unwrap() + } +} + +#[tokio::test] +async fn missing_native_transport_cannot_silently_turn_wireguard_into_quic() { + let discovery = SharedMemoryDiscovery::new(); + let ports = Arc::default(); + let a = Node::configured(&discovery, "wg", Arc::clone(&ports), false).await; + let b = Node::configured(&discovery, "wg", ports, false).await; + let (name, secret) = network("no-disguised-quic-fallback"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&a.agent, id, 1).await; + let initiator = if a.agent.endpoint_id().as_bytes() < b.agent.endpoint_id().as_bytes() { + &a + } else { + &b + }; + wait_until( + "missing carrier implementation is explicitly refused", + || async { + let status = initiator.agent.network_status(id).await.ok()?; + let peer = status.peers.first()?; + (peer + .data_error + .as_deref() + .is_some_and(|reason| reason.contains("registered")) + && matches!(peer.data_path, tsunagi::agent::DataPath::None)) + .then_some(()) + }, + ) + .await; + a.agent.shutdown().await; + b.agent.shutdown().await; + a.accept.abort(); + b.accept.abort(); +} + +#[tokio::test] +async fn wireguard_and_tls_carriers_both_carry_the_same_internal_cipher_and_exact_plaintext_counters() + { + for protocol in ["wg", "tcp-tls"] { + let discovery = SharedMemoryDiscovery::new(); + let ports = Arc::default(); + let a = Node::new(&discovery, protocol, Arc::clone(&ports)).await; + let b = Node::new(&discovery, protocol, ports).await; + let (name, secret) = network(protocol); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&a.agent, id, 1).await; + for node in [&a, &b] { + wait_until("real internal handshake over selected carrier", || async { + node.tunnel + .overview(id)? + .peers + .first()? + .tunnel + .as_ref()? + .health + .is_up() + .then_some(()) + }) + .await; + let status = node.agent.network_status(id).await.unwrap(); + assert_eq!(status.protocols, [protocol]); + assert!( + matches!(&status.peers[0].data_path, tsunagi::agent::DataPath::Direct { transport } if transport == protocol) + ); + } + for (source, destination) in [(&a, &b), (&b, &a)] { + let mut total = 0; + for len in [60, 1097, 1280] { + let mut packet = vec![0; len]; + packet[0] = 0x45; + packet[2..4].copy_from_slice(&(len as u16).to_be_bytes()); + packet[8] = 64; + packet[9] = 253; + packet[12..16].copy_from_slice(&source.addr(id).octets()); + packet[16..20].copy_from_slice(&destination.addr(id).octets()); + let packet = Bytes::from(packet); + source.tun().push_from_os(packet.clone()); + assert_eq!( + tokio::time::timeout(tsunagi::testing::DEADLINE, destination.tun().pop_to_os()) + .await + .unwrap() + .unwrap(), + packet + ); + total += len as u64; + } + wait_until( + "plaintext bytes count once without health probes or nested encryption overhead", + || async { + let tx = source + .tunnel + .overview(id)? + .peers + .first()? + .tunnel + .as_ref()? + .stats; + let rx = destination + .tunnel + .overview(id)? + .peers + .first()? + .tunnel + .as_ref()? + .stats; + (tx.tx_bytes == total + && rx.rx_bytes == total + && tx.tx_packets == 3 + && rx.rx_packets == 3) + .then_some(()) + }, + ) + .await; + } + a.agent.shutdown().await; + b.agent.shutdown().await; + a.accept.abort(); + b.accept.abort(); + } +} diff --git a/crates/tsunagi-wg-quic/tests/exit_node.rs b/crates/tsunagi-wg-quic/tests/exit_node.rs index 56229ba..37d165f 100644 --- a/crates/tsunagi-wg-quic/tests/exit_node.rs +++ b/crates/tsunagi-wg-quic/tests/exit_node.rs @@ -101,7 +101,10 @@ impl Node { tag, 1280, ) - .with_plugin(plugin.clone() as Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); diff --git a/crates/tsunagi-wg-quic/tests/interface_provisioning.rs b/crates/tsunagi-wg-quic/tests/interface_provisioning.rs index f24d2df..6a84150 100644 --- a/crates/tsunagi-wg-quic/tests/interface_provisioning.rs +++ b/crates/tsunagi-wg-quic/tests/interface_provisioning.rs @@ -47,7 +47,10 @@ impl HostedAgent { config_with(dir.path(), discovery) .with_overlay_ipv4_range(Some(DEFAULT_IPV4_RANGE)) .with_interface(factory, tag, 1280) - .with_plugin(plugin.clone() as Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); diff --git a/crates/tsunagi-wg-quic/tests/local_control.rs b/crates/tsunagi-wg-quic/tests/local_control.rs index f6c5591..34f001c 100644 --- a/crates/tsunagi-wg-quic/tests/local_control.rs +++ b/crates/tsunagi-wg-quic/tests/local_control.rs @@ -37,6 +37,7 @@ fn source(agent: Agent, plugin: Arc) -> Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -122,7 +126,10 @@ async fn a_client_sees_the_agent_and_its_overlay() { let agent_b = Agent::spawn( config_with(dir_b.path(), &discovery) .with_interface(Arc::new(MemoryTunFactory::new()), "tcb0", 1280) - .with_plugin(plugin_b.clone() as Arc), + .with_tunnel(plugin_b.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); diff --git a/crates/tsunagi-wg-quic/tests/wireguard.rs b/crates/tsunagi-wg-quic/tests/wireguard.rs index cd338c2..bd8ef17 100644 --- a/crates/tsunagi-wg-quic/tests/wireguard.rs +++ b/crates/tsunagi-wg-quic/tests/wireguard.rs @@ -121,7 +121,10 @@ impl WgAgent { let agent = Agent::spawn( config_with(dir.path(), discovery) .with_interface(Arc::new(tuns.clone()), tag, 1280) - .with_plugin(plugin.clone() as Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -193,7 +196,10 @@ impl WgAgent { config_with(root, discovery) .with_overlay_ipv4_range(range) .with_interface(Arc::new(tuns.clone()), tag, 1280) - .with_plugin(plugin.clone() as Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -225,7 +231,10 @@ impl WgAgent { .with_overlay_ipv4_range(Some(tsunagi::state::DEFAULT_IPV4_RANGE)) .with_interface(Arc::new(tuns.clone()), tag, 1280) .with_unreachable_data_peers(blocked) - .with_plugin(plugin.clone() as Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -254,7 +263,10 @@ impl WgAgent { config_with(dir.path(), discovery) .with_overlay_ipv4_range(Some(tsunagi::state::DEFAULT_IPV4_RANGE)) .with_interface(Arc::new(PretendHostTuns(tuns.clone())), tag, 1280) - .with_plugin(plugin.clone() as Arc), + .with_tunnel(plugin.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -458,10 +470,52 @@ async fn two_agents_carry_real_ip_packets_through_a_wireguard_tunnel() { let view = a.plugin.overview(network_id).unwrap(); let tunnel = view.peers[0].tunnel.as_ref().unwrap(); assert!(tunnel.health.is_up()); - assert!(tunnel.stats.tx_packets >= 1); - assert!(tunnel.stats.rx_packets >= 1); + wait_until( + "plaintext counters exclude carrier probes and encryption overhead", + || async { + let view = a.plugin.overview(network_id)?; + let stats = view.peers[0].tunnel.as_ref()?.stats; + (stats.tx_packets == 1 + && stats.rx_packets == 1 + && stats.tx_bytes == (20 + payload.len()) as u64 + && stats.rx_bytes == (20 + b"and back".len()) as u64) + .then_some(()) + }, + ) + .await; assert_eq!(tunnel.stats.dropped_wrong_source, 0); + let before = a.plugin.overview(network_id).unwrap().peers[0] + .tunnel + .as_ref() + .unwrap() + .stats; + a.agent.set_protocols(network_id, Vec::new()).await.unwrap(); + wait_until("no remaining physical path", || async { + matches!( + a.agent.network_status(network_id).await.ok()?.peers[0].data_path, + tsunagi::agent::DataPath::None + ) + .then_some(()) + }) + .await; + assert!(!a.plugin.carry( + network_id, + b.agent.endpoint_id(), + ipv4_packet(addr_a, addr_b, b"refused by transport") + )); + let after = a.plugin.overview(network_id).unwrap().peers[0] + .tunnel + .as_ref() + .unwrap() + .stats; + assert_eq!(after.counter_epoch, before.counter_epoch); + assert_eq!( + after.tx_bytes, before.tx_bytes, + "a refused send is not transmitted traffic" + ); + assert_eq!(after.tx_packets, before.tx_packets); + a.shutdown().await; b.shutdown().await; } @@ -1151,10 +1205,12 @@ async fn the_core_carries_the_payload_without_interpreting_it() { let capability = wait_until("the peer's capability arrived", || async { let status: NetworkStatus = a.agent.network_status(network_id).await.ok()?; - status - .peers - .first() - .and_then(|peer| peer.capabilities.first().cloned()) + status.peers.first().and_then(|peer| { + peer.capabilities + .iter() + .find(|cap| cap.protocol == WIREGUARD_PROTOCOL) + .cloned() + }) }) .await; assert_eq!(capability.protocol, WIREGUARD_PROTOCOL); @@ -1197,7 +1253,10 @@ async fn an_announcement_for_another_network_never_reaches_a_tunnel() { let attacker_dir = TempDir::new().unwrap(); let attacker = Agent::spawn( config_with(attacker_dir.path(), &discovery) - .with_plugin(forger.clone() as Arc), + .with_tunnel(forger.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -1246,7 +1305,11 @@ async fn a_peer_at_another_protocol_version_gets_no_data_plane_and_keeps_the_con let ahead = Arc::new(FromTheFuture); let dir = TempDir::new().unwrap(); let other = Agent::spawn( - config_with(dir.path(), &discovery).with_plugin(ahead.clone() as Arc), + config_with(dir.path(), &discovery) + .with_tunnel(ahead.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -1273,10 +1336,16 @@ async fn a_peer_at_another_protocol_version_gets_no_data_plane_and_keeps_the_con ); let status = here.agent.network_status(network_id).await.unwrap(); assert_eq!(status.peers.len(), 1, "the session is up all the same"); + assert_eq!( + status.peers[0].protocols, + ["wg-quic"], + "external capability remains independent" + ); assert!( - status.peers[0].protocols.is_empty(), - "nothing was agreed with it: {:?}", - status.peers[0].protocols + status.peers[0] + .data_error + .as_deref() + .is_some_and(|reason| reason.contains("update both agents")) ); // And the control plane carries a message to it regardless. @@ -1317,7 +1386,11 @@ async fn agreement_turns_on_the_protocol_version_and_nothing_else() { }); let dir = TempDir::new().unwrap(); let other = Agent::spawn( - config_with(dir.path(), &discovery).with_plugin(stranger.clone() as Arc), + config_with(dir.path(), &discovery) + .with_tunnel(stranger.clone() as Arc) + .with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )), ) .await .unwrap(); @@ -1332,8 +1405,12 @@ async fn agreement_turns_on_the_protocol_version_and_nothing_else() { status .peers .first()? - .protocols - .contains(&WIREGUARD_PROTOCOL.to_string()) + .capabilities + .iter() + .any(|cap| { + cap.protocol == WIREGUARD_PROTOCOL + && cap.version == tsunagi_wg_quic::ANNOUNCEMENT_VERSION + }) .then_some(()) }) .await; diff --git a/crates/tsunagi-wireguard/Cargo.toml b/crates/tsunagi-wireguard/Cargo.toml index 65e1418..11fccbd 100644 --- a/crates/tsunagi-wireguard/Cargo.toml +++ b/crates/tsunagi-wireguard/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "tsunagi-wireguard" version = "0.1.0" -description = "Pure WireGuard direct UDP transport and protocol plugin for tsunagi mesh networks." +description = "Independent authenticated WireGuard and Amnezia UDP carriers for tsunagi." edition.workspace = true rust-version.workspace = true -license.workspace = true +license = "WTFPL AND Apache-2.0" repository.workspace = true [dependencies] @@ -14,14 +14,10 @@ tokio = { workspace = true, features = ["net", "io-util", "time", "sync", "macro serde.workspace = true postcard.workspace = true bytes.workspace = true -thiserror.workspace = true tracing.workspace = true -data-encoding.workspace = true -hex.workspace = true -rusqlite = { version = "0.40", features = ["bundled"] } -subtle = "2.6" zeroize = { version = "1.9", features = ["derive"] } rand = "0.10" +chacha20 = "0.9" boringtun = { version = "0.7.1", default-features = false } netdev = "0.46" diff --git a/crates/tsunagi-wireguard/LICENSE-AmneziaWG-RS b/crates/tsunagi-wireguard/LICENSE-AmneziaWG-RS new file mode 100644 index 0000000..bc588c9 --- /dev/null +++ b/crates/tsunagi-wireguard/LICENSE-AmneziaWG-RS @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Friderika Coulson (NanoBee) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/tsunagi-wireguard/src/addresses.rs b/crates/tsunagi-wireguard/src/addresses.rs new file mode 100644 index 0000000..de866bd --- /dev/null +++ b/crates/tsunagi-wireguard/src/addresses.rs @@ -0,0 +1,168 @@ +//! Local underlay candidates, shared by the UDP carriers. +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +/// Discovers local network interface IPs to announce to peers, excluding loopback, overlay interfaces, and virtual bridges. +pub fn local_ip_candidates() -> Vec { + local_ip_candidates_excluding(None) +} + +pub(crate) fn local_ip_candidates_excluding( + excluded_interface: Option<&str>, +) -> Vec { + let mut candidates = Vec::new(); + for iface in netdev::get_interfaces() { + if excluded_interface.is_some_and(|name| { + iface.name.eq_ignore_ascii_case(name) + || iface + .friendly_name + .as_deref() + .is_some_and(|friendly| friendly.eq_ignore_ascii_case(name)) + }) || !is_candidate_interface(&iface) + { + continue; + } + for ip in &iface.ipv4 { + let addr = ip.addr(); + if is_candidate_ipv4(addr) { + let priority = ip_priority(&iface, IpAddr::V4(addr)); + candidates.push((priority, IpAddr::V4(addr))); + } + } + } + // Sort descending by priority so the most viable physical / WAN / LAN IP is candidate #0 + candidates.sort_by_key(|b| std::cmp::Reverse(b.0)); + let mut seen = std::collections::HashSet::new(); + let mut ips = Vec::new(); + for (_, ip) in candidates { + if seen.insert(ip) { + ips.push(ip); + } + } + ips +} + +pub(crate) fn is_candidate_interface(iface: &netdev::Interface) -> bool { + if !iface.is_up() || iface.is_loopback() { + return false; + } + + let name = iface.name.to_lowercase(); + let friendly = iface + .friendly_name + .as_deref() + .map(str::to_lowercase) + .unwrap_or_default(); + let desc = iface + .description + .as_deref() + .map(str::to_lowercase) + .unwrap_or_default(); + + // Skip Tsunagi / Wintun overlay interfaces + if name.starts_with("tsun") + || name.contains("tsunagi") + || name.contains("wintun") + || friendly.starts_with("tsun") + || friendly.contains("tsunagi") + || friendly.contains("wintun") + || desc.contains("wintun") + || desc.contains("tsunagi") + { + return false; + } + + // Skip container interfaces and virtual bridges purely internal to container engines + if name.starts_with("docker") + || name.starts_with("br-") + || name.starts_with("veth") + || name.starts_with("cni") + || name.starts_with("flannel") + || name.starts_with("cali") + || name.starts_with("kube") + || name.starts_with("dummy") + || name.starts_with("lxc") + || name.starts_with("podman") + || friendly.contains("wsl") + { + return false; + } + + true +} + +/// Reorders candidate addresses so that candidates on the same local subnet as one of our +/// active network interfaces appear first. +pub fn prioritize_local_subnet(candidates: &mut [SocketAddr]) { + let local_subnets: Vec<(Ipv4Addr, u8)> = netdev::get_interfaces() + .into_iter() + .filter(|iface| iface.is_up() && !iface.is_loopback()) + .flat_map(|iface| { + iface + .ipv4 + .into_iter() + .map(|net| (net.addr(), net.prefix_len())) + }) + .collect(); + + candidates.sort_by_key(|addr| { + let is_local = match addr.ip() { + IpAddr::V4(v4) => local_subnets + .iter() + .any(|(local_ip, prefix_len)| matches_subnet(*local_ip, *prefix_len, v4)), + IpAddr::V6(_) => false, + }; + if is_local { 0 } else { 1 } + }); +} + +fn matches_subnet(local_ip: Ipv4Addr, prefix_len: u8, target_ip: Ipv4Addr) -> bool { + if prefix_len == 0 { + return false; + } + let mask = if prefix_len >= 32 { + !0u32 + } else { + !0u32 << (32 - prefix_len) + }; + (u32::from(local_ip) & mask) == (u32::from(target_ip) & mask) +} + +pub(crate) fn is_candidate_ipv4(addr: std::net::Ipv4Addr) -> bool { + if addr.is_loopback() || addr.is_unspecified() || addr.is_link_local() || addr.is_broadcast() { + return false; + } + if addr.is_documentation() { + return false; + } + true +} + +pub(crate) fn ip_priority(iface: &netdev::Interface, ip: IpAddr) -> u32 { + let has_gateway = iface.gateway.is_some(); + let base = if has_gateway { 100 } else { 0 }; + match ip { + IpAddr::V4(v4) => { + let octets = v4.octets(); + // Tailscale: 100.64.0.0/10 + let is_tailscale = octets[0] == 100 && (octets[1] & 0xc0) == 64; + let is_private = v4.is_private(); + if !is_private && !is_tailscale { + // Public / globally routable IP + base + 50 + } else if has_gateway && is_private { + // Primary LAN IP with gateway (e.g. 192.168.1.117) + base + 40 + } else if is_tailscale { + // Tailscale / CGNAT + base + 30 + } else { + // Other private / virtual + base + 10 + } + } + IpAddr::V6(v6) => { + let segments = v6.segments(); + let is_global = (segments[0] & 0xe000) == 0x2000; + if is_global { base + 45 } else { base + 20 } + } + } +} diff --git a/crates/tsunagi-wireguard/src/announcement.rs b/crates/tsunagi-wireguard/src/announcement.rs deleted file mode 100644 index ba30acb..0000000 --- a/crates/tsunagi-wireguard/src/announcement.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! What a pure WireGuard peer tells the network about itself. -//! -//! Exchanged via [`tsunagi::dataplane::PluginCapability`]. The announcement -//! carries the WireGuard public key, the listening UDP port, and candidate IP addresses -//! for direct UDP packet transport. - -use serde::{Deserialize, Serialize}; - -use tsunagi::dataplane::PluginError; -use tsunagi::identity::NetworkId; - -use crate::keys::WgPublicKey; - -/// The wire protocol identifier for pure WireGuard. -pub const WG_PROTOCOL: &str = "wg"; - -/// Wire version of the `wg` announcement format. -pub const ANNOUNCEMENT_VERSION: u16 = 1; - -/// Wire version peers compare during capability exchange. -/// -/// This is the same as [`ANNOUNCEMENT_VERSION`]; keeping a separate name -/// makes the intent explicit in the CLI protocol registry. -pub const WG_WIRE_VERSION: u16 = ANNOUNCEMENT_VERSION; - -/// Maximum number of IP candidates allowed in an announcement. -pub const MAX_CANDIDATE_ADDRS: usize = 32; - -/// What one participant advertises for the pure WireGuard UDP data plane. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct WgAnnouncement { - /// Announcement format version. - pub version: u16, - /// The peer's WireGuard public key. - pub public_key: [u8; 32], - /// The network this key is for. - pub network: [u8; 32], - /// The UDP port the peer is listening on for WireGuard packets. - pub port: u16, - /// Candidate IP addresses where this peer can be reached on `port`. - #[serde(default)] - pub addrs: Vec, -} - -/// A peer announcement that has been validated against a specific network. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ValidatedAnnouncement { - /// The peer's WireGuard public key. - pub public_key: WgPublicKey, - /// The UDP port the peer listens on. - pub port: u16, - /// Validated candidate IP addresses. - pub addrs: Vec, -} - -impl WgAnnouncement { - /// Builds this agent's announcement. - pub fn new( - network: NetworkId, - public_key: &WgPublicKey, - port: u16, - addrs: Vec, - ) -> Self { - Self { - version: ANNOUNCEMENT_VERSION, - public_key: *public_key.as_bytes(), - network: *network.as_bytes(), - port, - addrs, - } - } - - /// Encodes the announcement into the opaque capability payload. - pub fn encode(&self) -> Result, PluginError> { - postcard::to_allocvec(self) - .map_err(|err| PluginError::Other(format!("cannot encode announcement: {err}"))) - } - - /// Decodes and validates a payload received from a peer. - pub fn decode_and_validate( - payload: &[u8], - network: NetworkId, - local_key: &WgPublicKey, - ) -> Result { - let announcement: Self = postcard::from_bytes(payload) - .map_err(|_| PluginError::Rejected("malformed WireGuard announcement".into()))?; - announcement.validate(network, local_key) - } - - fn validate( - self, - network: NetworkId, - local_key: &WgPublicKey, - ) -> Result { - if self.version != ANNOUNCEMENT_VERSION { - return Err(PluginError::Rejected(format!( - "peer speaks WireGuard announcement version {} but this build speaks \ - {ANNOUNCEMENT_VERSION}; one of the two needs updating", - self.version - ))); - } - - let public_key = WgPublicKey::from_bytes(self.public_key); - if public_key.is_zero() { - return Err(PluginError::Rejected( - "WireGuard public key is all zeroes".into(), - )); - } - if &public_key == local_key { - return Err(PluginError::Rejected( - "peer announced this agent's own WireGuard key".into(), - )); - } - if self.network != *network.as_bytes() { - return Err(PluginError::Rejected( - "announcement is for a different network".into(), - )); - } - if self.port == 0 { - return Err(PluginError::Rejected("WireGuard port cannot be 0".into())); - } - if self.addrs.len() > MAX_CANDIDATE_ADDRS { - return Err(PluginError::Rejected(format!( - "too many candidate addresses: {} > {MAX_CANDIDATE_ADDRS}", - self.addrs.len() - ))); - } - - let mut valid_addrs = Vec::new(); - for addr in self.addrs { - if !addr.is_unspecified() && !valid_addrs.contains(&addr) { - valid_addrs.push(addr); - } - } - - Ok(ValidatedAnnouncement { - public_key, - port: self.port, - addrs: valid_addrs, - }) - } -} diff --git a/crates/tsunagi-wireguard/src/device.rs b/crates/tsunagi-wireguard/src/device.rs deleted file mode 100644 index 0580325..0000000 --- a/crates/tsunagi-wireguard/src/device.rs +++ /dev/null @@ -1,686 +0,0 @@ -//! Userspace WireGuard device implementation over direct UDP links. -//! -//! Uses [`boringtun::noise::Tunn`] state machine to encrypt and decrypt -//! standard WireGuard protocol packets directly over a [`SharedLink`]. - -use std::collections::{HashMap, VecDeque}; -use std::net::Ipv4Addr; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; -use std::time::Duration; - -use boringtun::noise::{Tunn, TunnResult}; -use bytes::Bytes; -use iroh::EndpointId; -use tokio::task::JoinHandle; - -use tsunagi::dataplane::routing::{FlowId, flow::ip_flow}; -use tsunagi::dataplane::transport::{SharedLink, TransportError}; -use tsunagi::dataplane::{PacketSink, PluginContext, PluginError}; -use tsunagi::identity::NetworkId; - -use crate::keys::{WgPublicKey, WgSecretKey}; -use crate::transport::WireguardTransport; - -const TIMER_INTERVAL: Duration = Duration::from_millis(250); -const SCRATCH: usize = 4096; - -/// Maximum consecutive rekey timeout retransmissions before declaring a tunnel dead. -pub const MAX_REKEY_TIMEOUTS: u32 = 3; - -/// Overall maximum duration trying to establish or rekey a handshake before declaring it dead. -pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(15); - -struct FlowTunnel { - tunn: Tunn, - queued: VecDeque, - handshake_started_at: Option, - rekey_timeouts: u32, - failed: bool, -} - -fn is_data(result: &TunnResult<'_>) -> bool { - matches!(result, TunnResult::WriteToNetwork(bytes) if bytes.starts_with(&[4, 0, 0, 0])) -} - -impl FlowTunnel { - fn encapsulate<'a>( - &mut self, - packet: &[u8], - scratch: &'a mut [u8], - flow: FlowId, - ) -> (TunnResult<'a>, FlowId) { - let result = self.tunn.encapsulate(packet, scratch); - if is_data(&result) { - return (result, flow); - } - if matches!(result, TunnResult::WriteToNetwork(ref b) if !b.is_empty() && b[0] == 1) - && self.handshake_started_at.is_none() - { - self.handshake_started_at = Some(std::time::Instant::now()); - self.rekey_timeouts = 0; - self.failed = false; - } - if self.queued.len() < 256 { - self.queued.push_back(flow); - } - (result, 0) - } - - fn decapsulate<'a>( - &mut self, - packet: &[u8], - scratch: &'a mut [u8], - ) -> (TunnResult<'a>, FlowId) { - let result = self.tunn.decapsulate(None, packet, scratch); - if self.tunn.time_since_last_handshake().is_some() { - self.handshake_started_at = None; - self.rekey_timeouts = 0; - self.failed = false; - } - let mut flow = 0; - if packet.is_empty() { - if is_data(&result) { - flow = self.queued.pop_front().unwrap_or(0); - } else if let Some(pending) = self.queued.pop_front() { - self.queued.push_back(pending); - } - } - (result, flow) - } - - fn update_timers<'a>(&mut self, scratch: &'a mut [u8]) -> (TunnResult<'a>, bool) { - if self.failed { - return (TunnResult::Done, false); - } - - let result = self.tunn.update_timers(scratch); - if matches!( - result, - TunnResult::Err(boringtun::noise::errors::WireGuardError::ConnectionExpired) - ) || self.tunn.is_expired() - { - self.failed = true; - self.queued.clear(); - self.handshake_started_at = None; - self.rekey_timeouts = 0; - return (result, true); - } - - if let TunnResult::WriteToNetwork(ref bytes) = result - && !bytes.is_empty() - && bytes[0] == 1 - && self.handshake_started_at.is_none() - { - self.handshake_started_at = Some(std::time::Instant::now()); - } - - if let Some(started) = self.handshake_started_at - && started.elapsed() >= HANDSHAKE_TIMEOUT - { - self.failed = true; - self.queued.clear(); - self.handshake_started_at = None; - self.rekey_timeouts = 0; - return (result, true); - } - - (result, false) - } -} - -#[derive(Debug, Default)] -struct PeerCounters { - tx_packets: AtomicU64, - tx_bytes: AtomicU64, - rx_packets: AtomicU64, - rx_bytes: AtomicU64, - dropped_wrong_source: AtomicU64, - dropped_oversize: AtomicU64, - dropped_no_session: AtomicU64, - protocol_errors: AtomicU64, -} - -/// A snapshot of one peer's tunnel counters. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct PeerStats { - /// Plaintext packets encrypted and sent to this peer. - pub tx_packets: u64, - /// Plaintext bytes encrypted and sent to this peer. - pub tx_bytes: u64, - /// Plaintext packets decrypted from this peer and given to the OS. - pub rx_packets: u64, - /// Plaintext bytes decrypted from this peer and given to the OS. - pub rx_bytes: u64, - /// Packets dropped because their source was not this peer's address. - pub dropped_wrong_source: u64, - /// Packets dropped because they did not fit in one link datagram. - pub dropped_oversize: u64, - /// Packets dropped because there was no session to encrypt with yet. - pub dropped_no_session: u64, - /// WireGuard protocol errors, including packets that failed to decrypt. - pub protocol_errors: u64, -} - -/// Whether a peer's tunnel has completed a handshake. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PeerHealth { - /// Time since the last successful WireGuard handshake. - pub since_handshake: Option, -} - -impl PeerHealth { - /// Whether the tunnel has ever completed a handshake. - pub fn is_up(&self) -> bool { - self.since_handshake.is_some() - } -} - -struct Peer { - endpoint_id: EndpointId, - public_key: WgPublicKey, - overlay_v4: Mutex>, - tunn: Mutex, - link: SharedLink, - counters: Arc, - task: Mutex>>, -} - -impl std::fmt::Debug for Peer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Peer") - .field("peer", &self.endpoint_id.fmt_short().to_string()) - .field("public_key", &self.public_key) - .finish() - } -} - -impl Drop for Peer { - fn drop(&mut self) { - if let Ok(mut guard) = self.task.lock() - && let Some(task) = guard.take() - { - task.abort(); - } - } -} - -impl Peer { - fn stats(&self) -> PeerStats { - PeerStats { - tx_packets: self.counters.tx_packets.load(Ordering::Relaxed), - tx_bytes: self.counters.tx_bytes.load(Ordering::Relaxed), - rx_packets: self.counters.rx_packets.load(Ordering::Relaxed), - rx_bytes: self.counters.rx_bytes.load(Ordering::Relaxed), - dropped_wrong_source: self.counters.dropped_wrong_source.load(Ordering::Relaxed), - dropped_oversize: self.counters.dropped_oversize.load(Ordering::Relaxed), - dropped_no_session: self.counters.dropped_no_session.load(Ordering::Relaxed), - protocol_errors: self.counters.protocol_errors.load(Ordering::Relaxed), - } - } - - fn health(&self) -> PeerHealth { - let guard = match self.tunn.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - PeerHealth { - since_handshake: guard.tunn.time_since_last_handshake(), - } - } -} - -/// What a peer's tunnel looks like from outside. -#[derive(Debug, Clone)] -pub struct PeerSummary { - /// The peer's control plane identity. - pub endpoint_id: EndpointId, - /// The peer's WireGuard public key. - pub public_key: WgPublicKey, - /// The overlay address it holds, once agreed. - pub overlay_address_v4: Option, - /// Whether the tunnel has handshaken. - pub health: PeerHealth, - /// Traffic counters. - pub stats: PeerStats, - /// What the transport reports about the path carrying this tunnel. - pub path: String, - /// Largest datagram the link accepts. - pub max_datagram: usize, -} - -struct Inner { - network: NetworkId, - private_key: WgSecretKey, - peers: RwLock>>, - sink: Arc, - transport: Arc, - context: PluginContext, - next_index: AtomicU32, -} - -impl Inner { - fn on_tunnel_failed(&self, peer: EndpointId, public_key: WgPublicKey) { - self.transport.close_link(self.network, peer); - write_lock(&self.peers).remove(&public_key); - self.context.report_peer_protocol_failure( - self.network, - peer, - crate::announcement::WG_PROTOCOL, - "WireGuard rekey timeout", - ); - } -} - -impl std::fmt::Debug for Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Inner") - .field("network", &self.network.fmt_short()) - .finish() - } -} - -/// The userspace WireGuard tunnels of one network. -#[derive(Debug)] -pub struct WireguardDevice { - inner: Arc, - tasks: Vec>, -} - -impl WireguardDevice { - /// Starts the WireGuard tunnels for one network. - pub fn start( - network: NetworkId, - private_key: WgSecretKey, - sink: Arc, - transport: Arc, - context: PluginContext, - ) -> Self { - let inner = Arc::new(Inner { - network, - private_key, - peers: RwLock::new(HashMap::new()), - sink, - transport, - context, - next_index: AtomicU32::new(1), - }); - - let timers = tokio::spawn(drive_timers(Arc::clone(&inner))); - - Self { - inner, - tasks: vec![timers], - } - } - - /// Encrypts an IP packet and sends it to a peer over its direct link. - pub fn carry(&self, peer: EndpointId, packet: &[u8]) -> bool { - let found = read_lock(&self.inner.peers) - .values() - .find(|candidate| candidate.endpoint_id == peer) - .cloned(); - let Some(peer) = found else { - return false; - }; - - let mut scratch = [0u8; SCRATCH]; - let flow = ip_flow(packet); - let result = { - let mut tunn = match peer.tunn.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - let (result, selected_flow) = tunn.encapsulate(packet, &mut scratch, flow); - match result { - TunnResult::WriteToNetwork(out) => Some((out.len(), selected_flow)), - _ => None, - } - }; - let Some((len, selected_flow)) = result else { - peer.counters - .dropped_no_session - .fetch_add(1, Ordering::Relaxed); - return false; - }; - - send_to_peer_flow(&peer, &scratch[..len], selected_flow); - peer.counters.tx_packets.fetch_add(1, Ordering::Relaxed); - peer.counters - .tx_bytes - .fetch_add(packet.len() as u64, Ordering::Relaxed); - true - } - - /// Adds or replaces a peer and starts its tunnel. - pub fn add_peer( - &self, - endpoint_id: EndpointId, - public_key: WgPublicKey, - overlay_v4: Option, - link: SharedLink, - keepalive: Option, - ) -> Result<(), PluginError> { - if public_key == self.inner.private_key.public() { - return Err(PluginError::Rejected( - "refusing to add ourselves as a WireGuard peer".into(), - )); - } - - let index = self.inner.next_index.fetch_add(1, Ordering::Relaxed); - let tunn = Tunn::new( - self.inner.private_key.to_static_secret(), - public_key.into_x25519(), - None, - keepalive, - index, - None, - ); - - let peer = Arc::new(Peer { - endpoint_id, - public_key, - overlay_v4: Mutex::new(overlay_v4), - tunn: Mutex::new(FlowTunnel { - tunn, - queued: VecDeque::new(), - handshake_started_at: None, - rekey_timeouts: 0, - failed: false, - }), - link, - counters: Arc::new(PeerCounters::default()), - task: Mutex::new(None), - }); - - let task = tokio::spawn(read_from_link(Arc::clone(&self.inner), Arc::clone(&peer))); - if let Ok(mut guard) = peer.task.lock() { - *guard = Some(task); - } - - write_lock(&self.inner.peers).insert(public_key, Arc::clone(&peer)); - kick_handshake(&peer); - Ok(()) - } - - /// Removes a peer and stops its tunnel. - pub fn remove_peer(&self, public_key: &WgPublicKey) { - write_lock(&self.inner.peers).remove(public_key); - } - - /// Removes every peer whose key is not in `keep`. - pub fn retain_peers(&self, keep: &[WgPublicKey]) { - let stale: Vec = read_lock(&self.inner.peers) - .keys() - .filter(|key| !keep.contains(key)) - .copied() - .collect(); - for key in stale { - self.remove_peer(&key); - } - } - - /// Whether a peer's tunnel exists. - pub fn has_peer(&self, public_key: &WgPublicKey) -> bool { - read_lock(&self.inner.peers).contains_key(public_key) - } - - /// A snapshot of every peer. - pub fn peers(&self) -> Vec { - let mut peers: Vec = read_lock(&self.inner.peers) - .values() - .map(|peer| { - let overlay_v4 = match peer.overlay_v4.lock() { - Ok(guard) => *guard, - Err(poisoned) => *poisoned.into_inner(), - }; - PeerSummary { - endpoint_id: peer.endpoint_id, - public_key: peer.public_key, - overlay_address_v4: overlay_v4, - health: peer.health(), - stats: peer.stats(), - path: peer.link.path_description(), - max_datagram: peer.link.max_datagram_size(), - } - }) - .collect(); - peers.sort_by_key(|peer| peer.public_key); - peers - } -} - -impl Drop for WireguardDevice { - fn drop(&mut self) { - for task in &self.tasks { - task.abort(); - } - } -} - -fn read_lock(lock: &RwLock) -> std::sync::RwLockReadGuard<'_, T> { - match lock.read() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - } -} - -fn write_lock(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, T> { - match lock.write() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - } -} - -fn kick_handshake(peer: &Peer) { - let mut scratch = vec![0u8; SCRATCH]; - let len = { - let mut tunn = match peer.tunn.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - match tunn.tunn.format_handshake_initiation(&mut scratch, false) { - TunnResult::WriteToNetwork(out) => { - tunn.handshake_started_at = Some(std::time::Instant::now()); - tunn.rekey_timeouts = 0; - tunn.failed = false; - Some(out.len()) - } - _ => None, - } - }; - if let Some(len) = len { - send_to_peer(peer, &scratch[..len]); - } -} - -fn send_to_peer(peer: &Peer, payload: &[u8]) { - send_to_peer_flow(peer, payload, 0); -} - -fn send_to_peer_flow(peer: &Peer, payload: &[u8], flow: FlowId) { - match peer.link.send_flow(Bytes::copy_from_slice(payload), flow) { - Ok(()) => {} - Err(TransportError::TooLarge { .. }) => { - peer.counters - .dropped_oversize - .fetch_add(1, Ordering::Relaxed); - } - Err(TransportError::Closed) => {} - Err(err) => { - tracing::trace!(%err, "dropping a WireGuard packet the link refused"); - } - } -} - -async fn read_from_link(inner: Arc, peer: Arc) { - let mut scratch = vec![0u8; SCRATCH]; - loop { - let Some(datagram) = peer.link.recv().await else { - return; - }; - - let mut input: Option<&[u8]> = Some(&datagram); - loop { - let outcome = { - let mut tunn = match peer.tunn.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - let (result, flow) = tunn.decapsulate(input.unwrap_or(&[]), &mut scratch); - match result { - TunnResult::WriteToNetwork(out) => Outcome::ToNetwork(out.len(), flow), - TunnResult::WriteToTunnelV6(out, _) => Outcome::ToTunnel(out.len()), - TunnResult::WriteToTunnelV4(out, _) => Outcome::ToTunnel(out.len()), - TunnResult::Done => Outcome::Done, - TunnResult::Err(err) => { - tracing::trace!(?err, "WireGuard decapsulation failed"); - Outcome::Failed - } - } - }; - - match outcome { - Outcome::ToNetwork(len, flow) => { - send_to_peer_flow(&peer, &scratch[..len], flow); - input = None; - continue; - } - Outcome::ToTunnel(len) => { - let payload = Bytes::copy_from_slice(&scratch[..len]); - inner - .sink - .deliver(inner.network, peer.endpoint_id, payload) - .await; - peer.counters.rx_packets.fetch_add(1, Ordering::Relaxed); - peer.counters - .rx_bytes - .fetch_add(len as u64, Ordering::Relaxed); - break; - } - Outcome::Failed => { - peer.counters - .protocol_errors - .fetch_add(1, Ordering::Relaxed); - break; - } - Outcome::Done => break, - } - } - } -} - -enum Outcome { - ToNetwork(usize, FlowId), - ToTunnel(usize), - Done, - Failed, -} - -async fn drive_timers(inner: Arc) { - let mut ticker = tokio::time::interval(TIMER_INTERVAL); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - ticker.tick().await; - let peers: Vec> = read_lock(&inner.peers).values().cloned().collect(); - for peer in peers { - let mut scratch = vec![0u8; SCRATCH]; - let (len, failed) = { - let mut tunn = match peer.tunn.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - let (result, failed) = tunn.update_timers(&mut scratch); - let len = match result { - TunnResult::WriteToNetwork(out) => Some(out.len()), - TunnResult::Err(err) => { - tracing::trace!(?err, "WireGuard timer error"); - None - } - _ => None, - }; - (len, failed) - }; - if let Some(len) = len { - send_to_peer(&peer, &scratch[..len]); - } - if failed { - tracing::warn!( - peer = %peer.endpoint_id.fmt_short(), - "WireGuard tunnel failed due to rekey timeout, triggering fallback" - ); - inner.on_tunnel_failed(peer.endpoint_id, peer.public_key); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_flow_tunnel_rekey_timeout_triggers_failure() { - let my_secret = WgSecretKey::generate(); - let their_secret = WgSecretKey::generate(); - let their_public = their_secret.public(); - let tunn = Tunn::new( - my_secret.to_static_secret(), - their_public.into_x25519(), - None, - None, - 1, - None, - ); - - let mut tunnel = FlowTunnel { - tunn, - queued: VecDeque::new(), - handshake_started_at: Some(std::time::Instant::now() - Duration::from_secs(16)), - rekey_timeouts: 0, - failed: false, - }; - - let mut scratch = vec![0u8; SCRATCH]; - // When handshake timeout exceeds HANDSHAKE_TIMEOUT (15s), update_timers returns true (failed) - let (_, failed) = tunnel.update_timers(&mut scratch); - assert!(failed, "handshake timeout should report failure"); - assert!(tunnel.handshake_started_at.is_none()); - assert!(tunnel.failed, "tunnel should be marked as failed"); - - // Subsequent timer tick must not report failure again - let (_, failed_again) = tunnel.update_timers(&mut scratch); - assert!( - !failed_again, - "subsequent timer tick should not re-trigger failure" - ); - } - - #[test] - fn test_flow_tunnel_reset_on_handshake() { - let my_secret = WgSecretKey::generate(); - let their_secret = WgSecretKey::generate(); - let their_public = their_secret.public(); - let tunn = Tunn::new( - my_secret.to_static_secret(), - their_public.into_x25519(), - None, - None, - 1, - None, - ); - - let mut tunnel = FlowTunnel { - tunn, - queued: VecDeque::new(), - handshake_started_at: Some(std::time::Instant::now()), - rekey_timeouts: 2, - failed: true, - }; - - let mut scratch = vec![0u8; SCRATCH]; - let _ = tunnel.decapsulate(&[], &mut scratch); - // decapsulate without handshake response doesn't establish session - assert_eq!(tunnel.rekey_timeouts, 2); - } -} diff --git a/crates/tsunagi-wireguard/src/keys.rs b/crates/tsunagi-wireguard/src/keys.rs deleted file mode 100644 index 71a8d82..0000000 --- a/crates/tsunagi-wireguard/src/keys.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! WireGuard key material for pure WireGuard transport. -//! -//! Keys belong to this protocol plugin alone. They are not derived from -//! the iroh device key and not derived from the network secret, so compromising -//! or rotating one does not affect the others. -//! -//! Keys are X25519, encoded the way WireGuard encodes them: standard base64 -//! with padding, 44 characters. - -use boringtun::x25519; -use data_encoding::BASE64; -use zeroize::{Zeroize, Zeroizing}; - -use tsunagi::dataplane::PluginError; - -/// Length of a raw WireGuard key in bytes. -pub const KEY_LEN: usize = 32; - -/// Length of the base64 text form of a key. -pub const KEY_TEXT_LEN: usize = 44; - -/// Applies the X25519 clamping WireGuard applies to private keys. -fn clamp(bytes: &mut [u8; KEY_LEN]) { - bytes[0] &= 248; - bytes[31] &= 127; - bytes[31] |= 64; -} - -/// A WireGuard public key. -/// -/// Public, safe to log, and the identity a peer is known by inside the overlay. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct WgPublicKey([u8; KEY_LEN]); - -impl WgPublicKey { - /// Wraps raw key bytes. - pub fn from_bytes(bytes: [u8; KEY_LEN]) -> Self { - Self(bytes) - } - - /// The raw key bytes. - pub fn as_bytes(&self) -> &[u8; KEY_LEN] { - &self.0 - } - - /// The key in the form the WireGuard implementation expects. - pub(crate) fn into_x25519(self) -> x25519::PublicKey { - x25519::PublicKey::from(self.0) - } - - /// Whether this is the all-zero key, which is never a valid peer. - pub fn is_zero(&self) -> bool { - self.0 == [0u8; KEY_LEN] - } - - /// The base64 form WireGuard uses. - pub fn encode(&self) -> String { - BASE64.encode(&self.0) - } - - /// Parses the base64 form WireGuard uses. - pub fn decode(text: &str) -> Result { - if text.len() != KEY_TEXT_LEN { - return Err(PluginError::Rejected(format!( - "a WireGuard key is {KEY_TEXT_LEN} base64 characters, got {}", - text.len() - ))); - } - let raw = BASE64 - .decode(text.as_bytes()) - .map_err(|_| PluginError::Rejected("key is not valid base64".into()))?; - let bytes = <[u8; KEY_LEN]>::try_from(raw.as_slice()) - .map_err(|_| PluginError::Rejected("key is not 32 bytes".into()))?; - Ok(Self(bytes)) - } - - /// A short prefix for logs and diagnostics. - pub fn fmt_short(&self) -> String { - self.encode().chars().take(8).collect() - } -} - -impl std::fmt::Display for WgPublicKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.encode()) - } -} - -impl std::fmt::Debug for WgPublicKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WgPublicKey({})", self.fmt_short()) - } -} - -/// A WireGuard private key. -/// -/// Clamped on creation so it matches what `wg genkey` produces. Zeroized on -/// drop, and prints as `[redacted]` in Debug so it cannot leak into logs. -#[derive(Clone)] -pub struct WgSecretKey { - bytes: Zeroizing<[u8; KEY_LEN]>, - public: WgPublicKey, -} - -impl WgSecretKey { - /// Generates a fresh clamped random private key. - pub fn generate() -> Self { - let mut bytes = [0u8; KEY_LEN]; - rand::fill(&mut bytes); - clamp(&mut bytes); - let secret = x25519::StaticSecret::from(bytes); - let public = WgPublicKey(*x25519::PublicKey::from(&secret).as_bytes()); - Self { - bytes: Zeroizing::new(bytes), - public, - } - } - - /// Wraps existing key bytes, clamping them to a valid X25519 private key. - pub fn from_bytes(raw: &[u8; KEY_LEN]) -> Self { - let mut bytes = *raw; - clamp(&mut bytes); - let secret = x25519::StaticSecret::from(bytes); - let public = WgPublicKey(*x25519::PublicKey::from(&secret).as_bytes()); - Self { - bytes: Zeroizing::new(bytes), - public, - } - } - - /// The matching public key. - pub fn public(&self) -> WgPublicKey { - self.public - } - - /// The key in the form the WireGuard implementation expects. - pub(crate) fn to_static_secret(&self) -> x25519::StaticSecret { - x25519::StaticSecret::from(*self.bytes) - } - - /// The raw key bytes. - pub fn as_bytes(&self) -> &[u8; KEY_LEN] { - &self.bytes - } -} - -impl Drop for WgSecretKey { - fn drop(&mut self) { - self.bytes.zeroize(); - } -} - -impl std::fmt::Debug for WgSecretKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WgSecretKey") - .field("bytes", &"[redacted]") - .field("public", &self.public) - .finish() - } -} diff --git a/crates/tsunagi-wireguard/src/lib.rs b/crates/tsunagi-wireguard/src/lib.rs index 1b24f9e..44a89f1 100644 --- a/crates/tsunagi-wireguard/src/lib.rs +++ b/crates/tsunagi-wireguard/src/lib.rs @@ -1,28 +1,21 @@ -//! The pure WireGuard protocol plugin for Tsunagi: WireGuard cryptography over direct UDP. -//! -//! Deliberately a separate crate from the core, ensuring clear boundaries: -//! - Direct UDP datagram transport without QUIC encapsulation or double-encryption. -//! - Standard WireGuard cryptography via `boringtun`. -//! - Peer discovery and key exchange managed through the Tsunagi control plane. - +//! Authenticated, per-neighbour WireGuard UDP carriers. IP encryption is independent. #![deny(unsafe_code)] #![warn(missing_docs)] - -pub mod announcement; -pub mod device; -pub mod keys; -pub mod plugin; -pub mod store; -pub mod transport; - -pub use announcement::{ - ANNOUNCEMENT_VERSION, ValidatedAnnouncement, WG_PROTOCOL, WG_WIRE_VERSION, WgAnnouncement, -}; -pub use device::{PeerHealth, PeerStats, PeerSummary, WireguardDevice}; -pub use keys::{WgPublicKey, WgSecretKey}; -pub use plugin::{ - DEFAULT_MTU, MIN_MTU, NetworkOverview, PeerOverview, WIREGUARD_OVERHEAD, WireguardConfig, - WireguardPlugin, local_ip_candidates, -}; -pub use store::WgKeyStore; -pub use transport::{DEFAULT_PORT, MAX_UDP_DATAGRAM_SIZE, WireguardLink, WireguardTransport}; +mod addresses; +mod masking; +mod plugin; +pub mod profile; +mod session; +mod stun; +mod transport; +pub use addresses::{local_ip_candidates, prioritize_local_subnet}; +pub use plugin::AwgPlugin as UdpCarrier; +pub use transport::{AwgTransport as UdpTransport, PairOverview}; +/// External ordinary WireGuard carrier identifier. +pub const WG_PROTOCOL: &str = "wg"; +/// Wire version of the private UDP negotiation and carrier frames. +pub const WG_WIRE_VERSION: u16 = 2; +/// Masked carrier identifier used by the shared pair engine. +pub const AWG_PROTOCOL: &str = "awg"; +/// Wire version of masked carrier frames and private negotiation. +pub const AWG_WIRE_VERSION: u16 = 2; diff --git a/crates/tsunagi-awg/src/masking.rs b/crates/tsunagi-wireguard/src/masking.rs similarity index 100% rename from crates/tsunagi-awg/src/masking.rs rename to crates/tsunagi-wireguard/src/masking.rs diff --git a/crates/tsunagi-wireguard/src/plugin.rs b/crates/tsunagi-wireguard/src/plugin.rs index 47097e0..e2322fc 100644 --- a/crates/tsunagi-wireguard/src/plugin.rs +++ b/crates/tsunagi-wireguard/src/plugin.rs @@ -1,1055 +1,76 @@ -//! The pure WireGuard IP plugin implementation. -//! -//! Provides [`WireguardPlugin`] implementing [`IpPlugin`]. - -use std::collections::{BTreeSet, HashMap}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::path::PathBuf; -use std::sync::{Arc, Mutex, OnceLock}; -use std::time::Duration; - +//! Carrier capabilities only. End-to-end encryption belongs to the agent's tunnel. +use crate::transport::AwgTransport; use iroh::EndpointId; -use tokio::sync::{mpsc, oneshot}; -use tokio::task::JoinHandle; - -use tsunagi::BoxFuture; -use tsunagi::dataplane::PacketSink; -use tsunagi::dataplane::transport::SharedLink; -use tsunagi::dataplane::{IpPlugin, PluginCapability, PluginContext, PluginError, ProtocolOption}; -use tsunagi::identity::NetworkId; -use tsunagi::state::Ipv4Range; - -use crate::announcement::{ - ANNOUNCEMENT_VERSION, ValidatedAnnouncement, WG_PROTOCOL, WgAnnouncement, -}; -use crate::device::{PeerSummary, WireguardDevice}; -use crate::keys::{WgPublicKey, WgSecretKey}; -use crate::store::WgKeyStore; -use crate::transport::WireguardTransport; - -/// Smallest interface MTU the overlay accepts. -pub const MIN_MTU: u32 = 576; - -/// Default interface MTU. -pub const DEFAULT_MTU: u32 = 1280; - -/// WireGuard packet overhead. -pub const WIREGUARD_OVERHEAD: u32 = 32; - -/// Discovers local network interface IPs to announce to peers, excluding loopback, overlay interfaces, and virtual bridges. -pub fn local_ip_candidates() -> Vec { - let mut candidates = Vec::new(); - for iface in netdev::get_interfaces() { - if !is_candidate_interface(&iface) { - continue; - } - for ip in &iface.ipv4 { - let addr = ip.addr(); - if is_candidate_ipv4(addr) { - let priority = ip_priority(&iface, IpAddr::V4(addr)); - candidates.push((priority, IpAddr::V4(addr))); - } - } - } - // Sort descending by priority so the most viable physical / WAN / LAN IP is candidate #0 - candidates.sort_by_key(|b| std::cmp::Reverse(b.0)); - let mut seen = std::collections::HashSet::new(); - let mut ips = Vec::new(); - for (_, ip) in candidates { - if seen.insert(ip) { - ips.push(ip); - } - } - ips -} - -pub(crate) fn is_candidate_interface(iface: &netdev::Interface) -> bool { - if !iface.is_up() || iface.is_loopback() { - return false; - } - - let name = iface.name.to_lowercase(); - let friendly = iface - .friendly_name - .as_deref() - .map(str::to_lowercase) - .unwrap_or_default(); - let desc = iface - .description - .as_deref() - .map(str::to_lowercase) - .unwrap_or_default(); - - // Skip Tsunagi / Wintun overlay interfaces - if name.starts_with("tsun") - || name.contains("tsunagi") - || name.contains("wintun") - || friendly.starts_with("tsun") - || friendly.contains("tsunagi") - || friendly.contains("wintun") - || desc.contains("wintun") - || desc.contains("tsunagi") - { - return false; - } - - // Skip container interfaces and virtual bridges purely internal to container engines - if name.starts_with("docker") - || name.starts_with("br-") - || name.starts_with("veth") - || name.starts_with("cni") - || name.starts_with("flannel") - || name.starts_with("cali") - || name.starts_with("kube") - || name.starts_with("dummy") - || name.starts_with("lxc") - || name.starts_with("podman") - || friendly.contains("wsl") - { - return false; - } - - true -} - -/// Reorders candidate addresses so that candidates on the same local subnet as one of our -/// active network interfaces appear first. -pub fn prioritize_local_subnet(candidates: &mut [SocketAddr]) { - let local_subnets: Vec<(Ipv4Addr, u8)> = netdev::get_interfaces() - .into_iter() - .filter(|iface| iface.is_up() && !iface.is_loopback()) - .flat_map(|iface| { - iface - .ipv4 - .into_iter() - .map(|net| (net.addr(), net.prefix_len())) - }) - .collect(); - - candidates.sort_by_key(|addr| { - let is_local = match addr.ip() { - IpAddr::V4(v4) => local_subnets - .iter() - .any(|(local_ip, prefix_len)| matches_subnet(*local_ip, *prefix_len, v4)), - IpAddr::V6(_) => false, - }; - if is_local { 0 } else { 1 } - }); -} - -fn matches_subnet(local_ip: Ipv4Addr, prefix_len: u8, target_ip: Ipv4Addr) -> bool { - if prefix_len == 0 { - return false; - } - let mask = if prefix_len >= 32 { - !0u32 - } else { - !0u32 << (32 - prefix_len) - }; - (u32::from(local_ip) & mask) == (u32::from(target_ip) & mask) -} - -pub(crate) fn is_candidate_ipv4(addr: std::net::Ipv4Addr) -> bool { - if addr.is_loopback() || addr.is_unspecified() || addr.is_link_local() || addr.is_broadcast() { - return false; - } - if addr.is_documentation() { - return false; - } - true -} - -pub(crate) fn is_candidate_ip(addr: IpAddr) -> bool { - match addr { - IpAddr::V4(v4) => is_candidate_ipv4(v4), - IpAddr::V6(v6) => { - if v6.is_loopback() || v6.is_unspecified() { - return false; - } - let segments = v6.segments(); - // Filter link-local fe80::/10 - if (segments[0] & 0xffc0) == 0xfe80 { - return false; - } - true - } - } -} - -pub(crate) fn ip_priority(iface: &netdev::Interface, ip: IpAddr) -> u32 { - let has_gateway = iface.gateway.is_some(); - let base = if has_gateway { 100 } else { 0 }; - match ip { - IpAddr::V4(v4) => { - let octets = v4.octets(); - // Tailscale: 100.64.0.0/10 - let is_tailscale = octets[0] == 100 && (octets[1] & 0xc0) == 64; - let is_private = v4.is_private(); - if !is_private && !is_tailscale { - // Public / globally routable IP - base + 50 - } else if has_gateway && is_private { - // Primary LAN IP with gateway (e.g. 192.168.1.117) - base + 40 - } else if is_tailscale { - // Tailscale / CGNAT - base + 30 - } else { - // Other private / virtual - base + 10 - } - } - IpAddr::V6(v6) => { - let segments = v6.segments(); - let is_global = (segments[0] & 0xe000) == 0x2000; - if is_global { base + 45 } else { base + 20 } - } - } -} - -/// Configuration of the pure WireGuard plugin. -#[derive(Debug, Clone)] -pub struct WireguardConfig { - /// Directory for the plugin's own key store. - pub state_dir: PathBuf, - /// WireGuard keepalive in seconds. - pub keepalive: Option, - /// The largest packet a tunnel will carry. - pub mtu: u32, - /// UDP port to listen on. - pub port: Option, - /// How long to coalesce changes before reconciling. - pub reconcile_debounce: Duration, - /// Periodic reconcile interval. - pub reconcile_interval: Duration, -} - -impl WireguardConfig { - /// Creates a configuration rooted at `state_dir`. - pub fn new(state_dir: impl Into) -> Self { - Self { - state_dir: state_dir.into(), - keepalive: Some(25), - mtu: DEFAULT_MTU, - port: None, - reconcile_debounce: Duration::from_millis(200), - reconcile_interval: Duration::from_secs(15), - } - } - - /// Sets the interface MTU. - pub fn with_mtu(mut self, mtu: u32) -> Self { - self.mtu = mtu; - self - } - - /// Sets the UDP listen port. - pub fn with_port(mut self, port: u16) -> Self { - self.port = Some(port); - self - } - - /// Path of the plugin's key store. - pub fn key_store_path(&self) -> PathBuf { - self.state_dir.join("wireguard.sqlite") - } -} - -/// What this agent has set up for one network. -#[derive(Debug, Clone)] -pub struct NetworkOverview { - /// The network. - pub network: NetworkId, - /// The largest packet a tunnel in this network will carry. - pub mtu: u32, - /// This agent's WireGuard public key in this network. - pub public_key: WgPublicKey, - /// This agent's overlay address, once agreed. - pub overlay_address_v4: Option, - /// The IPv4 overlay range in use. - pub ipv4_range: Option, - /// Peers this agent knows about. - pub peers: Vec, -} - -/// One peer of the overlay. -#[derive(Debug, Clone)] -pub struct PeerOverview { - /// The peer's control plane identity. - pub endpoint_id: EndpointId, - /// The peer's WireGuard public key. - pub public_key: WgPublicKey, - /// The overlay address the network agreed it holds. - pub overlay_address_v4: Option, - /// Whether a data plane link to it exists. - pub has_link: bool, - /// The running tunnel, once there is a link. - pub tunnel: Option, -} - -impl PeerOverview { - /// Whether the tunnel to this peer has handshaken and can carry traffic. - pub fn is_up(&self) -> bool { - self.tunnel - .as_ref() - .is_some_and(|tunnel| tunnel.health.is_up()) - } -} +use std::sync::Arc; +use tsunagi::dataplane::{PluginCapability, PluginError, ProtocolOption}; +use tsunagi::{BoxFuture, NetworkId}; +/// Capability and lifecycle adapter for an encrypted UDP carrier. #[derive(Debug)] -struct NetworkState { - key: WgSecretKey, - device: Option>, - announcements: HashMap, - links: HashMap, - allocations: HashMap, - ipv4_range: Option, +pub struct AwgPlugin { + transport: Arc, } - -#[derive(Debug, Default)] -struct Shared { - networks: HashMap, -} - -#[derive(Debug)] -enum Command { - Prepare(NetworkId), - Sync(NetworkId), - Link { - network: NetworkId, - peer: EndpointId, - link: SharedLink, - }, - Teardown(NetworkId), - Stop(oneshot::Sender<()>), -} - -struct Worker { - config: WireguardConfig, - transport: Arc, - local_id: OnceLock, - store: WgKeyStore, - shared: Mutex, - context: OnceLock, - sink: OnceLock>, -} - -impl std::fmt::Debug for Worker { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Worker") - .field("store", &self.store.path()) - .field("port", &self.transport.bound_port()) - .finish() +impl AwgPlugin { + /// The carrier's own NAT discovery setting. + pub const OPTIONS: &'static [ProtocolOption] = &[ProtocolOption { + key: "stun", + value: "HOST:PORT|off", + help: "UDP public mapping discovery; disabled in local reach mode", + default: Some("stun.l.google.com:19302"), + }]; + /// Registers one carrier without creating an internal IP encryption engine. + pub fn new(transport: Arc) -> Arc { + Arc::new(Self { transport }) } } - -/// The pure WireGuard data plane plugin. -#[derive(Debug)] -pub struct WireguardPlugin { - worker: Arc, - commands: mpsc::Sender, - task: Mutex>>, -} - -impl WireguardPlugin { - /// Supported options for pure WireGuard. - pub const OPTIONS: &'static [ProtocolOption] = &[ - ProtocolOption { - key: "port", - value: "PORT", - help: "UDP port to listen on for WireGuard packets", - default: Some("51820"), - }, - ProtocolOption { - key: "keepalive", - value: "SECONDS", - help: "keeps a tunnel and NAT mapping warm; 0 turns it off", - default: Some("25"), - }, - ProtocolOption { - key: "mtu", - value: "BYTES", - help: "largest packet a tunnel will carry, at least 576", - default: Some("1280"), - }, - ]; - - /// Applies `key=value` settings to a configuration. - pub fn configure( - mut config: WireguardConfig, - options: &[(String, String)], - ) -> Result { - for (key, value) in options { - match key.as_str() { - "port" => { - let port: u16 = value.parse().map_err(|_| { - PluginError::Other(format!("port={value} is not a valid port number")) - })?; - config = config.with_port(port); - } - "keepalive" => { - let seconds: u16 = value.parse().map_err(|_| { - PluginError::Other(format!("keepalive={value} is not a number of seconds")) - })?; - config.keepalive = (seconds > 0).then_some(seconds); - } - "mtu" => { - let mtu: u32 = value.parse().map_err(|_| { - PluginError::Other(format!("mtu={value} is not a number of bytes")) - })?; - config = config.with_mtu(mtu); - } - other => { - let known: Vec<&str> = Self::OPTIONS.iter().map(|spec| spec.key).collect(); - return Err(PluginError::Other(format!( - "`{other}` is not a setting of {WG_PROTOCOL}; it takes {}", - known.join(", ") - ))); - } - } - } - Ok(config) - } - - /// Opens the plugin's key store, binds the UDP transport and starts the reconciliation worker. - pub async fn open( - config: WireguardConfig, - transport: Arc, - ) -> Result, PluginError> { - if config.mtu < MIN_MTU { - return Err(PluginError::Other(format!( - "an MTU of {} is below the {MIN_MTU} bytes every IPv4 host must be able to reassemble", - config.mtu - ))); - } - - let path = config.key_store_path(); - let store = tokio::task::spawn_blocking(move || WgKeyStore::open(path)) - .await - .map_err(|err| PluginError::Other(format!("key store task failed: {err}")))??; - - let worker = Arc::new(Worker { - config, - transport, - local_id: OnceLock::new(), - store, - shared: Mutex::new(Shared::default()), - context: OnceLock::new(), - sink: OnceLock::new(), - }); - - let (commands, receiver) = mpsc::channel(64); - let task = tokio::spawn(run(Arc::clone(&worker), receiver)); - - Ok(Arc::new(Self { - worker, - commands, - task: Mutex::new(Some(task)), - })) - } - - /// What this agent has set up for a network. - pub fn overview(&self, network: NetworkId) -> Option { - let shared = self.worker.lock_shared(); - let state = shared.networks.get(&network)?; - - let tunnels: HashMap = state - .device - .as_ref() - .map(|device| { - device - .peers() - .into_iter() - .map(|summary| (summary.public_key, summary)) - .collect() - }) - .unwrap_or_default(); - - let mut peers: Vec = state - .announcements - .iter() - .map(|(endpoint_id, announcement)| PeerOverview { - endpoint_id: *endpoint_id, - public_key: announcement.public_key, - overlay_address_v4: state.allocations.get(endpoint_id).copied(), - has_link: state.links.contains_key(endpoint_id), - tunnel: tunnels.get(&announcement.public_key).cloned(), - }) - .collect(); - peers.sort_by_key(|peer| peer.public_key); - - Some(NetworkOverview { - network, - mtu: self.worker.config.mtu, - public_key: state.key.public(), - overlay_address_v4: state.allocations.get(&self.worker.local_id()).copied(), - ipv4_range: state.ipv4_range, - peers, - }) - } - - fn nudge(&self, command: Command) { - if let Err(err) = self.commands.try_send(command) { - tracing::debug!(%err, "WireGuard command queue is busy"); - } - } -} - -impl Worker { - fn local_id(&self) -> EndpointId { - self.local_id.get().copied().unwrap_or_else(|| { - EndpointId::from_bytes(&[1u8; 32]).unwrap_or_else(|_| unreachable!("a fixed valid key")) - }) - } - - fn lock_shared(&self) -> std::sync::MutexGuard<'_, Shared> { - match self.shared.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - } - } - - fn report(&self, network: NetworkId, reason: impl std::fmt::Display) { - tracing::warn!(network = %network.fmt_short(), %reason, "pure WireGuard plugin error"); - if let Some(context) = self.context.get() { - context.report_error(network, WG_PROTOCOL, reason.to_string()); - } - } - - fn request_reannounce(&self, network: NetworkId) { - if let Some(context) = self.context.get() { - context.request_reannounce(network); - } - } - - async fn prepare(self: &Arc, network: NetworkId) -> Result { - let existing = { - let shared = self.lock_shared(); - shared - .networks - .get(&network) - .map(|state| state.device.is_some()) - }; - - if let Some(has_device) = existing { - if has_device { - return Ok(false); - } - self.ensure_device(network).await?; - return Ok(false); - } - - let worker = Arc::clone(self); - let key = tokio::task::spawn_blocking(move || worker.store.load_or_create(network)) - .await - .map_err(|err| PluginError::Other(format!("key store task failed: {err}")))??; - - { - let mut shared = self.lock_shared(); - shared.networks.entry(network).or_insert(NetworkState { - key, - device: None, - announcements: HashMap::new(), - links: HashMap::new(), - allocations: HashMap::new(), - ipv4_range: None, - }); - } - - let device = self.ensure_device(network).await; - if let Err(err) = device { - self.report(network, err); - } - Ok(true) - } - - async fn ensure_device(&self, network: NetworkId) -> Result<(), PluginError> { - let key = { - let shared = self.lock_shared(); - match shared.networks.get(&network) { - Some(state) if state.device.is_none() => state.key.clone(), - _ => return Ok(()), - } - }; - - let sink = match self.sink.get() { - Some(sink) => Arc::clone(sink), - None => Arc::new(tsunagi::dataplane::DiscardPackets) as Arc, - }; - let context = self - .context - .get() - .cloned() - .unwrap_or_else(PluginContext::detached); - let device = Arc::new(WireguardDevice::start( - network, - key, - sink, - Arc::clone(&self.transport), - context, - )); - - let mut shared = self.lock_shared(); - if let Some(state) = shared.networks.get_mut(&network) { - state.device = Some(device); - } - Ok(()) - } - - fn sync(&self, network: NetworkId) { - let mut shared = self.lock_shared(); - let Some(state) = shared.networks.get_mut(&network) else { - return; - }; - let Some(device) = state.device.clone() else { - return; - }; - - state.links.retain(|_, link| !link.is_closed()); - - let allocations = state.allocations.clone(); - let mut wanted: Vec = Vec::new(); - for (endpoint_id, announcement) in &state.announcements { - let Some(link) = state.links.get(endpoint_id) else { - continue; - }; - if link.is_closed() { - continue; - } - wanted.push(announcement.public_key); - if device.has_peer(&announcement.public_key) { - continue; - } - - let peer_v4 = allocations.get(endpoint_id).copied(); - if let Err(err) = device.add_peer( - *endpoint_id, - announcement.public_key, - peer_v4, - Arc::clone(link), - self.config.keepalive, - ) { - tracing::debug!(%err, "cannot start WireGuard tunnel"); - } - } - device.retain_peers(&wanted); - } - - async fn teardown(&self, network: NetworkId) { - self.lock_shared().networks.remove(&network); - } - - fn known_networks(&self) -> Vec { - self.lock_shared().networks.keys().copied().collect() - } -} - -async fn run(worker: Arc, mut commands: mpsc::Receiver) { - let mut pending: BTreeSet = BTreeSet::new(); - let mut deadline: Option = None; - let mut ticker = tokio::time::interval(worker.config.reconcile_interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - ticker.tick().await; - - loop { - let wait_until = deadline; - tokio::select! { - biased; - command = commands.recv() => { - let Some(command) = command else { break }; - match command { - Command::Prepare(network) => { - match worker.prepare(network).await { - Ok(true) => worker.request_reannounce(network), - Ok(false) => {} - Err(err) => worker.report(network, err), - } - pending.insert(network); - } - Command::Sync(network) => { - pending.insert(network); - } - Command::Link { network, peer, link } => { - { - let mut shared = worker.lock_shared(); - if let Some(state) = shared.networks.get_mut(&network) { - state.links.insert(peer, link); - } - } - pending.insert(network); - } - Command::Teardown(network) => { - pending.remove(&network); - worker.teardown(network).await; - continue; - } - Command::Stop(reply) => { - for network in worker.known_networks() { - worker.teardown(network).await; - } - let _ = reply.send(()); - return; - } - } - deadline = Some(tokio::time::Instant::now() + worker.config.reconcile_debounce); - } - _ = async { - match wait_until { - Some(at) => tokio::time::sleep_until(at).await, - None => std::future::pending::<()>().await, - } - }, if wait_until.is_some() => { - deadline = None; - for network in std::mem::take(&mut pending) { - worker.sync(network); - } - } - _ = ticker.tick() => { - for network in worker.known_networks() { - if let Err(err) = worker.ensure_device(network).await { - tracing::debug!(%err, "WireGuard device still unavailable"); - } - worker.sync(network); - } - } - } - } -} - -impl IpPlugin for WireguardPlugin { +impl tsunagi::dataplane::transport::DataCarrier for AwgPlugin { fn protocol_id(&self) -> &str { - WG_PROTOCOL + self.transport.protocol_id() } - fn protocol_version(&self) -> u16 { - ANNOUNCEMENT_VERSION + self.transport.wire_version() } - fn options(&self) -> &'static [ProtocolOption] { Self::OPTIONS } - - fn attach(&self, context: PluginContext) { - if let Some(local) = context.local_endpoint_id() { - let _ = self.worker.local_id.set(local); - } - let _ = self.worker.sink.set(context.packet_sink()); - let _ = self.worker.context.set(context); + fn on_interface_ready(&self, name: &str) { + self.transport.exclude_interface(name.into()); } - - fn on_network_activated(&self, network: NetworkId) { - self.nudge(Command::Prepare(network)); - } - - fn local_capability( - &self, - network: NetworkId, - ) -> Result, PluginError> { - let shared = self.worker.lock_shared(); - let Some(state) = shared.networks.get(&network) else { - drop(shared); - self.nudge(Command::Prepare(network)); - return Ok(None); - }; - - let bound_port = self.worker.transport.bound_port(); - let addrs = local_ip_candidates(); - let announcement = WgAnnouncement::new(network, &state.key.public(), bound_port, addrs); - + fn local_capability(&self, _: NetworkId) -> Result, PluginError> { Ok(Some(PluginCapability { - protocol: WG_PROTOCOL.to_string(), - version: ANNOUNCEMENT_VERSION, + protocol: self.protocol_id().into(), + version: self.protocol_version(), enabled: true, - data: announcement.encode()?, + data: Vec::new(), })) } - fn on_peer_capability( &self, network: NetworkId, peer: EndpointId, capability: &PluginCapability, ) -> Result<(), PluginError> { - let local_key = { - let shared = self.worker.lock_shared(); - match shared.networks.get(&network) { - Some(state) => state.key.public(), - None => { - drop(shared); - self.nudge(Command::Prepare(network)); - return Err(PluginError::Unavailable( - "WireGuard is not ready for this network yet".into(), - )); - } - } - }; - - let validated = WgAnnouncement::decode_and_validate(&capability.data, network, &local_key)?; - - let is_overlay_ip = |ip: &IpAddr| -> bool { - let shared = self.worker.lock_shared(); - shared.networks.values().any(|s| { - if let Some(range) = s.ipv4_range { - match ip { - IpAddr::V4(v4) => range.contains(*v4), - IpAddr::V6(_) => false, - } - } else { - false - } - }) - }; - - let socket_addrs: Vec = validated - .addrs - .iter() - .filter(|ip| is_candidate_ip(**ip) && !is_overlay_ip(ip)) - .map(|ip| SocketAddr::new(*ip, validated.port)) - .collect(); - - // Record candidate addresses in transport and optionally notify if we are the acceptor - let local_id = self.worker.local_id(); - let we_accept = local_id.as_bytes() > peer.as_bytes(); - self.worker - .transport - .notify_peer_capability(network, peer, &socket_addrs, we_accept); - - let changed = { - let mut shared = self.worker.lock_shared(); - match shared.networks.get_mut(&network) { - Some(state) => { - state.announcements.insert(peer, validated.clone()) != Some(validated) - } - None => false, - } - }; - if changed { - self.nudge(Command::Sync(network)); + if capability.protocol != self.protocol_id() + || capability.version != self.protocol_version() + || !capability.enabled + || !capability.data.is_empty() + { + return Err(PluginError::Rejected( + "unsupported UDP carrier capability".into(), + )); } + self.transport.allow_peer(network, peer); Ok(()) } - - fn on_address_allocation( - &self, - network: NetworkId, - range: Ipv4Range, - allocations: &[(EndpointId, Ipv4Addr)], - ) { - let changed = { - let mut shared = self.worker.lock_shared(); - match shared.networks.get_mut(&network) { - Some(state) => { - let fresh: HashMap = - allocations.iter().copied().collect(); - let changed = state.allocations != fresh || state.ipv4_range != Some(range); - state.allocations = fresh; - state.ipv4_range = Some(range); - changed - } - None => false, - } - }; - if changed { - self.nudge(Command::Sync(network)); - } - } - - fn carry(&self, network: NetworkId, peer: EndpointId, packet: bytes::Bytes) -> bool { - let shared = self.worker.lock_shared(); - shared - .networks - .get(&network) - .and_then(|state| state.device.as_ref()) - .is_some_and(|device| device.carry(peer, &packet)) - } - - fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) { - self.nudge(Command::Link { - network, - peer, - link, - }); - } - fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) { - let removed = { - let mut shared = self.worker.lock_shared(); - match shared.networks.get_mut(&network) { - Some(state) => { - let had_link = state.links.remove(&peer).is_some(); - state.announcements.remove(&peer).is_some() || had_link - } - None => false, - } - }; - if removed { - self.nudge(Command::Sync(network)); - } + self.transport.forget_peer(network, peer); } - fn on_network_deactivated(&self, network: NetworkId) { - self.nudge(Command::Teardown(network)); + self.transport.stop_network(network); } - - fn on_network_forgotten(&self, network: NetworkId) { - if let Err(err) = self.worker.store.forget(network) { - tracing::warn!(%err, "could not forget WireGuard key for network"); - } - } - - fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> { + fn shutdown(&self) -> BoxFuture<'_, ()> { Box::pin(async move { - let (tx, rx) = oneshot::channel(); - if self.commands.send(Command::Stop(tx)).await.is_ok() { - let _ = rx.await; - } - let task = match self.task.lock() { - Ok(mut guard) => guard.take(), - Err(poisoned) => poisoned.into_inner().take(), - }; - if let Some(task) = task { - let _ = task.await; - } + self.transport.shutdown(); }) } } - -#[cfg(test)] -mod tests { - use super::*; - use std::net::{Ipv4Addr, Ipv6Addr}; - - #[test] - fn test_is_candidate_interface_filters_overlay_and_bridges() { - let mut iface = netdev::Interface::dummy(); - iface.flags = 1; // UP - - // Standard Ethernet - should pass - iface.name = "eth0".to_string(); - assert!(is_candidate_interface(&iface)); - - // Standard Wi-Fi - should pass - iface.name = "wlan0".to_string(); - assert!(is_candidate_interface(&iface)); - - // Windows Ethernet with GUID name - should pass - iface.name = "{B6149D2C-1234-5678-9ABC-DEF012345678}".to_string(); - iface.friendly_name = Some("Ethernet".to_string()); - iface.description = Some("Intel(R) Ethernet Connection".to_string()); - assert!(is_candidate_interface(&iface)); - - // Linux tsun0 overlay - must be filtered - iface.name = "tsun0".to_string(); - iface.friendly_name = None; - iface.description = None; - assert!(!is_candidate_interface(&iface)); - - // Windows Wintun overlay (GUID name with tsun0 friendly name) - must be filtered! - iface.name = "{72D85D76-F1A5-42AC-BC49-2169527CDDDF}".to_string(); - iface.friendly_name = Some("tsun0".to_string()); - iface.description = Some("Wintun Userspace Tunnel".to_string()); - assert!(!is_candidate_interface(&iface)); - - // Windows Wintun overlay with description containing Wintun - must be filtered! - iface.friendly_name = Some("Local Area Connection 2".to_string()); - iface.description = Some("Wintun Userspace Tunnel".to_string()); - assert!(!is_candidate_interface(&iface)); - - // Docker bridge - must be filtered - iface.name = "docker0".to_string(); - iface.friendly_name = None; - iface.description = None; - assert!(!is_candidate_interface(&iface)); - - // Docker network bridge br-* - must be filtered - iface.name = "br-4f6d3a9b1c2e".to_string(); - assert!(!is_candidate_interface(&iface)); - - // Virtual ethernet veth* - must be filtered - iface.name = "veth12345".to_string(); - assert!(!is_candidate_interface(&iface)); - - // Libvirt NAT bridge virbr* - allowed for VM guests/hosts - iface.name = "virbr0".to_string(); - assert!(is_candidate_interface(&iface)); - - // CNI bridge - must be filtered - iface.name = "cni0".to_string(); - assert!(!is_candidate_interface(&iface)); - - // WSL vEthernet switch on Windows - must be filtered - iface.name = "{A1B2C3D4-E5F6-7890-1234-567890ABCDEF}".to_string(); - iface.friendly_name = Some("vEthernet (WSL)".to_string()); - iface.description = Some("Hyper-V Virtual Ethernet Adapter".to_string()); - assert!(!is_candidate_interface(&iface)); - - // Down interface - must be filtered - let mut down_iface = netdev::Interface::dummy(); - down_iface.flags = 0; // Not UP - down_iface.name = "eth0".to_string(); - assert!(!is_candidate_interface(&down_iface)); - } - - #[test] - fn test_is_candidate_ip() { - // Valid LAN IP - assert!(is_candidate_ip(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 117)))); - // Valid Tailscale IP - assert!(is_candidate_ip(IpAddr::V4(Ipv4Addr::new( - 100, 77, 155, 120 - )))); - // Valid Public IP - assert!(is_candidate_ip(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)))); - // Valid IPv6 GUA - assert!(is_candidate_ip(IpAddr::V6(Ipv6Addr::new( - 0x2a01, 0x4b00, 0xb8e3, 0x4e00, 0, 0, 0, 1 - )))); - - // Loopback - must be rejected - assert!(!is_candidate_ip(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)))); - assert!(!is_candidate_ip(IpAddr::V6(Ipv6Addr::LOCALHOST))); - - // Unspecified - must be rejected - assert!(!is_candidate_ip(IpAddr::V4(Ipv4Addr::UNSPECIFIED))); - assert!(!is_candidate_ip(IpAddr::V6(Ipv6Addr::UNSPECIFIED))); - - // Link-local - must be rejected - assert!(!is_candidate_ip(IpAddr::V4(Ipv4Addr::new(169, 254, 1, 1)))); - assert!(!is_candidate_ip(IpAddr::V6(Ipv6Addr::new( - 0xfe80, 0, 0, 0, 0, 0x1ff, 0xfe00, 1 - )))); - - // Broadcast - must be rejected - assert!(!is_candidate_ip(IpAddr::V4(Ipv4Addr::BROADCAST))); - } - - #[test] - fn test_ip_priority_ordering() { - let mut iface_lan = netdev::Interface::dummy(); - iface_lan.flags = 1; - iface_lan.gateway = Some(netdev::net::device::NetworkDevice::new()); - - let mut iface_vpn = netdev::Interface::dummy(); - iface_vpn.flags = 1; - iface_vpn.name = "tailscale0".to_string(); - - let lan_ip = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 117)); - let tailscale_ip = IpAddr::V4(Ipv4Addr::new(100, 77, 155, 120)); - - let p_lan = ip_priority(&iface_lan, lan_ip); - let p_ts = ip_priority(&iface_vpn, tailscale_ip); - - // LAN IP with gateway should score higher than VPN IP without gateway - assert!(p_lan > p_ts); - } - - #[test] - fn test_matches_subnet() { - let local_ip = Ipv4Addr::new(192, 168, 122, 151); - let same_subnet = Ipv4Addr::new(192, 168, 122, 1); - let diff_subnet = Ipv4Addr::new(192, 168, 1, 2); - - assert!(matches_subnet(local_ip, 24, same_subnet)); - assert!(!matches_subnet(local_ip, 24, diff_subnet)); - } -} diff --git a/crates/tsunagi-awg/src/profile.rs b/crates/tsunagi-wireguard/src/profile.rs similarity index 91% rename from crates/tsunagi-awg/src/profile.rs rename to crates/tsunagi-wireguard/src/profile.rs index 9ef8284..013abc9 100644 --- a/crates/tsunagi-awg/src/profile.rs +++ b/crates/tsunagi-wireguard/src/profile.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Serialize, Deserialize, zeroize::ZeroizeOnDrop)] pub struct PairProfile { id: [u8; 16], + masked: bool, prefix: u16, headers: [[u32; 2]; 4], header_key: [u8; 32], @@ -31,6 +32,7 @@ impl PairProfile { }); Self { id: rand::random(), + masked: true, prefix: rand::random_range(12..49), headers, header_key: rand::random(), @@ -42,6 +44,16 @@ impl PairProfile { signature: rand::random_range(0..2), } } + /// A fresh ordinary WireGuard profile, without masking or signature packets. + pub fn wireguard() -> Self { + let mut profile = Self::generate(); + profile.masked = false; + profile + } + /// Whether this profile uses Amnezia masking. + pub fn is_masked(&self) -> bool { + self.masked + } /// Public attempt identifier, useful for distinguishing retries without revealing secrets. pub fn id(&self) -> [u8; 16] { self.id @@ -109,6 +121,9 @@ impl PairProfile { } /// Masks one standard WireGuard message; invalid types and oversized packets are refused. pub fn encode(&self, raw: &[u8]) -> Option { + if !self.masked { + return (raw.len() <= 1200).then(|| Bytes::copy_from_slice(raw)); + } let kind = match raw.get(..4)? { [1, 0, 0, 0] => AwgKind::Initiation, [2, 0, 0, 0] => AwgKind::Response, @@ -132,6 +147,9 @@ impl PairProfile { } /// Unmasks a packet and restores standard headers before WireGuard validation. pub fn decode(&self, wire: &[u8]) -> Option> { + if !self.masked { + return (wire.len() <= 1200).then(|| wire.to_vec()); + } if wire.len() > 1200 { return None; } @@ -147,11 +165,18 @@ impl PairProfile { Some(message) } pub(crate) fn pad(&self, packet: Bytes) -> Bytes { - self.config().prepare_transport_plaintext(packet, 1200) + if self.masked { + self.config().prepare_transport_plaintext(packet, 1200) + } else { + packet + } } /// Generates structured signature packets with fresh fields, never static captures. /// These target the other Tsunagi peer, not DNS or SIP services. pub fn signature_packets(&self) -> Vec> { + if !self.masked { + return Vec::new(); + } if self.signature == 0 { // A complete DNS query. Counts, label lengths and qtype remain valid. let label = format!("n{:016x}", rand::random::()); @@ -169,6 +194,9 @@ impl PairProfile { } } pub(crate) fn junk_packets(&self) -> Vec> { + if !self.masked { + return Vec::new(); + } (0..self.junk_count) .map(|_| { let mut packet = vec![0; rand::random_range(self.junk_min..=self.junk_max).into()]; diff --git a/crates/tsunagi-awg/src/session.rs b/crates/tsunagi-wireguard/src/session.rs similarity index 95% rename from crates/tsunagi-awg/src/session.rs rename to crates/tsunagi-wireguard/src/session.rs index fbf1b81..9898083 100644 --- a/crates/tsunagi-awg/src/session.rs +++ b/crates/tsunagi-wireguard/src/session.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use iroh::EndpointId; use std::collections::{HashMap, VecDeque}; use std::net::{Ipv4Addr, SocketAddr}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::net::UdpSocket; @@ -21,6 +21,7 @@ const WINDOW: Duration = Duration::from_secs(20); pub(crate) struct PairLink { network: NetworkId, + protocol: &'static str, peer: EndpointId, pub(crate) attempt: [u8; 16], tx: mpsc::Sender, @@ -29,6 +30,7 @@ pub(crate) struct PairLink { closed: AtomicBool, remote: Mutex>, reason: Mutex>, + checks: Arc, } impl std::fmt::Debug for PairLink { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -39,6 +41,9 @@ impl std::fmt::Debug for PairLink { } } impl PairLink { + pub(crate) fn delivery_checks(&self) -> u64 { + self.checks.load(Ordering::Relaxed) + } pub(crate) fn has_authenticated_path(&self) -> bool { match self.remote.lock() { Ok(remote) => remote.is_some(), @@ -103,8 +108,8 @@ impl PacketLink for PairLink { fn path_description(&self) -> String { match self.remote.lock() { Ok(addr) => match *addr { - Some(addr) => format!("awg UDP via {addr}"), - None => "awg UDP, establishing".into(), + Some(addr) => format!("{} UDP via {addr}", self.protocol), + None => format!("{} UDP, establishing", self.protocol), }, Err(_) => "awg UDP".into(), } @@ -117,19 +122,21 @@ impl Drop for PairLink { } pub(crate) fn start( - network: NetworkId, - peer: EndpointId, + pair: (NetworkId, EndpointId), socket: UdpSocket, tunn: Tunn, profile: PairProfile, candidates: Vec, + protocol: &'static str, parent: watch::Receiver, ) -> Arc { + let (network, peer) = pair; let (tx, rx) = mpsc::channel(QUEUE); let (in_tx, in_rx) = mpsc::channel(QUEUE); let (stop, _) = watch::channel(false); let link = Arc::new(PairLink { network, + protocol, peer, attempt: profile.id(), tx, @@ -138,8 +145,10 @@ pub(crate) fn start( closed: AtomicBool::new(false), remote: Mutex::new(None), reason: Mutex::new(None), + checks: Arc::new(AtomicU64::new(0)), }); let weak = Arc::downgrade(&link); + let checks = link.checks.clone(); tokio::spawn(async move { let mut engine = Engine { stop: stop.subscribe(), @@ -157,6 +166,7 @@ pub(crate) fn start( assemblies: HashMap::new(), started: Instant::now(), last_authenticated: Instant::now(), + checks, }; engine.run(parent, &weak).await; if let Some(link) = weak.upgrade() { @@ -188,6 +198,7 @@ struct Engine { assemblies: HashMap, started: Instant, last_authenticated: Instant, + checks: Arc, } enum ResultOwned { Wire(Vec), @@ -352,6 +363,7 @@ impl Engine { { self.pending.remove(index); self.samples.push_back(true); + self.checks.fetch_add(1, Ordering::Relaxed); } } Some(3) => self.fragment(seq, payload), diff --git a/crates/tsunagi-wireguard/src/store.rs b/crates/tsunagi-wireguard/src/store.rs deleted file mode 100644 index 57ac898..0000000 --- a/crates/tsunagi-wireguard/src/store.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! The plugin's own key store. -//! -//! Deliberately a separate SQLite file from the agent's `state.sqlite`: plugin -//! keys are not the iroh identity and not the network secret, and their -//! lifecycle is the plugin's business alone. -//! -//! One key per network, so a participant presents a different WireGuard -//! identity in each network it belongs to. - -use std::path::{Path, PathBuf}; -use std::sync::Mutex; - -use rusqlite::{Connection, OptionalExtension, params}; - -use tsunagi::dataplane::PluginError; -use tsunagi::identity::NetworkId; - -use crate::keys::{KEY_LEN, WgSecretKey}; - -/// Schema version written by this build. -pub const SCHEMA_VERSION: i64 = 1; - -/// Per-network WireGuard private keys. -#[derive(Debug)] -pub struct WgKeyStore { - conn: Mutex, - path: PathBuf, -} - -impl WgKeyStore { - /// Opens, creating the file and its directory if needed. - pub fn open(path: impl AsRef) -> Result { - let path = path.as_ref().to_path_buf(); - if let Some(parent) = path.parent() { - tsunagi::storage::create_dir(parent) - .map_err(|err| PluginError::Other(format!("cannot create {parent:?}: {err}")))?; - } - let existed = path.exists(); - let conn = Connection::open(&path).map_err(|err| { - PluginError::Other(format!("cannot open the WireGuard key store: {err}")) - })?; - tsunagi::storage::restrict_path_permissions(&path) - .map_err(|err| PluginError::Other(format!("cannot secure the key store: {err}")))?; - - conn.busy_timeout(std::time::Duration::from_secs(5)) - .and_then(|()| conn.pragma_update(None, "journal_mode", "WAL")) - .and_then(|()| conn.pragma_update(None, "synchronous", "NORMAL")) - .map_err(|err| PluginError::Other(format!("cannot configure the key store: {err}")))?; - - if existed { - let integrity: String = conn - .query_row("PRAGMA integrity_check", [], |row| row.get(0)) - .map_err(|err| { - PluginError::Other(format!("WireGuard key store is unusable: {err}")) - })?; - if integrity != "ok" { - return Err(PluginError::Other(format!( - "WireGuard key store at {} is corrupt and will not be recreated: {integrity}", - path.display() - ))); - } - } - - let found: i64 = conn - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .map_err(|err| PluginError::Other(format!("cannot read the schema version: {err}")))?; - if found > SCHEMA_VERSION { - return Err(PluginError::Other(format!( - "WireGuard key store schema {found} is newer than {SCHEMA_VERSION}" - ))); - } - if found < SCHEMA_VERSION { - conn.execute_batch( - "BEGIN; - CREATE TABLE IF NOT EXISTS network_keys ( - network_id BLOB PRIMARY KEY, - secret BLOB NOT NULL, - created_at INTEGER NOT NULL - ); - PRAGMA user_version = 1; - COMMIT;", - ) - .map_err(|err| PluginError::Other(format!("cannot create the schema: {err}")))?; - } - - Ok(Self { - conn: Mutex::new(conn), - path, - }) - } - - /// Path of the underlying file. - pub fn path(&self) -> &Path { - &self.path - } - - /// Loads the stored key for `network`, or generates, stores and returns a fresh one. - pub fn load_or_create(&self, network: NetworkId) -> Result { - let guard = self - .conn - .lock() - .map_err(|_| PluginError::Other("key store lock is poisoned".into()))?; - - let existing: Option> = guard - .query_row( - "SELECT secret FROM network_keys WHERE network_id = ?1", - params![network.as_bytes().as_slice()], - |row| row.get(0), - ) - .optional() - .map_err(|err| PluginError::Other(format!("cannot read the key store: {err}")))?; - - if let Some(secret) = existing { - let bytes = <[u8; KEY_LEN]>::try_from(secret.as_slice()).map_err(|_| { - PluginError::Other(format!( - "stored WireGuard key for {} is {} bytes, expected {KEY_LEN}", - network.fmt_short(), - secret.len() - )) - })?; - return Ok(WgSecretKey::from_bytes(&bytes)); - } - - let fresh = WgSecretKey::generate(); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_err(|err| PluginError::Other(format!("system clock is broken: {err}")))? - .as_secs() as i64; - - guard - .execute( - "INSERT INTO network_keys (network_id, secret, created_at) VALUES (?1, ?2, ?3)", - params![ - network.as_bytes().as_slice(), - fresh.as_bytes().as_slice(), - now - ], - ) - .map_err(|err| PluginError::Other(format!("cannot write the key store: {err}")))?; - - Ok(fresh) - } - - /// Removes the stored key for `network`. - pub fn forget(&self, network: NetworkId) -> Result<(), PluginError> { - let guard = self - .conn - .lock() - .map_err(|_| PluginError::Other("key store lock is poisoned".into()))?; - guard - .execute( - "DELETE FROM network_keys WHERE network_id = ?1", - params![network.as_bytes().as_slice()], - ) - .map_err(|err| { - PluginError::Other(format!("cannot delete from the key store: {err}")) - })?; - Ok(()) - } -} diff --git a/crates/tsunagi-awg/src/stun.rs b/crates/tsunagi-wireguard/src/stun.rs similarity index 100% rename from crates/tsunagi-awg/src/stun.rs rename to crates/tsunagi-wireguard/src/stun.rs diff --git a/crates/tsunagi-wireguard/src/transport.rs b/crates/tsunagi-wireguard/src/transport.rs index bbb3929..31a26a5 100644 --- a/crates/tsunagi-wireguard/src/transport.rs +++ b/crates/tsunagi-wireguard/src/transport.rs @@ -1,799 +1,790 @@ -//! Pure WireGuard UDP packet transport and link implementation. -//! -//! Provides [`WireguardTransport`] implementing [`PacketTransport`] -//! and [`WireguardLink`] implementing [`PacketLink`]. - -use std::collections::{HashMap, HashSet}; -use std::net::SocketAddr; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; - -use bytes::Bytes; -use iroh::EndpointId; -use tokio::net::UdpSocket; -use tokio::sync::{Notify, mpsc, oneshot}; - -use tsunagi::BoxFuture; -use tsunagi::dataplane::transport::{ - InboundLink, PacketLink, PacketTransport, SharedLink, TransportError, +//! Private pair negotiation. No secret profiles in capabilities or durable state. +use crate::{ + AWG_PROTOCOL, AWG_WIRE_VERSION, + profile::PairProfile, + session::{self, PairLink}, }; -use tsunagi::identity::{DeviceIdentity, NetworkId}; +use boringtun::{ + noise::Tunn, + x25519::{PublicKey, StaticSecret}, +}; +use iroh::EndpointId; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::time::Duration; +use tokio::net::UdpSocket; +use tokio::sync::{mpsc, oneshot, watch}; +use tsunagi::dataplane::transport::TransportPayload; +use tsunagi::dataplane::transport::{ + ControlSender, InboundLink, PacketLink, PacketTransport, SharedLink, TransportError, +}; +use tsunagi::{BoxFuture, NetworkId}; -/// Default UDP port to bind and listen on for pure WireGuard. -pub const DEFAULT_PORT: u16 = 51820; - -/// Maximum payload length for a single framed datagram over UDP. -pub const MAX_UDP_DATAGRAM_SIZE: usize = 65507; - -const PROBE_MAGIC: &[u8; 19] = b"TSUNAGI_WG_PROBE_V1"; -const PROBE_TYPE_PING: u8 = 1; -const PROBE_TYPE_PONG: u8 = 2; -const PROBE_LEN: usize = 19 + 1 + 32 + 32 + 32 + 8; // 124 bytes - -fn encode_probe( - msg_type: u8, - sender: EndpointId, - receiver: EndpointId, - network: NetworkId, - cookie: u64, -) -> Vec { - let mut buf = Vec::with_capacity(PROBE_LEN); - buf.extend_from_slice(PROBE_MAGIC); - buf.push(msg_type); - buf.extend_from_slice(sender.as_bytes()); - buf.extend_from_slice(receiver.as_bytes()); - buf.extend_from_slice(network.as_bytes()); - buf.extend_from_slice(&cookie.to_be_bytes()); - buf +type Pair = (NetworkId, EndpointId); +#[derive(Serialize, Deserialize)] +enum Negotiation { + Offer { + version: u16, + profile: PairProfile, + key: [u8; 32], + #[serde(deserialize_with = "bounded_addresses")] + addresses: Vec, + }, + Answer { + attempt: [u8; 16], + key: [u8; 32], + #[serde(deserialize_with = "bounded_addresses")] + addresses: Vec, + }, } - -struct DecodedProbe { - msg_type: u8, - sender: EndpointId, - receiver: EndpointId, - network: NetworkId, - cookie: u64, -} - -fn decode_probe(bytes: &[u8]) -> Option { - if bytes.len() < PROBE_LEN || &bytes[..19] != PROBE_MAGIC { - return None; +fn bounded_addresses<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Vec; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("at most 32 UDP candidates") + } + fn visit_seq>( + self, + mut sequence: A, + ) -> Result { + if sequence.size_hint().is_some_and(|n| n > 32) { + return Err(serde::de::Error::custom("too many UDP candidates")); + } + let mut addresses = Vec::with_capacity(sequence.size_hint().unwrap_or(0).min(32)); + while let Some(address) = sequence.next_element()? { + if addresses.len() == 32 { + return Err(serde::de::Error::custom("too many UDP candidates")); + } + addresses.push(address); + } + Ok(addresses) + } } - let msg_type = bytes[19]; - if msg_type != PROBE_TYPE_PING && msg_type != PROBE_TYPE_PONG { - return None; + deserializer.deserialize_seq(Visitor) +} +struct Pending { + profile: PairProfile, + key: StaticSecret, + socket: UdpSocket, + reply: oneshot::Sender>, +} +struct Inner { + stopped: watch::Sender, + local: EndpointId, + protocol: &'static str, + port: u16, + bind: Ipv4Addr, + stun: Option, + excluded_interface: Mutex>, + busy: Mutex>, + control: OnceLock>, + allowed: Mutex>, + pending: Mutex>, + links: Mutex>>, + inbound: mpsc::Sender, + port_by_pair: Mutex>, +} +/// Public diagnostics for one pair. No key or masking parameters are exposed. +#[derive(Debug, Clone)] +pub struct PairOverview { + /// Other endpoint's authenticated identity. + pub peer: EndpointId, + /// Fresh connection-attempt identifier. + pub attempt: [u8; 16], + /// Last known UDP port; None if diagnostics race with pair retirement. + pub local_port: Option, + /// Actual transport path, if established. + pub path: String, + /// Whether this attempt has been retired. + pub closed: bool, + /// Authenticated probe replies received over this physical pair. + pub delivery_checks: u64, +} +/// A bounded set of independently configured pair engines. +pub struct AwgTransport { + inner: Arc, + commands: mpsc::Sender<(NetworkId, EndpointId, Negotiation)>, + incoming: tokio::sync::Mutex>, +} +impl std::fmt::Debug for AwgTransport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AwgTransport") + .field("local", &self.inner.local) + .finish_non_exhaustive() } - let sender = EndpointId::from_bytes(&bytes[20..52].try_into().ok()?).ok()?; - let receiver = EndpointId::from_bytes(&bytes[52..84].try_into().ok()?).ok()?; - let network = NetworkId::from_bytes(bytes[84..116].try_into().ok()?); - let cookie = u64::from_be_bytes(bytes[116..124].try_into().ok()?); - Some(DecodedProbe { - msg_type, - sender, - receiver, - network, - cookie, - }) +} +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + match mutex.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + } +} +fn error(text: &str) -> TransportError { + TransportError::Other(text.into()) +} +async fn addresses(inner: &Inner, socket: &UdpSocket) -> Result, TransportError> { + let local = socket + .local_addr() + .map_err(|_| error("UDP socket address unavailable"))?; + let mut candidates = if inner.bind.is_loopback() { + vec![local] + } else { + crate::addresses::local_ip_candidates_excluding(lock(&inner.excluded_interface).as_deref()) + .into_iter() + .filter(|ip| ip.is_ipv4()) + .take(31) + .map(|ip| SocketAddr::new(ip, local.port())) + .collect() + }; + if let Some(server) = &inner.stun + && let Some(mapped) = crate::stun::mapped(socket, server).await + && !candidates.contains(&mapped) + { + candidates.push(mapped); + } + if candidates.is_empty() { + return Err(error("UDP has no UDP address candidates")); + } + Ok(candidates) +} +async fn bind_pair(inner: &Inner) -> Result { + match UdpSocket::bind((inner.bind, inner.port)).await { + Ok(socket) => Ok(socket), + Err(err) if inner.port != 0 && err.kind() == std::io::ErrorKind::AddrInUse => { + UdpSocket::bind((inner.bind, 0)) + .await + .map_err(|_| error("UDP pair bind failed")) + } + Err(_) => Err(error("UDP pair bind failed")), + } +} +fn validate_endpoint(key: &[u8; 32], addresses: &[SocketAddr]) -> Result<(), TransportError> { + if *key == [0; 32] + || addresses.is_empty() + || addresses.len() > 32 + || addresses.iter().any(|address| { + address.port() == 0 + || !address.is_ipv4() + || address.ip().is_unspecified() + || address.ip().is_multicast() + || matches!(address.ip(), std::net::IpAddr::V4(ip) if ip.is_broadcast()) + }) + { + return Err(error("invalid AWG endpoint")); + } + Ok(()) } -/// An authenticated datagram link carried over direct UDP. -#[derive(Debug)] -pub struct WireguardLink { - network: NetworkId, - peer: EndpointId, - target_addr: Arc>, - candidates: Arc>>, - confirmed: Arc, - tx: mpsc::Sender, - inbound_tx: mpsc::Sender, - rx: tokio::sync::Mutex>, - closed_notify: Arc, - is_closed: Arc, -} - -impl WireguardLink { - /// Creates a new `WireguardLink` targeting `target_addr` over `socket`. - pub fn new( - network: NetworkId, - peer: EndpointId, - target_addr: SocketAddr, - candidates: Vec, - socket: Arc, - ) -> Self { - let (tx_out, mut rx_out) = mpsc::channel::(1024); - let (tx_in, rx_in) = mpsc::channel::(1024); - - let is_closed = Arc::new(AtomicBool::new(false)); - let confirmed = Arc::new(AtomicBool::new(false)); - let closed_notify = Arc::new(Notify::new()); - - let target_holder = Arc::new(RwLock::new(target_addr)); - let candidates_holder = Arc::new(RwLock::new(candidates)); - - let socket_w = Arc::clone(&socket); - let target_r = Arc::clone(&target_holder); - let candidates_r = Arc::clone(&candidates_holder); - let confirmed_r = Arc::clone(&confirmed); - let is_closed_w = Arc::clone(&is_closed); - let closed_notify_w = Arc::clone(&closed_notify); - +impl AwgTransport { + /// Starts the coordinator. Production binds wildcard IPv4; tests use loopback. + pub fn new(local: EndpointId, bind: Ipv4Addr) -> Arc { + Self::new_with_stun(local, bind, None) + } + /// Enables an optional STUN server (host:port) for this pair's own socket. + /// Local-only callers pass None; failure still permits local UDP candidates. + pub fn new_with_stun(local: EndpointId, bind: Ipv4Addr, stun: Option) -> Arc { + Self::for_protocol(local, bind, stun, AWG_PROTOCOL, 0) + } + /// Builds an ordinary WireGuard UDP carrier with independent per-link keys. + pub fn wireguard(local: EndpointId, bind: Ipv4Addr, stun: Option) -> Arc { + Self::for_protocol(local, bind, stun, "wg", 0) + } + /// Uses a preferred ordinary WireGuard port; additional pairs get dynamic ports. + pub fn wireguard_with_port( + local: EndpointId, + bind: Ipv4Addr, + stun: Option, + port: u16, + ) -> Arc { + Self::for_protocol(local, bind, stun, "wg", port) + } + /// Excludes the agent's own interface, including custom names, from local candidates. + pub fn exclude_interface(&self, name: String) { + *lock(&self.inner.excluded_interface) = Some(name); + } + /// Wire version of this carrier's private negotiation and framing. + pub fn wire_version(&self) -> u16 { + if self.inner.protocol == "wg" { + crate::WG_WIRE_VERSION + } else { + AWG_WIRE_VERSION + } + } + /// The external wire protocol, independent of the internal tunnel. + pub fn protocol_id(&self) -> &str { + self.inner.protocol + } + fn for_protocol( + local: EndpointId, + bind: Ipv4Addr, + stun: Option, + protocol: &'static str, + port: u16, + ) -> Arc { + let (commands, mut rx) = mpsc::channel(64); + let (inbound, incoming) = mpsc::channel(64); + let (stopped, _) = watch::channel(false); + let inner = Arc::new(Inner { + stopped, + local, + protocol, + port, + bind, + stun, + excluded_interface: Mutex::new(None), + busy: Mutex::default(), + control: OnceLock::new(), + allowed: Mutex::default(), + pending: Mutex::default(), + links: Mutex::default(), + inbound, + port_by_pair: Mutex::default(), + }); + let weak = Arc::downgrade(&inner); + let mut stopped = inner.stopped.subscribe(); tokio::spawn(async move { - while let Some(msg) = rx_out.recv().await { - if msg.len() > MAX_UDP_DATAGRAM_SIZE { + while let Some((network, peer, message)) = + tokio::select! { biased; _=stopped.changed()=>None, message=rx.recv()=>message } + { + let Some(inner) = weak.upgrade() else { + break; + }; + if !lock(&inner.allowed).contains(&(network, peer)) + || !lock(&inner.busy).insert((network, peer)) + { continue; } - // In WireGuard protocol, type 1 is Handshake Initiation (148 bytes). - let is_handshake_init = !msg.is_empty() && msg[0] == 1; - if confirmed_r.load(Ordering::Acquire) && !is_handshake_init { - let target = match target_r.read() { - Ok(g) => *g, - Err(p) => *p.into_inner(), - }; - if let Err(err) = socket_w.send_to(&msg, target).await { - tracing::trace!(%err, %target, "failed to send WireGuard UDP datagram"); - } - } else { - let targets = match candidates_r.read() { - Ok(g) => g.clone(), - Err(p) => p.into_inner().clone(), - }; - if targets.is_empty() { - let target = match target_r.read() { - Ok(g) => *g, - Err(p) => *p.into_inner(), - }; - let _ = socket_w.send_to(&msg, target).await; - } else { - for target in targets { - let _ = socket_w.send_to(&msg, target).await; - } - } - } + tokio::spawn(async move { + let mut stop = inner.stopped.subscribe(); + tokio::select! {biased; _=stop.changed()=>{}, _=async { + if let Err(err)=receive(&inner,(network,peer),message).await {tracing::debug!(%err,"UDP pair negotiation rejected");} + }=>{}} + lock(&inner.busy).remove(&(network, peer)); + }); } - is_closed_w.store(true, Ordering::Release); - closed_notify_w.notify_waiters(); }); - - Self { - network, - peer, - target_addr: target_holder, - candidates: candidates_holder, - confirmed, - tx: tx_out, - inbound_tx: tx_in, - rx: tokio::sync::Mutex::new(rx_in), - closed_notify, - is_closed, - } - } - - /// Delivers an inbound packet received from UDP into this link's receive queue. - pub fn deliver_inbound(&self, payload: Bytes) -> bool { - if self.is_closed.load(Ordering::Acquire) { - return false; - } - self.confirmed.store(true, Ordering::Release); - self.inbound_tx.try_send(payload).is_ok() - } - - /// Marks this link as confirmed by a probe response or verified packet. - pub fn confirm(&self) { - self.confirmed.store(true, Ordering::Release); - } - - /// Whether this link has had verified two-way direct connectivity. - pub fn is_confirmed(&self) -> bool { - self.confirmed.load(Ordering::Acquire) - } - - /// Updates candidate addresses for this peer. - pub fn update_candidates(&self, addrs: impl IntoIterator) { - if let Ok(mut guard) = self.candidates.write() { - let new_candidates: Vec = addrs - .into_iter() - .filter(|a| !a.ip().is_unspecified()) - .collect(); - if !new_candidates.is_empty() { - *guard = new_candidates; - } - } - } - - /// Updates the target socket address (roaming / NAT hole punch update). - pub fn update_target(&self, new_target: SocketAddr) { - if new_target.ip().is_unspecified() { - return; - } - self.confirmed.store(true, Ordering::Release); - if let Ok(mut guard) = self.target_addr.write() - && *guard != new_target - { - tracing::debug!(peer = %self.peer.fmt_short(), old = %*guard, new = %new_target, "WireGuard endpoint roamed"); - *guard = new_target; - } - } - - /// Explicitly closes this link, notifying any pending readers or waiters. - pub fn close(&self) { - if !self.is_closed.swap(true, Ordering::AcqRel) { - self.closed_notify.notify_waiters(); - } - } - - /// Returns the current target address. - pub fn target_addr(&self) -> SocketAddr { - match self.target_addr.read() { - Ok(g) => *g, - Err(p) => *p.into_inner(), - } - } -} - -impl PacketLink for WireguardLink { - fn close(&self) { - WireguardLink::close(self); - } - fn network(&self) -> NetworkId { - self.network - } - - fn peer(&self) -> EndpointId { - self.peer - } - - fn max_datagram_size(&self) -> usize { - MAX_UDP_DATAGRAM_SIZE - } - - fn send(&self, payload: Bytes) -> Result<(), TransportError> { - if payload.len() > MAX_UDP_DATAGRAM_SIZE { - return Err(TransportError::TooLarge { - size: payload.len(), - limit: MAX_UDP_DATAGRAM_SIZE, - }); - } - if self.is_closed.load(Ordering::Acquire) { - return Err(TransportError::Closed); - } - self.tx - .try_send(payload) - .map_err(|_| TransportError::Closed) - } - - fn recv(&self) -> BoxFuture<'_, Option> { - Box::pin(async move { - if self.is_closed.load(Ordering::Acquire) { - return None; - } - tokio::select! { - biased; - _ = self.closed_notify.notified() => None, - msg = async { - let mut guard = self.rx.lock().await; - guard.recv().await - } => msg, - } + Arc::new(Self { + inner, + commands, + incoming: tokio::sync::Mutex::new(incoming), }) } - - fn closed(&self) -> BoxFuture<'_, ()> { - Box::pin(async move { - if self.is_closed.load(Ordering::Acquire) { - return; - } - self.closed_notify.notified().await; - }) - } - - fn is_closed(&self) -> bool { - self.is_closed.load(Ordering::Acquire) - } - - fn path_description(&self) -> String { - format!("wireguard via {}", self.target_addr()) - } -} - -type InboundCallback = Box; -type PendingProbes = Arc>>>; - -/// Direct UDP WireGuard packet transport. -pub struct WireguardTransport { - local_id: EndpointId, - listen_port: u16, - socket: Arc, - peer_addresses: RwLock>>, - peer_networks: RwLock>>, - links: RwLock>>, - inbound_cb: Arc>>, - pending_probes: PendingProbes, -} - -impl std::fmt::Debug for WireguardTransport { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WireguardTransport") - .field("local_id", &self.local_id.fmt_short().to_string()) - .field("listen_port", &self.listen_port) - .finish() - } -} - -impl WireguardTransport { - /// Binds a UDP socket for pure WireGuard transport. - /// - /// Attempts `requested_port` (default 51820). If that fails, falls back to an ephemeral port. - pub async fn bind( - identity: &DeviceIdentity, - requested_port: Option, - ) -> Result> { - let target_port = requested_port.unwrap_or(DEFAULT_PORT); - let bind_addr = SocketAddr::from(([0, 0, 0, 0], target_port)); - - let socket = match UdpSocket::bind(bind_addr).await { - Ok(s) => s, - Err(err) if target_port > 0 => { - tracing::warn!(%err, port = target_port, "failed to bind WireGuard UDP port, falling back to ephemeral port"); - UdpSocket::bind(SocketAddr::from(([0, 0, 0, 0], 0))).await? - } - Err(err) => return Err(Box::new(err)), - }; - - let bound_port = socket.local_addr()?.port(); - tracing::info!(port = bound_port, "WireGuard UDP transport bound"); - - Ok(Self { - local_id: identity.endpoint_id(), - listen_port: bound_port, - socket: Arc::new(socket), - peer_addresses: RwLock::new(HashMap::new()), - peer_networks: RwLock::new(HashMap::new()), - links: RwLock::new(HashMap::new()), - inbound_cb: Arc::new(Mutex::new(None)), - pending_probes: Arc::new(Mutex::new(HashMap::new())), - }) - } - - /// The local port this transport is listening on. - pub fn bound_port(&self) -> u16 { - self.listen_port - } - - /// Associates a peer with a network ID. - pub fn set_peer_network(&self, peer: EndpointId, network: NetworkId) { - if let Ok(mut guard) = self.peer_networks.write() { - guard.entry(peer).or_default().insert(network); + /// Registers a peer after its authenticated capability has been validated. + pub fn allow_peer(&self, network: NetworkId, peer: EndpointId) { + let mut allowed = lock(&self.inner.allowed); + if !*self.inner.stopped.borrow() && allowed.len() < 128 { + allowed.insert((network, peer)); } } - - /// Records an observed UDP target address for a peer. - pub fn set_peer_addr(&self, peer: EndpointId, addr: SocketAddr) { - if !addr.is_ipv4() || addr.ip().is_unspecified() { - return; - } - if let Ok(mut guard) = self.peer_addresses.write() { - let addrs = guard.entry(peer).or_default(); - addrs.retain(|a| *a != addr); - addrs.insert(0, addr); - } - } - - /// Records multiple observed UDP target addresses for a peer. - pub fn set_peer_addrs(&self, peer: EndpointId, addrs: impl IntoIterator) { - if let Ok(mut guard) = self.peer_addresses.write() { - let entry = guard.entry(peer).or_default(); - for addr in addrs { - if addr.is_ipv4() && !addr.ip().is_unspecified() && !entry.contains(&addr) { - entry.push(addr); - } - } - } - } - - /// Returns candidate socket addresses known for a peer. - pub fn peer_addrs(&self, peer: EndpointId) -> Vec { - let guard = match self.peer_addresses.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.get(&peer).cloned().unwrap_or_default() - } - - /// Explicitly closes any active direct link for the given network and peer. - pub fn close_link(&self, network: NetworkId, peer: EndpointId) { - let link = { - let guard = match self.links.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard.get(&(network, peer)).cloned() - }; - if let Some(link) = link { + /// Retires a pair when its network membership session goes away. + pub fn forget_peer(&self, network: NetworkId, peer: EndpointId) { + let pair = (network, peer); + lock(&self.inner.allowed).remove(&pair); + lock(&self.inner.pending).remove(&pair); + lock(&self.inner.port_by_pair).remove(&pair); + if let Some(link) = lock(&self.inner.links).remove(&pair) { link.close(); } } - - /// Notifies the transport that a peer's capability was announced. - /// - /// When `we_accept` is true, sends a hole-punch probe to candidate addresses - /// to open stateful NAT firewalls, but does NOT create an unverified direct link - /// in the hub before packets actually flow. - pub fn notify_peer_capability( - &self, - network: NetworkId, - peer: EndpointId, - addrs: &[SocketAddr], - we_accept: bool, - ) { - self.set_peer_network(peer, network); - let ipv4_addrs: Vec = addrs.iter().copied().filter(|a| a.is_ipv4()).collect(); - self.set_peer_addrs(peer, ipv4_addrs.iter().copied()); - - // Update candidate addresses on any existing link for this peer - if let Ok(guard) = self.links.read() - && let Some(link) = guard.get(&(network, peer)) - { - link.update_candidates(ipv4_addrs.iter().copied()); - } - - // Send hole punching probe if we accept and have candidate addresses. - if we_accept && !ipv4_addrs.is_empty() { - let socket = Arc::clone(&self.socket); - let local_id = self.local_id; - tokio::spawn(async move { - let cookie: u64 = rand::random(); - let probe = encode_probe(PROBE_TYPE_PING, local_id, peer, network, cookie); - for addr in ipv4_addrs { - let _ = socket.send_to(&probe, addr).await; - } - }); + /// Stops all pair engines in this network. + pub fn stop_network(&self, network: NetworkId) { + let peers: Vec<_> = lock(&self.inner.allowed) + .iter() + .filter(|(n, _)| *n == network) + .map(|(_, p)| *p) + .collect(); + for peer in peers { + self.forget_peer(network, peer); } } - - /// Runs the receive loop for incoming UDP datagrams. - pub async fn accept_loop(&self, on_inbound: F) - where - F: Fn(InboundLink) + Send + Sync + 'static, - { - { - let mut guard = match self.inbound_cb.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - *guard = Some(Box::new(on_inbound)); + /// Stops all sockets and coordinator work without waiting for remote peers. + pub fn shutdown(&self) { + self.inner.stopped.send_replace(true); + lock(&self.inner.allowed).clear(); + lock(&self.inner.pending).clear(); + for (_, link) in lock(&self.inner.links).drain() { + link.close(); } - - let socket = Arc::clone(&self.socket); - let mut buf = [0u8; MAX_UDP_DATAGRAM_SIZE]; - - while let Ok((len, src_addr)) = socket.recv_from(&mut buf).await { - if src_addr.ip().is_unspecified() { - continue; - } - - // 1. Direct UDP probe packet handling - if len >= PROBE_LEN - && &buf[..19] == PROBE_MAGIC - && let Some(probe) = decode_probe(&buf[..len]) - { - if probe.receiver != self.local_id { - continue; - } - - if probe.msg_type == PROBE_TYPE_PING { - let is_known = { - let guard = match self.peer_networks.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard - .get(&probe.sender) - .is_some_and(|nets| nets.contains(&probe.network)) - }; - if is_known { - let pong = encode_probe( - PROBE_TYPE_PONG, - self.local_id, - probe.sender, - probe.network, - probe.cookie, - ); - let _ = socket.send_to(&pong, src_addr).await; - - self.set_peer_addr(probe.sender, src_addr); - - let link = { - let mut guard = match self.links.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(existing) = guard.get(&(probe.network, probe.sender)) - && !existing.is_closed() - { - existing.update_target(src_addr); - existing.confirm(); - None - } else { - let candidates = self.peer_addrs(probe.sender); - let link = Arc::new(WireguardLink::new( - probe.network, - probe.sender, - src_addr, - candidates, - Arc::clone(&self.socket), - )); - link.confirm(); - guard.insert((probe.network, probe.sender), Arc::clone(&link)); - Some(link) - } - }; - - if let Some(link) = link { - let cb_guard = match self.inbound_cb.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(ref cb) = *cb_guard { - cb(InboundLink { - network: probe.network, - peer: probe.sender, - protocol: crate::announcement::WG_PROTOCOL.to_string(), - link, - }); - } - } - } - continue; - } else if probe.msg_type == PROBE_TYPE_PONG { - self.set_peer_addr(probe.sender, src_addr); - let waiter = { - let mut probes = match self.pending_probes.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - probes.remove(&(probe.network, probe.sender, probe.cookie)) - }; - if let Some(tx) = waiter { - let _ = tx.send(src_addr); - } - continue; - } - } - - // 2. Data / handshake packet handling - let datagram = Bytes::copy_from_slice(&buf[..len]); - - // Identify which peer this packet comes from - let matched_peer = { - let links_guard = match self.links.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - links_guard - .values() - .find(|link| link.target_addr() == src_addr) - .map(|link| link.peer()) - .or_else(|| { - let addrs_guard = match self.peer_addresses.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - addrs_guard - .iter() - .find(|(_, addrs)| addrs.contains(&src_addr)) - .map(|(peer, _)| *peer) - .or_else(|| { - links_guard - .values() - .find(|link| link.target_addr().ip() == src_addr.ip()) - .map(|link| link.peer()) - .or_else(|| { - addrs_guard - .iter() - .find(|(_, addrs)| { - addrs.iter().any(|a| a.ip() == src_addr.ip()) - }) - .map(|(peer, _)| *peer) - }) - }) - }) - }; - - let Some(peer) = matched_peer else { - continue; - }; - - self.set_peer_addr(peer, src_addr); - - // Deliver to ALL active links for this peer (across all networks this peer shares with us) - let peer_links: Vec> = { - let guard = match self.links.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard - .values() - .filter(|l| l.peer() == peer && !l.is_closed()) - .cloned() - .collect() - }; - - if !peer_links.is_empty() { - for link in peer_links { - link.update_target(src_addr); - link.deliver_inbound(datagram.clone()); - } - continue; - } - - // If no active link exists for this peer yet, create inbound links for all networks known for this peer - let networks: Vec = { - let guard = match self.peer_networks.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - guard - .get(&peer) - .map(|s| s.iter().copied().collect()) - .unwrap_or_default() - }; - - for network in networks { - let link = { - let mut guard = match self.links.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(existing) = guard.get(&(network, peer)) - && !existing.is_closed() - { - existing.update_target(src_addr); - existing.deliver_inbound(datagram.clone()); - continue; - } - let candidates = self.peer_addrs(peer); - let link = Arc::new(WireguardLink::new( - network, - peer, - src_addr, - candidates, - Arc::clone(&self.socket), - )); - link.update_target(src_addr); - link.deliver_inbound(datagram.clone()); - guard.insert((network, peer), Arc::clone(&link)); - link - }; - - let cb_guard = match self.inbound_cb.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(ref cb) = *cb_guard { - cb(InboundLink { - network, - peer, - protocol: crate::announcement::WG_PROTOCOL.to_string(), - link, - }); - } - } + lock(&self.inner.port_by_pair).clear(); + } + /// Receives a successfully negotiated and authenticated inbound pair link. + pub async fn next_inbound(&self) -> Option { + let mut stopped = self.inner.stopped.subscribe(); + if *stopped.borrow() { + return None; } + let mut incoming = self.incoming.lock().await; + tokio::select! {biased; _=stopped.changed()=>None, link=incoming.recv()=>link} + } + /// Takes a public snapshot of this network's pair attempts. + pub fn overview(&self, network: NetworkId) -> Vec { + let links = lock(&self.inner.links); + let ports = lock(&self.inner.port_by_pair); + links + .iter() + .filter(|((n, _), _)| *n == network) + .map(|((_, peer), link)| PairOverview { + peer: *peer, + attempt: link.attempt, + local_port: ports.get(&(network, *peer)).copied(), + path: link.path_description(), + closed: link.is_closed(), + delivery_checks: link.delivery_checks(), + }) + .collect() } } - -impl PacketTransport for WireguardTransport { +impl PacketTransport for AwgTransport { fn name(&self) -> &str { - "wireguard" + self.inner.protocol } - fn as_any(&self) -> &dyn std::any::Any { self } - + fn attach_control(&self, control: Arc) { + let _ = self.inner.control.set(control); + } + fn on_control( + &self, + network: NetworkId, + peer: EndpointId, + protocol: &str, + payload: &[u8], + ) -> Result<(), TransportError> { + if protocol != self.inner.protocol || !lock(&self.inner.allowed).contains(&(network, peer)) + { + return Err(TransportError::Declined(protocol.into())); + } + if payload.len() > 4096 { + return Err(error("UDP negotiation too large")); + } + let message: Negotiation = + postcard::from_bytes(payload).map_err(|_| error("malformed AWG negotiation"))?; + match &message { + Negotiation::Offer { + version, + profile, + key, + addresses, + } => { + if *version != self.wire_version() || peer.as_bytes() >= self.inner.local.as_bytes() + { + return Err(error("invalid AWG offer role/version")); + } + profile.validate().map_err(error)?; + if profile.is_masked() != (self.inner.protocol == AWG_PROTOCOL) { + return Err(error( + "UDP carrier profile does not match selected protocol", + )); + } + validate_endpoint(key, addresses)?; + } + Negotiation::Answer { key, addresses, .. } => validate_endpoint(key, addresses)?, + } + self.commands + .try_send((network, peer, message)) + .map_err(|_| error("UDP negotiation queue full")) + } fn open<'a>( &'a self, network: NetworkId, peer: EndpointId, - _protocol: &'a str, + protocol: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { - let mut addrs = self.peer_addrs(peer); - if addrs.is_empty() { - return Err(TransportError::Unreachable(format!( - "no candidate UDP address known for peer {}", - peer.fmt_short() - ))); - } - crate::plugin::prioritize_local_subnet(&mut addrs); - - // If an existing confirmed link is alive, reuse it + let pair = (network, peer); + if protocol != self.inner.protocol + || self.inner.local.as_bytes() >= peer.as_bytes() + || !lock(&self.inner.allowed).contains(&pair) { - let guard = match self.links.read() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(existing) = guard.get(&(network, peer)) - && !existing.is_closed() - && existing.is_confirmed() - { - existing.update_candidates(addrs); - return Ok(Arc::clone(existing) as SharedLink); - } + return Err(TransportError::Declined(protocol.into())); } - - // Probe candidate addresses with PING - let cookie: u64 = rand::random(); - let (tx, rx) = oneshot::channel::(); - { - let mut probes = match self.pending_probes.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - probes.insert((network, peer, cookie), tx); + let control = self + .inner + .control + .get() + .ok_or_else(|| error("UDP control plane unavailable"))?; + let socket = bind_pair(&self.inner).await?; + let profile = if self.inner.protocol == AWG_PROTOCOL { + PairProfile::generate() + } else { + PairProfile::wireguard() + }; + let attempt = profile.id(); + let key = StaticSecret::from(rand::random::<[u8; 32]>()); + let public = PublicKey::from(&key).to_bytes(); + let message = Negotiation::Offer { + version: self.wire_version(), + profile: profile.clone(), + key: public, + addresses: addresses(&self.inner, &socket).await?, + }; + let (reply, rx) = oneshot::channel(); + if lock(&self.inner.pending).len() >= 128 { + return Err(error("too many AWG pair negotiations")); } - - let probe_ping = encode_probe(PROBE_TYPE_PING, self.local_id, peer, network, cookie); - let socket = Arc::clone(&self.socket); - let candidates = addrs.clone(); - - let probe_task = tokio::spawn(async move { - for i in 0..3 { - for target in &candidates { - let _ = socket.send_to(&probe_ping, *target).await; - } - if i < 2 { - tokio::time::sleep(std::time::Duration::from_millis(350)).await; - } - } - }); - - let confirmed_addr = - match tokio::time::timeout(std::time::Duration::from_millis(1200), rx).await { - Ok(Ok(addr)) => { - probe_task.abort(); - addr - } - _ => { - probe_task.abort(); - let mut probes = match self.pending_probes.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - probes.remove(&(network, peer, cookie)); - return Err(TransportError::Unreachable(format!( - "direct UDP probe to peer {} timed out", - peer.fmt_short() - ))); - } - }; - - let link = { - let mut guard = match self.links.write() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - if let Some(existing) = guard.get(&(network, peer)) - && !existing.is_closed() - { - existing.update_target(confirmed_addr); - existing.update_candidates(addrs); - existing.confirm(); - return Ok(Arc::clone(existing) as SharedLink); - } - let link = Arc::new(WireguardLink::new( + lock(&self.inner.pending).insert( + pair, + Pending { + profile, + key, + socket, + reply, + }, + ); + // A cancellation guard also retires partially started attempts. + let mut guard = AttemptGuard { + inner: Arc::downgrade(&self.inner), + pair, + attempt, + completed: false, + }; + let bytes = + postcard::to_stdvec(&message).map_err(|_| error("UDP offer encoding failed"))?; + control + .send( network, peer, - confirmed_addr, - addrs, - Arc::clone(&self.socket), - )); - link.confirm(); - guard.insert((network, peer), Arc::clone(&link)); - link - }; - - Ok(link as SharedLink) + self.inner.protocol, + TransportPayload::new(bytes), + ) + .await?; + let result = tokio::time::timeout(Duration::from_secs(15), rx) + .await + .map_err(|_| error("UDP negotiation/handshake timed out"))? + .map_err(|_| error("UDP attempt cancelled"))?; + guard.completed = result.is_ok(); + result }) } } +struct AttemptGuard { + inner: Weak, + pair: Pair, + attempt: [u8; 16], + completed: bool, +} +impl Drop for AttemptGuard { + fn drop(&mut self) { + if let Some(inner) = self.inner.upgrade() { + let mut pending = lock(&inner.pending); + if pending + .get(&self.pair) + .is_some_and(|p| p.profile.id() == self.attempt) + { + pending.remove(&self.pair); + } + drop(pending); + if !self.completed + && let Some(link) = lock(&inner.links).get(&self.pair) + && link.attempt == self.attempt + { + link.close(); + } + } + } +} +async fn receive( + inner: &Arc, + pair: Pair, + message: Negotiation, +) -> Result<(), TransportError> { + if !lock(&inner.allowed).contains(&pair) { + return Err(error("UDP peer no longer authenticated")); + } + match message { + Negotiation::Offer { + profile, + key, + addresses: candidates, + .. + } => { + if let Some(link) = lock(&inner.links).get(&pair) { + if link.attempt == profile.id() { + return Err(error("duplicate AWG offer")); + } + link.close(); + } + let socket = bind_pair(inner).await?; + let local_port = socket + .local_addr() + .map_err(|_| error("UDP socket address unavailable"))? + .port(); + let local_key = StaticSecret::from(rand::random::<[u8; 32]>()); + let public = PublicKey::from(&local_key).to_bytes(); + let answer = Negotiation::Answer { + attempt: profile.id(), + key: public, + addresses: addresses(inner, &socket).await?, + }; + let tunn = Tunn::new( + local_key, + PublicKey::from(key), + Some(profile.psk), + Some(1), + rand::random(), + None, + ); + + let link = session::start( + pair, + socket, + tunn, + profile, + candidates, + inner.protocol, + inner.stopped.subscribe(), + ); + if !lock(&inner.allowed).contains(&pair) { + link.close(); + return Err(error("UDP membership ended")); + } + lock(&inner.links).insert(pair, link.clone()); + lock(&inner.port_by_pair).insert(pair, local_port); + let control = inner + .control + .get() + .ok_or_else(|| error("UDP control unavailable"))?; + if let Err(err) = control + .send( + pair.0, + pair.1, + inner.protocol, + TransportPayload::new( + postcard::to_stdvec(&answer) + .map_err(|_| error("UDP answer encoding failed"))?, + ), + ) + .await + { + link.close(); + return Err(err); + } + publish_when_up(Arc::downgrade(inner), pair, link, None); + } + Negotiation::Answer { + attempt, + key, + addresses: candidates, + } => { + if !lock(&inner.pending) + .get(&pair) + .is_some_and(|p| p.profile.id() == attempt) + { + return Err(error("stale AWG answer")); + } + let pending = lock(&inner.pending) + .remove(&pair) + .ok_or_else(|| error("UDP attempt cancelled"))?; + let local_port = pending + .socket + .local_addr() + .map_err(|_| error("UDP socket address unavailable"))? + .port(); + let tunn = Tunn::new( + pending.key, + PublicKey::from(key), + Some(pending.profile.psk), + Some(1), + rand::random(), + None, + ); + + let link = session::start( + pair, + pending.socket, + tunn, + pending.profile, + candidates, + inner.protocol, + inner.stopped.subscribe(), + ); + if !lock(&inner.allowed).contains(&pair) { + link.close(); + return Err(error("UDP membership ended")); + } + lock(&inner.links).insert(pair, link.clone()); + lock(&inner.port_by_pair).insert(pair, local_port); + publish_when_up(Arc::downgrade(inner), pair, link, Some(pending.reply)); + } + } + Ok(()) +} +fn publish_when_up( + inner: Weak, + pair: Pair, + link: Arc, + reply: Option>>, +) { + tokio::spawn(async move { + let mut reply = reply; + let result = tokio::time::timeout(Duration::from_secs(14), async { + loop { + if link.is_closed() || reply.as_ref().is_some_and(oneshot::Sender::is_closed) { + return Err(error("UDP pair closed during handshake")); + } + if link.has_authenticated_path() { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await; + if !matches!(result, Ok(Ok(()))) { + link.close(); + if let Some(reply) = reply.take() { + let _ = reply.send(Err(error("UDP authenticated data path timed out"))); + } + return; + } + let Some(inner) = inner.upgrade() else { + link.close(); + return; + }; + if !lock(&inner.allowed).contains(&pair) { + link.close(); + return; + } + if link.is_closed() + || !lock(&inner.links) + .get(&pair) + .is_some_and(|current| current.attempt == link.attempt) + { + link.close(); + return; + } + let shared: SharedLink = link.clone(); + if let Some(reply) = reply { + if reply.send(Ok(shared)).is_err() { + link.close(); + } + } else if inner + .inbound + .try_send(InboundLink { + network: pair.0, + peer: pair.1, + protocol: inner.protocol.into(), + link: shared, + }) + .is_err() + { + link.close(); + } + }); +} +impl Drop for AwgTransport { + fn drop(&mut self) { + self.shutdown(); + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[tokio::test] + async fn concurrent_admission_is_bounded_across_networks_and_retirement_releases_slots() { + let identity = tsunagi::identity::DeviceIdentity::generate(); + let transport = AwgTransport::new(identity.endpoint_id(), Ipv4Addr::LOCALHOST); + let peer = tsunagi::identity::DeviceIdentity::generate().endpoint_id(); + std::thread::scope(|scope| { + for thread in 0..8u8 { + let transport = &transport; + scope.spawn(move || { + for item in 0..32u8 { + let mut id = [0; 32]; + id[0] = thread; + id[1] = item; + transport.allow_peer(NetworkId::from_bytes(id), peer); + } + }); + } + }); + assert_eq!(lock(&transport.inner.allowed).len(), 128); + let network = lock(&transport.inner.allowed).iter().next().unwrap().0; + transport.stop_network(network); + let new = NetworkId::from_bytes([255; 32]); + transport.allow_peer(new, peer); + assert!(lock(&transport.inner.allowed).contains(&(new, peer))); + assert_eq!(lock(&transport.inner.allowed).len(), 128); + transport.shutdown(); + assert!(lock(&transport.inner.allowed).is_empty()); + } + #[tokio::test] + async fn private_negotiation_bounds_roles_and_membership_before_starting_engines() { + let mut ids = [ + tsunagi::identity::DeviceIdentity::generate().endpoint_id(), + tsunagi::identity::DeviceIdentity::generate().endpoint_id(), + ]; + ids.sort_by_key(|id| *id.as_bytes()); + let peer = ids[0]; + let local = ids[1]; + let network = NetworkId::from_bytes([7; 32]); + let transport = AwgTransport::new(local, Ipv4Addr::LOCALHOST); + let offer = |version, key, addresses| Negotiation::Offer { + version, + profile: PairProfile::generate(), + key, + addresses, + }; + let address = SocketAddr::from((Ipv4Addr::LOCALHOST, 12345)); + let encode = |message: &Negotiation| postcard::to_stdvec(message).unwrap(); + let valid = encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![address])); + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &valid) + .is_err() + ); + transport.allow_peer(network, peer); + assert!( + transport + .on_control(network, peer, "wrong-protocol", &valid) + .is_err() + ); + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &[0; 4097]) + .is_err() + ); + for bytes in [ + encode(&offer(99, [1; 32], vec![address])), + encode(&offer(AWG_WIRE_VERSION, [0; 32], vec![address])), + encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![])), + encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![address; 33])), + ] { + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &bytes) + .is_err() + ); + } + for len in 0..valid.len() { + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &valid[..len]) + .is_err() + ); + } + let wrong_role = AwgTransport::new(peer, Ipv4Addr::LOCALHOST); + wrong_role.allow_peer(network, local); + assert!( + wrong_role + .on_control(network, local, AWG_PROTOCOL, &valid) + .is_err() + ); + transport.forget_peer(network, peer); + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &valid) + .is_err() + ); + assert!(transport.overview(network).is_empty()); + transport.shutdown(); + wrong_role.shutdown(); + } +} diff --git a/crates/tsunagi-wireguard/tests/announcement_tests.rs b/crates/tsunagi-wireguard/tests/announcement_tests.rs deleted file mode 100644 index 945c044..0000000 --- a/crates/tsunagi-wireguard/tests/announcement_tests.rs +++ /dev/null @@ -1,86 +0,0 @@ -#![allow(missing_docs)] -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::net::IpAddr; - -use tsunagi::identity::NetworkId; -use tsunagi_wireguard::{ANNOUNCEMENT_VERSION, WgAnnouncement, WgPublicKey, WgSecretKey}; - -#[test] -fn announcement_round_trip() { - let network = NetworkId::from_bytes([42u8; 32]); - let key = WgSecretKey::generate().public(); - let local = WgSecretKey::generate().public(); - - let addrs: Vec = vec![ - "192.168.1.100".parse().unwrap(), - "10.0.0.5".parse().unwrap(), - ]; - - let announcement = WgAnnouncement::new(network, &key, 51820, addrs.clone()); - let encoded = announcement.encode().expect("encode"); - - let validated = WgAnnouncement::decode_and_validate(&encoded, network, &local).expect("decode"); - - assert_eq!(validated.public_key, key); - assert_eq!(validated.port, 51820); - assert_eq!(validated.addrs, addrs); -} - -#[test] -fn announcement_rejects_zero_key() { - let network = NetworkId::from_bytes([42u8; 32]); - let local = WgSecretKey::generate().public(); - let zero = WgPublicKey::from_bytes([0u8; 32]); - - let announcement = WgAnnouncement::new(network, &zero, 51820, vec![]); - let encoded = announcement.encode().unwrap(); - - let err = WgAnnouncement::decode_and_validate(&encoded, network, &local).unwrap_err(); - assert!(err.to_string().contains("all zeroes")); -} - -#[test] -fn announcement_rejects_own_key() { - let network = NetworkId::from_bytes([42u8; 32]); - let local = WgSecretKey::generate().public(); - - let announcement = WgAnnouncement::new(network, &local, 51820, vec![]); - let encoded = announcement.encode().unwrap(); - - let err = WgAnnouncement::decode_and_validate(&encoded, network, &local).unwrap_err(); - assert!(err.to_string().contains("own WireGuard key")); -} - -#[test] -fn announcement_rejects_wrong_network() { - let network_a = NetworkId::from_bytes([1u8; 32]); - let network_b = NetworkId::from_bytes([2u8; 32]); - let key = WgSecretKey::generate().public(); - let local = WgSecretKey::generate().public(); - - let announcement = WgAnnouncement::new(network_a, &key, 51820, vec![]); - let encoded = announcement.encode().unwrap(); - - let err = WgAnnouncement::decode_and_validate(&encoded, network_b, &local).unwrap_err(); - assert!(err.to_string().contains("different network")); -} - -#[test] -fn announcement_rejects_zero_port() { - let network = NetworkId::from_bytes([42u8; 32]); - let key = WgSecretKey::generate().public(); - let local = WgSecretKey::generate().public(); - - let announcement = WgAnnouncement { - version: ANNOUNCEMENT_VERSION, - public_key: *key.as_bytes(), - network: *network.as_bytes(), - port: 0, - addrs: vec![], - }; - let encoded = announcement.encode().unwrap(); - - let err = WgAnnouncement::decode_and_validate(&encoded, network, &local).unwrap_err(); - assert!(err.to_string().contains("cannot be 0")); -} diff --git a/crates/tsunagi-wireguard/tests/plugin_tests.rs b/crates/tsunagi-wireguard/tests/plugin_tests.rs deleted file mode 100644 index 0a5cbc5..0000000 --- a/crates/tsunagi-wireguard/tests/plugin_tests.rs +++ /dev/null @@ -1,378 +0,0 @@ -#![allow(missing_docs)] -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; - -use bytes::Bytes; -use tokio::sync::mpsc; -use tsunagi::dataplane::transport::{InboundLink, PacketTransport}; -use tsunagi::dataplane::{IpPlugin, PacketSink, PluginContext}; -use tsunagi::identity::{DeviceIdentity, NetworkId}; -use tsunagi_wireguard::{WireguardConfig, WireguardPlugin, WireguardTransport}; - -struct TestSink { - delivered: mpsc::Sender<(NetworkId, iroh::EndpointId, Bytes)>, -} - -impl PacketSink for TestSink { - fn deliver( - &self, - network: NetworkId, - from: iroh::EndpointId, - packet: Bytes, - ) -> tsunagi::BoxFuture<'_, ()> { - let delivered = self.delivered.clone(); - Box::pin(async move { - let _ = delivered.send((network, from, packet)).await; - }) - } -} - -fn sample_ipv4_packet() -> Bytes { - let mut packet = vec![0u8; 40]; - packet[0] = 0x45; // IPv4, IHL 5 - packet[2..4].copy_from_slice(&40u16.to_be_bytes()); // total len 40 - packet[8] = 64; // TTL - packet[9] = 17; // UDP - packet[12..16].copy_from_slice(&[10, 0, 0, 1]); // src 10.0.0.1 - packet[16..20].copy_from_slice(&[10, 0, 0, 2]); // dst 10.0.0.2 - Bytes::from(packet) -} - -#[tokio::test] -async fn test_wireguard_plugin_end_to_end() { - let _ = tracing_subscriber::fmt() - .with_env_filter("trace") - .with_test_writer() - .try_init(); - - let identity_a = DeviceIdentity::generate(); - let peer_a = identity_a.endpoint_id(); - - let identity_b = DeviceIdentity::generate(); - let peer_b = identity_b.endpoint_id(); - - let network = NetworkId::from_bytes([77u8; 32]); - - let tmp_a = tempfile::tempdir().expect("tmp A"); - let tmp_b = tempfile::tempdir().expect("tmp B"); - - // Transports - let transport_b = Arc::new( - WireguardTransport::bind(&identity_b, Some(0)) - .await - .expect("bind B"), - ); - let port_b = transport_b.bound_port(); - - let transport_a = Arc::new( - WireguardTransport::bind(&identity_a, Some(0)) - .await - .expect("bind A"), - ); - let port_a = transport_a.bound_port(); - - // Plugins - let cfg_a = WireguardConfig::new(tmp_a.path()); - let plugin_a = WireguardPlugin::open(cfg_a, Arc::clone(&transport_a)) - .await - .expect("open plugin A"); - - let cfg_b = WireguardConfig::new(tmp_b.path()); - let plugin_b = WireguardPlugin::open(cfg_b, Arc::clone(&transport_b)) - .await - .expect("open plugin B"); - - // Attach sinks - let (sink_a_tx, mut sink_a_rx) = mpsc::channel(10); - plugin_a.attach(PluginContext::with_sink(Arc::new(TestSink { - delivered: sink_a_tx, - }))); - - let (sink_b_tx, mut sink_b_rx) = mpsc::channel(10); - plugin_b.attach(PluginContext::with_sink(Arc::new(TestSink { - delivered: sink_b_tx, - }))); - - // Capability exchange - plugin_a.on_network_activated(network); - plugin_b.on_network_activated(network); - - let cap_a = loop { - if let Some(cap) = plugin_a.local_capability(network).expect("cap A") { - break cap; - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - let cap_b = loop { - if let Some(cap) = plugin_b.local_capability(network).expect("cap B") { - break cap; - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - plugin_a - .on_peer_capability(network, peer_b, &cap_b) - .expect("A on cap B"); - plugin_b - .on_peer_capability(network, peer_a, &cap_a) - .expect("B on cap A"); - - // Register loopback address candidates - let addr_b: SocketAddr = format!("127.0.0.1:{port_b}").parse().unwrap(); - transport_a.set_peer_addr(peer_b, addr_b); - - let addr_a: SocketAddr = format!("127.0.0.1:{port_a}").parse().unwrap(); - transport_b.set_peer_addr(peer_a, addr_a); - - // Spawn accept loop on A - let transport_a_clone = Arc::clone(&transport_a); - tokio::spawn(async move { - transport_a_clone.accept_loop(|_| {}).await; - }); - - // Spawn accept loop on B - let (inbound_tx, mut inbound_rx) = mpsc::channel::(1); - let transport_b_clone = Arc::clone(&transport_b); - tokio::spawn(async move { - transport_b_clone - .accept_loop(move |inbound| { - let _ = inbound_tx.try_send(inbound); - }) - .await; - }); - - // Open link from A to B - let link_a = tokio::time::timeout( - Duration::from_secs(5), - transport_a.open(network, peer_b, "wg"), - ) - .await - .expect("open should not timeout") - .expect("open should succeed"); - - plugin_a.on_peer_link(network, peer_b, Arc::clone(&link_a)); - - // Wait for inbound link on B and hand to plugin B - let inbound_b = tokio::time::timeout(Duration::from_secs(5), inbound_rx.recv()) - .await - .expect("accept should not timeout") - .expect("inbound link should arrive"); - plugin_b.on_peer_link(network, peer_a, inbound_b.link); - - // Wait briefly for WireGuard handshake to complete - let mut up = false; - for _ in 0..50 { - if let Some(overview_a) = plugin_a.overview(network) - && overview_a.peers.iter().any(|p| p.is_up()) - { - up = true; - break; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - assert!(up, "WireGuard handshake should complete between peers"); - - // Send IP packet from A to B - let packet_a_to_b = sample_ipv4_packet(); - let sent = plugin_a.carry(network, peer_b, packet_a_to_b.clone()); - assert!(sent, "plugin A carry should succeed"); - - let received_b = tokio::time::timeout(Duration::from_secs(5), sink_b_rx.recv()) - .await - .expect("recv B timeout") - .expect("recv B option"); - - assert_eq!(received_b.0, network); - assert_eq!(received_b.1, peer_a); - assert_eq!(received_b.2, packet_a_to_b); - - // Send IP packet from B to A - let packet_b_to_a = sample_ipv4_packet(); - let sent_reply = plugin_b.carry(network, peer_a, packet_b_to_a.clone()); - assert!(sent_reply, "plugin B carry should succeed"); - - let received_a = tokio::time::timeout(Duration::from_secs(5), sink_a_rx.recv()) - .await - .expect("recv A timeout") - .expect("recv A option"); - - assert_eq!(received_a.0, network); - assert_eq!(received_a.1, peer_b); - assert_eq!(received_a.2, packet_b_to_a); -} - -#[tokio::test] -async fn test_wireguard_two_networks_same_peers() { - let identity_a = DeviceIdentity::generate(); - let peer_a = identity_a.endpoint_id(); - - let identity_b = DeviceIdentity::generate(); - let peer_b = identity_b.endpoint_id(); - - let net1 = NetworkId::from_bytes([11u8; 32]); - let net2 = NetworkId::from_bytes([22u8; 32]); - - let tmp_a = tempfile::tempdir().expect("tmp A"); - let tmp_b = tempfile::tempdir().expect("tmp B"); - - let transport_b = Arc::new( - WireguardTransport::bind(&identity_b, Some(0)) - .await - .expect("bind B"), - ); - let port_b = transport_b.bound_port(); - - let transport_a = Arc::new( - WireguardTransport::bind(&identity_a, Some(0)) - .await - .expect("bind A"), - ); - let port_a = transport_a.bound_port(); - - let plugin_a = - WireguardPlugin::open(WireguardConfig::new(tmp_a.path()), Arc::clone(&transport_a)) - .await - .expect("open plugin A"); - - let plugin_b = - WireguardPlugin::open(WireguardConfig::new(tmp_b.path()), Arc::clone(&transport_b)) - .await - .expect("open plugin B"); - - let (sink_a_tx, mut sink_a_rx) = mpsc::channel(20); - plugin_a.attach(PluginContext::with_sink(Arc::new(TestSink { - delivered: sink_a_tx, - }))); - - let (sink_b_tx, mut sink_b_rx) = mpsc::channel(20); - plugin_b.attach(PluginContext::with_sink(Arc::new(TestSink { - delivered: sink_b_tx, - }))); - - // Activate both networks - for net in [net1, net2] { - plugin_a.on_network_activated(net); - plugin_b.on_network_activated(net); - } - - // Exchange capabilities for both networks - for net in [net1, net2] { - let cap_a = loop { - if let Some(cap) = plugin_a.local_capability(net).expect("cap A") { - break cap; - } - tokio::time::sleep(Duration::from_millis(20)).await; - }; - let cap_b = loop { - if let Some(cap) = plugin_b.local_capability(net).expect("cap B") { - break cap; - } - tokio::time::sleep(Duration::from_millis(20)).await; - }; - plugin_a - .on_peer_capability(net, peer_b, &cap_b) - .expect("A on cap B"); - plugin_b - .on_peer_capability(net, peer_a, &cap_a) - .expect("B on cap A"); - } - - let addr_b: SocketAddr = format!("127.0.0.1:{port_b}").parse().unwrap(); - transport_a.set_peer_addr(peer_b, addr_b); - - let addr_a: SocketAddr = format!("127.0.0.1:{port_a}").parse().unwrap(); - transport_b.set_peer_addr(peer_a, addr_a); - - // Accept loops - let transport_a_clone = Arc::clone(&transport_a); - tokio::spawn(async move { - transport_a_clone.accept_loop(|_| {}).await; - }); - - let (inbound_tx, mut inbound_rx) = mpsc::channel::(10); - let transport_b_clone = Arc::clone(&transport_b); - tokio::spawn(async move { - transport_b_clone - .accept_loop(move |inbound| { - let _ = inbound_tx.try_send(inbound); - }) - .await; - }); - - // Open links for both networks from A to B - for net in [net1, net2] { - let link_a = transport_a - .open(net, peer_b, "wg") - .await - .expect("open A link"); - plugin_a.on_peer_link(net, peer_b, link_a); - } - - // Deliver inbound links to B - for _ in 0..2 { - let inbound = tokio::time::timeout(Duration::from_secs(5), inbound_rx.recv()) - .await - .expect("inbound timeout") - .expect("inbound option"); - plugin_b.on_peer_link(inbound.network, peer_a, inbound.link); - } - - // Wait for both handshakes to complete on both networks - for net in [net1, net2] { - let mut up = false; - for _ in 0..50 { - if let Some(overview) = plugin_a.overview(net) - && overview.peers.iter().any(|p| p.is_up()) - { - up = true; - break; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - assert!(up, "handshake on {net:?} should complete"); - } - - // Send on net1 - let pkt1 = sample_ipv4_packet(); - assert!(plugin_a.carry(net1, peer_b, pkt1.clone())); - let recv1 = tokio::time::timeout(Duration::from_secs(5), sink_b_rx.recv()) - .await - .expect("recv1 timeout") - .expect("recv1 option"); - assert_eq!(recv1.0, net1); - assert_eq!(recv1.2, pkt1); - - // Send on net2 from A to B - let mut pkt2_bytes = sample_ipv4_packet().to_vec(); - pkt2_bytes[15] = 99; // different byte - let pkt2 = Bytes::from(pkt2_bytes); - assert!(plugin_a.carry(net2, peer_b, pkt2.clone())); - let recv2 = tokio::time::timeout(Duration::from_secs(5), sink_b_rx.recv()) - .await - .expect("recv2 timeout") - .expect("recv2 option"); - assert_eq!(recv2.0, net2); - assert_eq!(recv2.2, pkt2); - - // Send reply on net1 from B to A - assert!(plugin_b.carry(net1, peer_a, pkt1.clone())); - let reply1 = tokio::time::timeout(Duration::from_secs(5), sink_a_rx.recv()) - .await - .expect("reply1 timeout") - .expect("reply1 option"); - assert_eq!(reply1.0, net1); - assert_eq!(reply1.2, pkt1); - - // Send reply on net2 from B to A - assert!(plugin_b.carry(net2, peer_a, pkt2.clone())); - let reply2 = tokio::time::timeout(Duration::from_secs(5), sink_a_rx.recv()) - .await - .expect("reply2 timeout") - .expect("reply2 option"); - assert_eq!(reply2.0, net2); - assert_eq!(reply2.2, pkt2); -} diff --git a/crates/tsunagi-wireguard/tests/transport_tests.rs b/crates/tsunagi-wireguard/tests/transport_tests.rs index d1cf929..18993ae 100644 --- a/crates/tsunagi-wireguard/tests/transport_tests.rs +++ b/crates/tsunagi-wireguard/tests/transport_tests.rs @@ -1,220 +1,94 @@ -#![allow(missing_docs)] +//! Real control authentication and independently encrypted ordinary WG carriers. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; - use bytes::Bytes; -use tokio::sync::mpsc; -use tsunagi::dataplane::transport::{InboundLink, PacketTransport}; -use tsunagi::identity::{DeviceIdentity, NetworkId}; -use tsunagi_wireguard::WireguardTransport; +use std::{net::Ipv4Addr, time::Duration}; +use tempfile::TempDir; +use tsunagi::{ + Agent, + dataplane::transport::{PacketTransport, TransportError}, + discovery::SharedMemoryDiscovery, + identity::DeviceIdentity, + testing::{config_with, network, wait_for_peers}, +}; +use tsunagi_wireguard::{UdpTransport, profile::PairProfile}; #[tokio::test] -async fn test_wireguard_transport_loopback() { - let identity_a = DeviceIdentity::generate(); - let peer_a = identity_a.endpoint_id(); - - let identity_b = DeviceIdentity::generate(); - let peer_b = identity_b.endpoint_id(); - - let network = NetworkId::from_bytes([42u8; 32]); - - // Bind transport B on port 0 (ephemeral) - let transport_b = Arc::new( - WireguardTransport::bind(&identity_b, Some(0)) +async fn ordinary_wireguard_carries_opaque_frames_after_real_membership_authentication() { + let discovery = SharedMemoryDiscovery::new(); + let mut nodes = Vec::new(); + for _ in 0..2 { + let root = TempDir::new().unwrap(); + let config = config_with(root.path(), &discovery); + let identity = tsunagi::storage::StateStore::open(config.paths.state_db()) + .unwrap() + .load_or_create_device_identity() + .unwrap(); + let transport = UdpTransport::wireguard(identity.endpoint_id(), Ipv4Addr::LOCALHOST, None); + let agent = Agent::spawn(config.with_custom_transport("wg", transport.clone())) .await - .expect("bind server should succeed"), - ); - let port_b = transport_b.bound_port(); - assert!(port_b > 0); - - // Bind transport A on port 0 (ephemeral) - let transport_a = Arc::new( - WireguardTransport::bind(&identity_a, Some(0)) - .await - .expect("bind client should succeed"), - ); - let port_a = transport_a.bound_port(); - assert!(port_a > 0); - - // Register B's address and network on transport A - let addr_b: SocketAddr = format!("127.0.0.1:{port_b}").parse().unwrap(); - transport_a.set_peer_addr(peer_b, addr_b); - transport_a.set_peer_network(peer_b, network); - - // Register A's address and network on transport B - let addr_a: SocketAddr = format!("127.0.0.1:{port_a}").parse().unwrap(); - transport_b.set_peer_addr(peer_a, addr_a); - transport_b.set_peer_network(peer_a, network); - - // Spawn accept loop on A - let transport_a_clone = Arc::clone(&transport_a); - tokio::spawn(async move { - transport_a_clone.accept_loop(|_| {}).await; - }); - - // Spawn accept loop on B - let (inbound_tx, mut inbound_rx) = mpsc::channel::(1); - let transport_b_clone = Arc::clone(&transport_b); - tokio::spawn(async move { - transport_b_clone - .accept_loop(move |inbound| { - let _ = inbound_tx.try_send(inbound); - }) - .await; - }); - - // Open link from A to B - let link_a = tokio::time::timeout( - Duration::from_secs(5), - transport_a.open(network, peer_b, "wg"), - ) - .await - .expect("open should not timeout") - .expect("open should succeed"); - - // Send from A to B to trigger inbound link discovery on B - let data_a_to_b = Bytes::from_static(b"wireguard packet from A to B"); - link_a.send(data_a_to_b.clone()).expect("send A->B"); - - // Receive inbound link on B - let inbound_b = tokio::time::timeout(Duration::from_secs(5), inbound_rx.recv()) + .unwrap(); + nodes.push((root, agent, transport)); + } + nodes.sort_by_key(|(_, agent, _)| *agent.endpoint_id().as_bytes()); + let (name, secret) = network("outer-wireguard"); + let id = nodes[0].1.join_network(&name, &secret).await.unwrap(); + nodes[1].1.join_network(&name, &secret).await.unwrap(); + for (index, (_, agent, transport)) in nodes.iter().enumerate() { + wait_for_peers(agent, id, 1).await; + transport.allow_peer(id, nodes[1 - index].1.endpoint_id()); + } + let outgoing = nodes[0] + .2 + .open(id, nodes[1].1.endpoint_id(), "wg") .await - .expect("accept should not timeout") - .expect("inbound link should arrive"); - - assert_eq!(inbound_b.network, network); - assert_eq!(inbound_b.peer, peer_a); - assert_eq!(inbound_b.protocol, "wg"); - let link_b = inbound_b.link; - - // Check packet received on B - let received_on_b = tokio::time::timeout(Duration::from_secs(2), link_b.recv()) + .unwrap(); + let incoming = tokio::time::timeout(Duration::from_secs(5), nodes[1].2.next_inbound()) .await - .expect("recv B timeout") - .expect("recv B option"); - assert_eq!(received_on_b, data_a_to_b); - - // Send from B to A - let data_b_to_a = Bytes::from_static(b"wireguard packet from B to A"); - link_b.send(data_b_to_a.clone()).expect("send B->A"); - let received_on_a = tokio::time::timeout(Duration::from_secs(2), link_a.recv()) - .await - .expect("recv A timeout") - .expect("recv A option"); - assert_eq!(received_on_a, data_b_to_a); - - // Path descriptions - assert!(link_a.path_description().contains("wireguard via")); - assert!(link_b.path_description().contains("wireguard via")); -} - -#[tokio::test] -async fn test_wireguard_transport_unreachable_probe_times_out() { - let identity_a = DeviceIdentity::generate(); - let identity_b = DeviceIdentity::generate(); - let peer_b = identity_b.endpoint_id(); - let network = NetworkId::from_bytes([99u8; 32]); - - let transport_a = Arc::new( - WireguardTransport::bind(&identity_a, Some(0)) - .await - .expect("bind client should succeed"), - ); - - // Register a non-responsive address for peer B (TEST-NET-1 documentation IP which drops packets) - let unreachable_addr: SocketAddr = "192.0.2.1:51820".parse().unwrap(); - transport_a.set_peer_addr(peer_b, unreachable_addr); - transport_a.set_peer_network(peer_b, network); - - // Spawn accept loop on A - let transport_a_clone = Arc::clone(&transport_a); - tokio::spawn(async move { - transport_a_clone.accept_loop(|_| {}).await; - }); - - // Opening to unreachable candidate must fail with Unreachable rather than returning a dead direct link - let result = transport_a.open(network, peer_b, "wg").await; - match result { - Err(tsunagi::dataplane::transport::TransportError::Unreachable(msg)) => { - assert!(msg.contains("timed out")); - } - other => panic!("expected Unreachable error, got {other:?}"), + .unwrap() + .unwrap() + .link; + for (from, to) in [(&outgoing, &incoming), (&incoming, &outgoing)] { + let payload = Bytes::from(vec![0x37; 65_536]); + from.send(payload.clone()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), to.recv()) + .await + .unwrap() + .unwrap(), + payload + ); + assert!(from.path_description().starts_with("wg UDP via")); + assert!(matches!( + from.send(Bytes::from(vec![0; 65_537])), + Err(TransportError::TooLarge { .. }) + )); + } + outgoing.close(); + incoming.close(); + for (_, agent, transport) in nodes { + transport.shutdown(); + agent.shutdown().await; } } #[tokio::test] -async fn test_wireguard_link_close_and_transport_close_link() { - let identity_a = DeviceIdentity::generate(); - let peer_a = identity_a.endpoint_id(); - let identity_b = DeviceIdentity::generate(); - let peer_b = identity_b.endpoint_id(); - let network = NetworkId::from_bytes([55u8; 32]); - - let transport_b = Arc::new( - WireguardTransport::bind(&identity_b, Some(0)) - .await - .unwrap(), - ); - let port_b = transport_b.bound_port(); - - let transport_a = Arc::new( - WireguardTransport::bind(&identity_a, Some(0)) - .await - .unwrap(), - ); - let port_a = transport_a.bound_port(); - - let addr_b: SocketAddr = format!("127.0.0.1:{port_b}").parse().unwrap(); - transport_a.set_peer_addr(peer_b, addr_b); - transport_a.set_peer_network(peer_b, network); - - let addr_a: SocketAddr = format!("127.0.0.1:{port_a}").parse().unwrap(); - transport_b.set_peer_addr(peer_a, addr_a); - transport_b.set_peer_network(peer_a, network); - - let transport_a_clone = Arc::clone(&transport_a); - tokio::spawn(async move { - transport_a_clone.accept_loop(|_| {}).await; - }); - - let transport_b_clone = Arc::clone(&transport_b); - tokio::spawn(async move { - transport_b_clone.accept_loop(|_| {}).await; - }); - - let link_a = transport_a - .open(network, peer_b, "wg") - .await - .expect("open link A"); - assert!(!link_a.is_closed()); - - // Spawn a reader task waiting on recv() - let link_a_clone = Arc::clone(&link_a); - let recv_task = tokio::spawn(async move { link_a_clone.recv().await }); - - // Explicitly closing via transport.close_link() - transport_a.close_link(network, peer_b); - assert!(link_a.is_closed()); - - // Wait on closed() notification - tokio::time::timeout(Duration::from_secs(1), link_a.closed()) - .await - .expect("link.closed() should resolve promptly"); - - // recv() must have returned None - let recv_res = tokio::time::timeout(Duration::from_secs(1), recv_task) - .await - .expect("recv task should complete") - .expect("task join"); - assert_eq!(recv_res, None); - - // Send after close must return Closed error - let send_res = link_a.send(Bytes::from_static(b"after close")); - assert!(matches!( - send_res, - Err(tsunagi::dataplane::transport::TransportError::Closed) - )); +async fn an_unannounced_peer_cannot_open_or_negotiate_a_wireguard_carrier() { + let local = DeviceIdentity::generate().endpoint_id(); + let peer = DeviceIdentity::generate().endpoint_id(); + let transport = UdpTransport::wireguard(local, Ipv4Addr::LOCALHOST, None); + let id = tsunagi::NetworkId::from_bytes([7; 32]); + assert!(transport.open(id, peer, "wg").await.is_err()); + assert!(transport.on_control(id, peer, "wg", &[0; 40]).is_err()); + transport.shutdown(); +} + +#[test] +fn ordinary_profiles_preserve_the_wireguard_header_and_emit_no_masking_packets() { + let profile = PairProfile::wireguard(); + let mut message = vec![0x42; 148]; + message[..4].copy_from_slice(&[1, 0, 0, 0]); + assert_eq!(profile.encode(&message).unwrap().as_ref(), message); + assert_eq!(profile.decode(&message).unwrap(), message); + assert!(profile.signature_packets().is_empty()); + assert!(!profile.is_masked()); } diff --git a/crates/tsunagi/src/main.rs b/crates/tsunagi/src/main.rs index 230ca7a..8534998 100644 --- a/crates/tsunagi/src/main.rs +++ b/crates/tsunagi/src/main.rs @@ -18,7 +18,6 @@ mod vip_cli; use clap::{Args, Parser, Subcommand, ValueEnum}; use tsunagi::agent::Event; use tsunagi::config::{AgentConfig, StoragePaths, TransportPolicy}; -use tsunagi::dataplane::IpPlugin; use tsunagi::discovery::{ CompositeDiscovery, MainlineDiscovery, NetworkDiscovery, StaticBootstrap, }; @@ -28,14 +27,9 @@ use tsunagi::overlay::{MemoryTunFactory, TunFactory}; use tsunagi::state::Ipv4Range; use tsunagi::{Agent, NetworkId}; use tsunagi_awg::{AWG_PROTOCOL, AWG_WIRE_VERSION, AwgPlugin, AwgTransport}; -use tsunagi_tcp_tls::{ - TCP_TLS_PROTOCOL, TCP_TLS_VERSION, TcpTlsCodec, TcpTlsConfig, TcpTlsPlugin, TcpTlsTransport, -}; +use tsunagi_tcp_tls::{TCP_TLS_PROTOCOL, TcpTlsCarrier, TcpTlsConfig, TcpTlsTransport}; use tsunagi_wg_quic::{WireguardConfig, WireguardPlugin}; -use tsunagi_wireguard::{ - WG_PROTOCOL, WG_WIRE_VERSION, WireguardConfig as PureWgConfig, WireguardPlugin as PureWgPlugin, - WireguardTransport as PureWgTransport, -}; +use tsunagi_wireguard::WG_PROTOCOL; /// A small agent for private mesh networks. #[derive(Debug, Parser)] @@ -112,7 +106,8 @@ struct DnsArgs { #[derive(Debug, Args)] #[command( - after_help = "Examples:\n tsng protocols\n tsng protocols home\n tsng protocols home awg,wg-quic\n tsng protocols home none\n\nThe selection is local to this device, saved per network and applied immediately.\nBoth endpoints need a common protocol at the same wire version." + after_help = "Examples:\n tsng protocols\n tsng protocols home\n tsng protocols home awg,wg-quic\n tsng protocols home none\n\nThe selection is local to this device, saved per network and applied immediately.\nThe choice governs local incoming and outgoing data links only. +Adjacent peers need a common external transport; control and discovery are independent." )] struct ProtocolsArgs { #[command(flatten)] @@ -1460,13 +1455,20 @@ struct ProtocolSpec { options: &'static [tsunagi::dataplane::ProtocolOption], } +const WG_OPTIONS: &[tsunagi::dataplane::ProtocolOption] = &[tsunagi::dataplane::ProtocolOption { + key: "port", + value: "PORT", + help: "preferred local UDP port; additional pairs use dynamic ports", + default: Some("51820"), +}]; + /// Every protocol this build has. const PROTOCOLS: &[ProtocolSpec] = &[ ProtocolSpec { name: WG_PROTOCOL, - version: WG_WIRE_VERSION, + version: tsunagi_wireguard::WG_WIRE_VERSION, summary: "Pure WireGuard direct UDP transport", - options: PureWgPlugin::OPTIONS, + options: WG_OPTIONS, }, ProtocolSpec { name: AWG_PROTOCOL, @@ -1475,16 +1477,16 @@ const PROTOCOLS: &[ProtocolSpec] = &[ options: AwgPlugin::OPTIONS, }, ProtocolSpec { - name: tsunagi_wg_quic::WIREGUARD_PROTOCOL, - version: tsunagi_wg_quic::ANNOUNCEMENT_VERSION, - summary: "WireGuard encryption over iroh QUIC datagrams with NAT traversal and relays", - options: WireguardPlugin::OPTIONS, + name: "wg-quic", + version: 1, + summary: "iroh QUIC datagram carrier with NAT traversal and relays", + options: &[], }, ProtocolSpec { name: TCP_TLS_PROTOCOL, - version: TCP_TLS_VERSION, - summary: "Noise E2EE carried over TCP TLS 1.3", - options: TcpTlsCodec::OPTIONS, + version: 2, + summary: "TCP TLS 1.3 carrier", + options: TcpTlsConfig::OPTIONS, }, ]; @@ -1520,12 +1522,44 @@ fn parse_settings(raw: &[String]) -> Result, Box) -> Result, String> { + let mut tunnel: Vec<(String, String)> = Vec::new(); + for setting in settings + .iter() + .filter(|s| matches!(s.key.as_str(), "keepalive" | "mtu")) + { + if setting + .protocol + .as_ref() + .is_some_and(|p| !PROTOCOLS.iter().any(|s| s.name == p)) + { + return Err(format!( + "unknown protocol in legacy `{}` setting", + setting.key + )); + } + if let Some((_, previous)) = tunnel.iter().find(|(key, _)| key == &setting.key) { + if previous != &setting.value { + return Err(format!("conflicting shared `{}` settings", setting.key)); + } + } else { + tunnel.push((setting.key.clone(), setting.value.clone())); + } + } + settings.retain(|s| !matches!(s.key.as_str(), "keepalive" | "mtu")); + Ok(tunnel) +} + /// The settings meant for one protocol, refusing any that fit nowhere. fn settings_for(spec: &ProtocolSpec, settings: &[Setting]) -> Vec<(String, String)> { let mut taken = Vec::new(); for setting in settings { let aimed_here = match &setting.protocol { - Some(name) => name == spec.name, + Some(name) => { + name == spec.name && spec.options.iter().any(|option| option.key == setting.key) + } // Unqualified settings go to whichever protocol declares the // key. With one selected that is the obvious reading; with // several, write `protocol:key=value`. @@ -1575,10 +1609,7 @@ fn show_protocols() -> Result<(), Box> { /// published. struct AgentControl { agent: Agent, - pure_wg: Option>, - awg: Option>, plugin: Option>, - tcp_tls: Option>, /// The resolver, which this owns so it can be switched while running. dns: Arc>>, /// Where the setting is remembered, so it survives a restart. @@ -1631,16 +1662,7 @@ impl tsunagi::ipc::ReportSource for AgentControl { fn report(&self) -> tsunagi::BoxFuture<'_, tsunagi::ipc::StatusReport> { Box::pin(async move { let dns = self.dns_state().await; - build_report( - &self.agent, - self.pure_wg.as_deref(), - self.awg.as_deref(), - self.plugin.as_deref(), - self.tcp_tls.as_deref(), - dns, - &self.paths, - ) - .await + build_report(&self.agent, self.plugin.as_deref(), dns, &self.paths).await }) } @@ -2869,7 +2891,10 @@ async fn leave_network( storage.remove_network(network.network_id).await?; // What a protocol kept for it goes too; there is no plugin loaded here // to be asked, so the one this build has is asked directly. - forget_protocol_state(paths, network.network_id); + let cleanup_paths = paths.clone(); + let cleanup_network = network.network_id; + tokio::task::spawn_blocking(move || forget_protocol_state(&cleanup_paths, cleanup_network)) + .await?; storage.release_ownership_lock(); println!("left `{name}` ({}) locally", short(&id, 10)); eprintln!( @@ -2898,17 +2923,18 @@ fn forget_protocol_state(paths: &StoragePaths, network: tsunagi::NetworkId) { Err(err) => eprintln!("warning: the wg-quic key store could not be opened: {err}"), } - let pure_store = PureWgConfig::new(paths.state_dir.join("wg")); - match tsunagi_wireguard::WgKeyStore::open(pure_store.key_store_path()) { - Ok(store) => { - if let Err(err) = store.forget(network) { - eprintln!("warning: the wg key for it could not be removed: {err}"); + for name in ["wg", "awg"] { + let path = paths.state_dir.join(name).join("wireguard.sqlite"); + if path.exists() { + match tsunagi_wg_quic::WgKeyStore::open(path) { + Ok(store) => { + if let Err(err) = store.forget(network) { + eprintln!("warning: old data key could not be removed: {err}"); + } + } + Err(err) => eprintln!("warning: old data key store could not be opened: {err}"), } } - Err(err) if !pure_store.key_store_path().exists() => { - let _ = err; - } - Err(err) => eprintln!("warning: the wg key store could not be opened: {err}"), } } @@ -3443,6 +3469,7 @@ struct MemberRow<'a> { path: Option<&'a tsunagi::ipc::DataPathReport>, /// What to call the member the first of those links goes to. via_label: Option, + data_error: Option<&'a str>, /// The name it claimed that another member holds, and what to call the /// holder. name_taken: Option<(&'a str, String)>, @@ -3488,6 +3515,7 @@ fn member_rows<'a>(network: &'a tsunagi::ipc::NetworkReport, own_id: &str) -> Ve online: false, path: None, via_label: None, + data_error: None, name_taken: None, transport: None, rtt_ms: None, @@ -3520,6 +3548,7 @@ fn member_rows<'a>(network: &'a tsunagi::ipc::NetworkReport, own_id: &str) -> Ve row.path = Some(&peer.path); row.transport = Some(&peer.transport); row.rtt_ms = peer.rtt_ms; + row.data_error = peer.data_error.as_deref(); } if let Some(overlay) = &network.overlay { for peer in &overlay.peers { @@ -3900,9 +3929,6 @@ fn member_row(row: &MemberRow<'_>) -> report::Row { } use tsunagi::ipc::DataPathReport; - let direct_control = row - .transport - .is_none_or(|transport| transport.eq_ignore_ascii_case("direct")); let tunnel_up = row.tunnel.is_some_and(|tunnel| tunnel.is_up()); let has_overlay = row.tunnel.is_some(); // Online is online: a member reached through others is as much here as @@ -3916,7 +3942,7 @@ fn member_row(row: &MemberRow<'_>) -> report::Row { "data path via {} · {hops} hops", row.via_label.as_deref().unwrap_or("another member") ), - false, + true, ), Some(DataPathReport::None) | None => ("connected · no data path yet".to_string(), false), }; @@ -3939,51 +3965,32 @@ fn member_row(row: &MemberRow<'_>) -> report::Row { } detail.extend(taken); - let health = if !data_path_good || !direct_control || (has_overlay && !tunnel_up) { + let health = if !data_path_good || row.data_error.is_some() || (has_overlay && !tunnel_up) { Health::Degraded } else { Health::Good }; let mut out = Row::new(health, row.label(), detail); + if let Some(reason) = row.data_error { + out = out.with_note(reason.to_owned()); + } if let Some(tunnel) = row.tunnel { - let proto = if tunnel.protocol.is_empty() { - "wg-quic" - } else { - &tunnel.protocol - }; out = out.with_note(match tunnel.handshake_secs_ago { - Some(secs) => { - let health = if proto == "tcp-tls" { - format!("uptime {secs}s") + Some(_) => format!( + "tx {} rx {} · IP bytes ↑{} ↓{}{} · {}", + tunnel.tx_packets, + tunnel.rx_packets, + tunnel.tx_bytes, + tunnel.rx_bytes, + if tunnel.dropped > 0 { + format!(", {} dropped", tunnel.dropped) } else { - format!("handshake {secs}s ago") - }; - format!( - "end-to-end tunnel {proto} · {health} · tx {} rx {}{} · transport detail: {}", - tunnel.tx_packets, - tunnel.rx_packets, - if tunnel.dropped > 0 { - format!(", {} dropped", tunnel.dropped) - } else { - String::new() - }, - tunnel.path - ) - } - None => { - if proto == "wg-quic" || proto == "wg" { - format!( - "end-to-end tunnel {proto} · no WireGuard handshake yet; the tunnel \ - cannot carry traffic" - ) - } else { - format!( - "end-to-end tunnel {proto} · no {proto} handshake yet; the tunnel \ - cannot carry traffic" - ) - } - } + String::new() + }, + tunnel.path + ), + None => "encrypted data channel is not ready yet".into(), }); } out @@ -4018,7 +4025,7 @@ fn host_section(observed: &Observed) -> report::Section { host.push(Row::new( Health::Info, "implementation", - "userspace overlay (boringtun WireGuard, TCP TLS 1.3); no kernel module needed", + "userspace overlay with independent encrypted data transports; no kernel module needed", )); { if cfg!(target_os = "linux") { @@ -4683,7 +4690,8 @@ async fn up(args: UpArgs) -> Result<(), Box> { .collect(); config.default_protocols = Some(initial); - let settings = parse_settings(&args.protocol_options)?; + let mut settings = parse_settings(&args.protocol_options)?; + let tunnel_settings = take_tunnel_settings(&mut settings)?; // A setting nobody takes is a mistake, not a preference: one that was // silently dropped looks exactly like one that did not work. for setting in &settings { @@ -4702,13 +4710,9 @@ async fn up(args: UpArgs) -> Result<(), Box> { // The interface belongs to the agent, so it is configured once whatever // was selected to carry traffic over it. - let mut wireguard = None; let mut awg_transport = None; - let mut awg_plugin = None; let mut tcp_tls_transport = None; - let mut tcp_tls_plugin = None; - let mut pure_wg_plugin: Option> = None; - let mut pure_wg_transport: Option> = None; + let mut pure_wg_transport: Option> = None; if !PROTOCOLS.is_empty() { let tun_factory: Arc = if args.no_tun { Arc::new(MemoryTunFactory::new()) @@ -4719,6 +4723,25 @@ async fn up(args: UpArgs) -> Result<(), Box> { config = config.with_interface(tun_factory, args.interface.clone(), mtu); } + let mut tunnel_config = WireguardConfig::new(paths.state_dir.join("wg-quic")); + tunnel_config.legacy_key_stores = ["wg", "awg"] + .into_iter() + .map(|name| paths.state_dir.join(name).join("wireguard.sqlite")) + .collect(); + if let Some(mtu) = args.mtu { + tunnel_config = tunnel_config.with_mtu(mtu); + } + let tunnel = + WireguardPlugin::open(WireguardPlugin::configure(tunnel_config, &tunnel_settings)?).await?; + config = config.with_tunnel(tunnel.clone()); + let wireguard = Some(tunnel); + + let identity_db = paths.state_db(); + let identity = tokio::task::spawn_blocking(move || { + tsunagi::storage::StateStore::open(identity_db)?.load_or_create_device_identity() + }) + .await??; + for spec in PROTOCOLS { // Unqualified options keep targeting the initial selection; explicitly // scoped options also configure protocols that will be enabled later. @@ -4732,16 +4755,6 @@ async fn up(args: UpArgs) -> Result<(), Box> { let options = settings_for(spec, &scoped); match spec.name { AWG_PROTOCOL => { - let mut wg = WireguardConfig::new(paths.state_dir.join("awg")); - if let Some(mtu) = args.mtu { - wg = wg.with_mtu(mtu); - } - let tunnel_options: Vec<_> = options - .iter() - .filter(|(key, _)| key != "stun") - .cloned() - .collect(); - let wg = WireguardPlugin::configure(wg, &tunnel_options)?; let server = options .iter() .find(|(key, _)| key == "stun") @@ -4752,42 +4765,32 @@ async fn up(args: UpArgs) -> Result<(), Box> { } let stun = (!matches!(args.reach, Reach::Local) && server != "off") .then(|| server.to_owned()); - let state_db = paths.state_db(); - let identity = tokio::task::spawn_blocking(move || { - tsunagi::storage::StateStore::open(state_db)?.load_or_create_device_identity() - }) - .await??; let transport = AwgTransport::new_with_stun( identity.endpoint_id(), std::net::Ipv4Addr::UNSPECIFIED, stun, ); - let plugin = AwgPlugin::open(wg, transport.clone()).await?; + transport.exclude_interface(args.interface.clone()); + let plugin = AwgPlugin::new(transport.clone()); config = config - .with_plugin(plugin.clone() as Arc) + .with_carrier( + plugin.clone() as Arc + ) .with_custom_transport( AWG_PROTOCOL, transport.clone() as Arc, ); awg_transport = Some(transport); - awg_plugin = Some(plugin); } - tsunagi_wg_quic::WIREGUARD_PROTOCOL => { - let mut wg = WireguardConfig::new(paths.state_dir.join("wg-quic")); - if let Some(mtu) = args.mtu { - wg = wg.with_mtu(mtu); - } - let wg = WireguardPlugin::configure(wg, &options)?; - let plugin = WireguardPlugin::open(wg).await?; - config = config.with_plugin(plugin.clone() as Arc); - wireguard = Some(plugin); + "wg-quic" => { + config = config.with_carrier(Arc::new( + tsunagi::dataplane::transport::iroh_link::IrohCarrier, + )); } TCP_TLS_PROTOCOL => { let tls_cfg = TcpTlsConfig::new(); - let tls_cfg = TcpTlsCodec::configure(tls_cfg, &options)?; - let identity = tsunagi::storage::StateStore::open(paths.state_db())? - .load_or_create_device_identity()?; + let tls_cfg = TcpTlsConfig::configure(tls_cfg, &options)?; let transport = Arc::new( TcpTlsTransport::bind(&identity, tls_cfg.port, tls_cfg.sni) .await @@ -4798,46 +4801,44 @@ async fn up(args: UpArgs) -> Result<(), Box> { transport.set_peer_addr(ep_addr.id, *ip); } } - let codec = TcpTlsCodec::new(identity.endpoint_id(), Arc::clone(&transport)); - let plugin = Arc::new(TcpTlsPlugin::new(codec)); + let plugin = + TcpTlsCarrier::with_interface(transport.clone(), args.interface.clone()); config = config - .with_plugin(plugin.clone() as Arc) + .with_carrier( + plugin.clone() as Arc + ) .with_custom_transport( TCP_TLS_PROTOCOL, Arc::clone(&transport) as Arc, ); tcp_tls_transport = Some(transport); - tcp_tls_plugin = Some(plugin); } WG_PROTOCOL => { - let mut wg_cfg = PureWgConfig::new(paths.state_dir.join("wg")); - if let Some(mtu) = args.mtu { - wg_cfg = wg_cfg.with_mtu(mtu); - } - let wg_cfg = PureWgPlugin::configure(wg_cfg, &options)?; - let identity = tsunagi::storage::StateStore::open(paths.state_db())? - .load_or_create_device_identity()?; - let transport = Arc::new( - PureWgTransport::bind(&identity, wg_cfg.port) - .await - .map_err(|err| err.to_string())?, + let port = options + .iter() + .find(|(key, _)| key == "port") + .map(|(_, value)| value.parse::()) + .transpose()? + .unwrap_or(51820); + let transport = AwgTransport::wireguard_with_port( + identity.endpoint_id(), + std::net::Ipv4Addr::UNSPECIFIED, + (!matches!(args.reach, Reach::Local)).then(|| "stun.l.google.com:19302".into()), + port, ); - for ep_addr in &bootstrap { - for addr in ep_addr.ip_addrs() { - transport.set_peer_addr(ep_addr.id, *addr); - } - } - let plugin = PureWgPlugin::open(wg_cfg, Arc::clone(&transport)).await?; + transport.exclude_interface(args.interface.clone()); + let plugin = AwgPlugin::new(transport.clone()); config = config - .with_plugin(plugin.clone() as Arc) + .with_carrier( + plugin.clone() as Arc + ) .with_custom_transport( WG_PROTOCOL, Arc::clone(&transport) as Arc, ); pure_wg_transport = Some(transport); - pure_wg_plugin = Some(plugin); } other => return Err(format!("`{other}` is listed but not built in").into()), } @@ -4902,18 +4903,13 @@ async fn up(args: UpArgs) -> Result<(), Box> { }); } if let Some(transport) = pure_wg_transport { - let agent_inbound = agent.clone(); + let inbound_agent = agent.clone(); tokio::spawn(async move { - transport - .accept_loop(move |inbound| { - let agent = agent_inbound.clone(); - tokio::spawn(async move { - if let Err(err) = agent.install_inbound_link(inbound).await { - tracing::debug!(%err, "failed to install inbound wg link"); - } - }); - }) - .await; + while let Some(inbound) = transport.next_inbound().await { + if let Err(err) = inbound_agent.install_inbound_link(inbound).await { + tracing::debug!(%err, "failed to install inbound wg link"); + } + } }); } @@ -4952,17 +4948,11 @@ async fn up(args: UpArgs) -> Result<(), Box> { // is not fatal: the agent itself works fine without it. let control = { let agent = agent.clone(); - let pure_wg = pure_wg_plugin.clone(); - let awg = awg_plugin.clone(); let plugin = wireguard.clone(); - let tcp_tls = tcp_tls_plugin.clone(); let dns = Arc::clone(&dns); let source: Arc = Arc::new(AgentControl { agent, - pure_wg, - awg, plugin, - tcp_tls, dns, paths: paths.clone(), }); @@ -5045,10 +5035,7 @@ async fn up(args: UpArgs) -> Result<(), Box> { /// both. async fn build_report( agent: &Agent, - pure_wg: Option<&PureWgPlugin>, - awg: Option<&AwgPlugin>, wireguard: Option<&WireguardPlugin>, - tcp_tls: Option<&TcpTlsPlugin>, dns: Option, paths: &StoragePaths, ) -> tsunagi::ipc::StatusReport { @@ -5067,226 +5054,37 @@ async fn build_report( .networks .iter() .map(|network| { - let awg_view = awg.and_then(|plugin| plugin.overview(network.network_id)); let wg_view = wireguard.and_then(|plugin| plugin.overview(network.network_id)); - let pure_wg_view = pure_wg.and_then(|plugin| plugin.overview(network.network_id)); - let tcp_tls_reports = tcp_tls - .map(|p| p.active_peers(network.network_id)) - .unwrap_or_default(); - - // Use whichever WireGuard view is available for the overlay - // address and range — wg-quic and pure wg both derive them - // identically from the control plane's allocations. - let wg_overlay_addr = wg_view - .as_ref() - .and_then(|v| v.overlay_address_v4) - .or_else(|| pure_wg_view.as_ref().and_then(|v| v.overlay_address_v4)) - .or_else(|| awg_view.as_ref().and_then(|v| v.overlay_address_v4)); + let wg_overlay_addr = wg_view.as_ref().and_then(|v| v.overlay_address_v4); let wg_prefix_len = wg_view .as_ref() .and_then(|v| v.ipv4_range) - .or_else(|| pure_wg_view.as_ref().and_then(|v| v.ipv4_range)) - .or_else(|| awg_view.as_ref().and_then(|v| v.ipv4_range)) .map_or(0, |r| r.prefix_len); - let has_any_plugin = wg_view.is_some() - || pure_wg_view.is_some() - || awg_view.is_some() - || !tcp_tls_reports.is_empty(); + let has_any_plugin = wg_view.is_some(); let overlay = if has_any_plugin && overlay.is_some() { let mut peers: Vec = Vec::new(); - // Collect all unique peer IDs known in this network. - let mut all_peer_ids: Vec = Vec::new(); - for member in &network.members { - if member.endpoint_id != status.endpoint_id - && !all_peer_ids.contains(&member.endpoint_id) - { - all_peer_ids.push(member.endpoint_id); - } - } - if let Some(ref view) = pure_wg_view { + if let Some(view) = &wg_view { for peer in &view.peers { - if !all_peer_ids.contains(&peer.endpoint_id) { - all_peer_ids.push(peer.endpoint_id); - } - } - } - if let Some(ref view) = wg_view { - for peer in &view.peers { - if !all_peer_ids.contains(&peer.endpoint_id) { - all_peer_ids.push(peer.endpoint_id); - } - } - } - if let Some(ref view) = awg_view { - for peer in &view.peers { - if !all_peer_ids.contains(&peer.endpoint_id) { - all_peer_ids.push(peer.endpoint_id); - } - } - } - for report in &tcp_tls_reports { - if !all_peer_ids.contains(&report.peer) { - all_peer_ids.push(report.peer); - } - } - - for peer_id in all_peer_ids { - let pure_wg_match = pure_wg_view - .as_ref() - .and_then(|v| v.peers.iter().find(|p| p.endpoint_id == peer_id)); - let awg_match = awg_view - .as_ref() - .and_then(|v| v.peers.iter().find(|p| p.endpoint_id == peer_id)); - let tcp_tls_match = tcp_tls_reports.iter().find(|r| r.peer == peer_id); - let wg_quic_match = wg_view - .as_ref() - .and_then(|v| v.peers.iter().find(|p| p.endpoint_id == peer_id)); - - let member_addr = network - .members - .iter() - .find(|m| m.endpoint_id == peer_id) - .and_then(|m| m.overlay_address_v4) - .map(|a| a.to_string()); - - if let Some(pure_peer) = pure_wg_match - && let Some(tunnel) = &pure_peer.tunnel - { + let tunnel = peer.tunnel.as_ref(); peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: "wg".into(), - public_key: pure_peer.public_key.to_string(), - address: pure_peer - .overlay_address_v4 - .map(|a| a.to_string()) - .or(member_addr), - handshake_secs_ago: tunnel.health.since_handshake.map(|s| s.as_secs()), - tx_packets: tunnel.stats.tx_packets, - rx_packets: tunnel.stats.rx_packets, - tx_bytes: tunnel.stats.tx_bytes, - rx_bytes: tunnel.stats.rx_bytes, - dropped: tunnel.stats.dropped_wrong_source - + tunnel.stats.dropped_oversize, - protocol_errors: tunnel.stats.protocol_errors, - path: tunnel.path.clone(), - }); - } else if let Some(tcp_tls) = tcp_tls_match { - let pubkey = pure_wg_match - .map(|p| p.public_key.to_string()) - .or_else(|| wg_quic_match.map(|p| p.public_key.to_string())) - .or_else(|| awg_match.map(|p| p.public_key.to_string())) - .unwrap_or_default(); - peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: tcp_tls.protocol.clone(), - public_key: pubkey, - address: member_addr, - handshake_secs_ago: Some(tcp_tls.uptime_secs), - tx_packets: tcp_tls.tx_packets, - rx_packets: tcp_tls.rx_packets, - tx_bytes: tcp_tls.tx_bytes, - rx_bytes: tcp_tls.rx_bytes, - dropped: tcp_tls.dropped, - protocol_errors: 0, - path: tcp_tls.path.clone(), - }); - } else if let Some(awg_peer) = awg_match - && let Some(tunnel) = &awg_peer.tunnel - { - peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: AWG_PROTOCOL.into(), - public_key: awg_peer.public_key.to_string(), - address: awg_peer - .overlay_address_v4 - .map(|a| a.to_string()) - .or(member_addr), - handshake_secs_ago: tunnel.health.since_handshake.map(|s| s.as_secs()), - tx_packets: tunnel.stats.tx_packets, - rx_packets: tunnel.stats.rx_packets, - tx_bytes: tunnel.stats.tx_bytes, - rx_bytes: tunnel.stats.rx_bytes, - dropped: tunnel.stats.dropped_wrong_source - + tunnel.stats.dropped_oversize, - protocol_errors: tunnel.stats.protocol_errors, - path: tunnel.path.clone(), - }); - } else if let Some(wg_peer) = wg_quic_match - && let Some(tunnel) = &wg_peer.tunnel - { - peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: "wg-quic".into(), - public_key: wg_peer.public_key.to_string(), - address: wg_peer - .overlay_address_v4 - .map(|a| a.to_string()) - .or(member_addr), - handshake_secs_ago: tunnel.health.since_handshake.map(|s| s.as_secs()), - tx_packets: tunnel.stats.tx_packets, - rx_packets: tunnel.stats.rx_packets, - tx_bytes: tunnel.stats.tx_bytes, - rx_bytes: tunnel.stats.rx_bytes, - dropped: tunnel.stats.dropped_wrong_source - + tunnel.stats.dropped_oversize, - protocol_errors: tunnel.stats.protocol_errors, - path: tunnel.path.clone(), - }); - } else if let Some(pure_peer) = pure_wg_match { - peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: "wg".into(), - public_key: pure_peer.public_key.to_string(), - address: pure_peer - .overlay_address_v4 - .map(|a| a.to_string()) - .or(member_addr), - handshake_secs_ago: None, - tx_packets: 0, - rx_packets: 0, - tx_bytes: 0, - rx_bytes: 0, - dropped: 0, - protocol_errors: 0, - path: "no data link".into(), - }); - } else if let Some(awg_peer) = awg_match { - peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: AWG_PROTOCOL.into(), - public_key: awg_peer.public_key.to_string(), - address: awg_peer - .overlay_address_v4 - .map(|a| a.to_string()) - .or(member_addr), - handshake_secs_ago: None, - tx_packets: 0, - rx_packets: 0, - tx_bytes: 0, - rx_bytes: 0, - dropped: 0, - protocol_errors: 0, - path: "no data link".into(), - }); - } else if let Some(wg_peer) = wg_quic_match { - peers.push(OverlayPeerReport { - endpoint_id: peer_id.to_string(), - protocol: "wg-quic".into(), - public_key: wg_peer.public_key.to_string(), - address: wg_peer - .overlay_address_v4 - .map(|a| a.to_string()) - .or(member_addr), - handshake_secs_ago: None, - tx_packets: 0, - rx_packets: 0, - tx_bytes: 0, - rx_bytes: 0, - dropped: 0, - protocol_errors: 0, - path: "no data link".into(), + counter_epoch: tunnel.map_or(0, |t| t.stats.counter_epoch), + endpoint_id: peer.endpoint_id.to_string(), + protocol: String::new(), + public_key: peer.public_key.to_string(), + address: peer.overlay_address_v4.map(|a| a.to_string()), + handshake_secs_ago: tunnel + .and_then(|t| t.health.since_handshake.map(|s| s.as_secs())), + tx_packets: tunnel.map_or(0, |t| t.stats.tx_packets), + rx_packets: tunnel.map_or(0, |t| t.stats.rx_packets), + tx_bytes: tunnel.map_or(0, |t| t.stats.tx_bytes), + rx_bytes: tunnel.map_or(0, |t| t.stats.rx_bytes), + dropped: tunnel.map_or(0, |t| { + t.stats.dropped_wrong_source + t.stats.dropped_oversize + }), + protocol_errors: tunnel.map_or(0, |t| t.stats.protocol_errors), + path: tunnel.map_or_else(|| "no data path".into(), |t| t.path.clone()), }); } } @@ -5349,6 +5147,7 @@ async fn build_report( .peers .iter() .map(|peer| PeerReport { + data_error: peer.data_error.clone(), endpoint_id: peer.endpoint_id.to_string(), hostname: peer.hostname.clone(), transport: peer.transport.to_string(), @@ -5628,11 +5427,11 @@ fn log_event(event: &Event) { path, max_datagram, } => { - tracing::info!( - network = %network.fmt_short(), - peer = %peer.fmt_short(), - "Data link established for {protocol} ({path}, MTU datagram: {max_datagram}B)" - ); + if protocol == tsunagi_wg_quic::WIREGUARD_PROTOCOL { + tracing::info!(network = %network.fmt_short(), peer = %peer.fmt_short(), "Data route available ({path})"); + } else { + tracing::info!(network = %network.fmt_short(), peer = %peer.fmt_short(), "Data link established for {protocol} ({path}, MTU datagram: {max_datagram}B)"); + } } Event::DataLinkDown { network, @@ -5781,6 +5580,7 @@ mod status_tests { relay_sent_via: 0, relay_received_via: 0, peers: vec![PeerReport { + data_error: None, endpoint_id: ONLINE.into(), hostname: Some("music".into()), transport: "direct".into(), @@ -6271,7 +6071,7 @@ mod status_tests { "names the member it goes through: {text}" ); assert!(!text.contains("offline"), "{text}"); - assert_eq!(out.worst(), Health::Degraded, "{text}"); + assert_eq!(out.worst(), Health::Good, "{text}"); } #[test] @@ -6305,14 +6105,14 @@ mod status_tests { } #[test] - fn a_relayed_peer_is_graded_as_degraded_quality() { + fn relayed_control_does_not_degrade_a_working_data_path() { let mut network = network_after_a_peer_returned(); network.peers[0].transport = "relay".into(); let mut out = report::Report::new(); out.push(network_section(&network, OWN, false)); - assert_eq!(out.worst(), Health::Degraded, "{}", out.render(false)); - assert!(out.render(false).contains("relay")); + assert_eq!(out.worst(), Health::Good, "{}", out.render(false)); + assert!(out.render(false).contains("control path relay")); } #[test] @@ -6324,7 +6124,10 @@ mod status_tests { out.push(network_section(&network, OWN, false)); let text = out.render(false); assert_eq!(out.worst(), Health::Degraded, "{text}"); - assert!(text.contains("no WireGuard handshake yet"), "{text}"); + assert!( + text.contains("encrypted data channel is not ready yet"), + "{text}" + ); } #[test] @@ -6347,7 +6150,7 @@ mod status_tests { } #[test] - fn a_tcp_tls_tunnel_shows_healthy_state_and_metrics() { + fn a_tcp_tls_carrier_shows_healthy_state_and_metrics() { let mut network = network_after_a_peer_returned(); let mut t = tunnel(ONLINE, Some(42)); t.protocol = "tcp-tls".into(); @@ -6360,17 +6163,14 @@ mod status_tests { out.push(network_section(&network, OWN, false)); let text = out.render(false); assert_eq!(out.worst(), Health::Good, "{text}"); - assert!(text.contains("end-to-end tunnel tcp-tls"), "{text}"); - assert!(text.contains("uptime 42s"), "{text}"); + assert!(!text.contains("end-to-end"), "{text}"); + assert!(!text.contains("handshake"), "{text}"); assert!(text.contains("tx 100 rx 200"), "{text}"); - assert!( - text.contains("transport detail: tcp-tls via 192.0.2.1:443"), - "{text}" - ); + assert!(text.contains("tcp-tls via 192.0.2.1:443"), "{text}"); } #[test] - fn end_to_end_tunnel_and_data_transport_are_named_separately() { + fn status_names_only_the_external_data_transport() { let mut network = network_after_a_peer_returned(); network.peers[0].path = tsunagi::ipc::DataPathReport::Direct { transport: "tcp-tls".into(), @@ -6386,20 +6186,41 @@ mod status_tests { let text = out.render(false); assert!(text.contains("data path direct over tcp-tls"), "{text}"); assert!(text.contains("control path direct"), "{text}"); - assert!( - text.contains("end-to-end tunnel wg · handshake 1s ago"), - "{text}" - ); - assert!( - text.contains("transport detail: tcp-tls via 192.0.2.1:443"), - "{text}" - ); + assert!(!text.contains("end-to-end"), "{text}"); + assert!(!text.contains("handshake"), "{text}"); + assert!(text.contains("tcp-tls via 192.0.2.1:443"), "{text}"); } } #[cfg(test)] mod network_tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + #[test] + fn historic_cipher_options_configure_one_tunnel_and_never_reach_carriers() { + let mut settings = super::parse_settings(&[ + "keepalive=10".into(), + "wg:keepalive=10".into(), + "wg-quic:mtu=1400".into(), + "wg:port=0".into(), + ]) + .unwrap(); + let shared = super::take_tunnel_settings(&mut settings).unwrap(); + assert_eq!( + shared, + [ + ("keepalive".into(), "10".into()), + ("mtu".into(), "1400".into()) + ] + ); + assert_eq!(settings.len(), 1); + assert_eq!(settings[0].key, "port"); + let mut conflict = + super::parse_settings(&["wg:keepalive=10".into(), "awg:keepalive=20".into()]).unwrap(); + assert!(super::take_tunnel_settings(&mut conflict).is_err()); + let mut unknown = super::parse_settings(&["typo:mtu=1400".into()]).unwrap(); + assert!(super::take_tunnel_settings(&mut unknown).is_err()); + } + #[test] fn default_carriers_include_awg_and_explicit_order_is_preserved() { use clap::Parser; diff --git a/docs/amnezia.md b/docs/amnezia.md index 417b02e..bc61e86 100644 --- a/docs/amnezia.md +++ b/docs/amnezia.md @@ -17,13 +17,16 @@ tsng protocols home awg ``` Changes are local to this device, persist across restarts, and retire existing -data links and end-to-end channels for that network before renegotiation. +physical data links for that network before renegotiation. The single internal +WireGuard channel, its keys, session and counters survive. Authenticated control sessions, membership, addresses and other networks stay live. Removing every protocol (`none` in the CLI, all checkboxes off in the tray) -leaves the network control-only. This is different from automatic carrier -fallback, which preserves the selected end-to-end channel. +leaves the network control-only. Re-enabling a carrier resumes the existing +internal channel, subject to ordinary WireGuard and application timeouts. The +choice applies to both incoming and outgoing local data links; a relay may use +a different transport on its next hop. Control and discovery are independent. -`awg` is Tsunagi AWG wire version 1. Its packet masking engine is adapted from +`awg` is Tsunagi AWG wire version 2. Its packet masking engine is adapted from [RunaRoo/AmneziaWG-RS](https://github.com/RunaRoo/AmneziaWG-RS), with the pinned source and license recorded in [UPSTREAM.md](../crates/tsunagi-awg/UPSTREAM.md). It uses the existing boringtun @@ -36,7 +39,7 @@ not implemented: this carrier connects authenticated Tsunagi members. There is one independent UDP socket and cancellable packet-engine task per `(network, peer)` pair. The smaller endpoint id initiates, following the same rule as other data carriers. Both sides must announce enabled AWG wire version -1. An authenticated iroh control session is required throughout; AWG supplies +2. An authenticated iroh control session is required throughout; AWG supplies no alternate discovery or membership mechanism. Every `open` generates new masking parameters, a header-protection key, an @@ -100,25 +103,37 @@ candidates. IPv6-only paths, symmetric NAT and networks blocking arbitrary UDP may fail AWG and fall back; there is no AWG-over-QUIC substitution or AWG TURN relay hidden behind its name. -Data delivery checks go through the actual end-to-end encrypted tunnel, once -per second, independently of control-plane RTT. Requests and replies bind a +Data delivery checks go through the specific authenticated physical carrier, +once per second, independently of control-plane RTT or the shared internal +tunnel. A successful relay path cannot mask a broken direct carrier. Requests and replies bind a fresh private marker/key to the sender's endpoint id. Unanswered requests expire after three seconds. After a 20-second warmup, at least 15 completed observations and at least 60% loss in the last 20 observations retire a carrier. Probes are -consumed locally after source/destination admission and never enter the TUN. +consumed at the carrier boundary and never enter the TUN or user IP counters. AWG additionally checks encrypted outer delivery throughout the pair lifetime. Setup failure or persistent runtime loss advances to the next mutually supported carrier in the initiator's order. Earlier expired cooldowns do not jump ahead of untried carriers. Retry cooldowns remain bounded; reaching AWG again negotiates new parameters. Control sessions and the logical end-to-end tunnel survive a -carrier change. Status names the end-to-end protocol and the actual carrier -separately, so `wg` over `awg` is a valid result. +carrier change. Status and the tray show the actual external transport or relay +route and data readiness. There is no user-selectable internal cipher. -Control wire version 6 introduces the private transport message. All members -must upgrade together; device identities, network ids, handshake transcript -encoding, signed state and stored network configuration are unchanged. Data -ALPN remains `tsunagi/data/5`; the relay envelope is unchanged. +The internal capability is `wireguard-e2e` v4; external `wg` and `awg` use v2, +`wg-quic` v1 and `tcp-tls` v2. All members must upgrade together. Control ALPN +`tsunagi/ctrl/8`, discovery, membership and signed state are unchanged. The data +ALPN is `tsunagi/data/6`; the relay envelope format is unchanged. Local IPC is v23, +so agent, CLI and tray must match. Historical `wg-quic/wireguard.sqlite` is the +canonical internal key store. Missing keys are imported transactionally from +`wg` then `awg`, without replacing an existing canonical key or modifying a +source database. Corrupt key stores fail rather than silently regenerating keys. +Device identities, network ids, addresses and saved transport choices survive. + +Ordinary `wg` uses the same independent pair engine with standard WireGuard +headers, no masking/junk, fresh hop keys and PSK, and a preferred local UDP port +(default 51820; additional pairs use dynamic ports). The whole relay envelope is +inside the hop encryption. It is separate from the internal WireGuard channel. +Neither UDP mode is a client for an externally configured WireGuard server. ## Verification diff --git a/docs/architecture.md b/docs/architecture.md index a1be023..d360ad1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,14 +11,15 @@ messages. On top of it, this crate's protocol handles membership authentication, participant announcements, capability exchange and — later — state synchronisation and delivery of IP-plugin data. -**Data plane.** Separate plugins create IP connectivity. WireGuard is the first -one and is implemented in userspace — see [wireguard.md](wireguard.md). Plugin +**Data plane.** One internal WireGuard plugin creates IP connectivity in userspace — see [wireguard.md](wireguard.md). Plugin keys, configuration and lifecycle are separate from iroh identity and from the network secret. The core moves an opaque, bounded payload and never parses it. **The transport in between.** Plugins do not open connections. They are handed a `PacketLink` — an authenticated, unreliable datagram channel to one peer for -one protocol — and never learn how it is carried. +the internal cipher — and never learns how it is carried. External `DataCarrier` +adapters own their reachability capabilities and physical link lifecycle; they +never receive plaintext overlay IP packets. The separation between the two planes is **logical, not physical**. The control plane uses iroh; the data plane can use iroh on its own ALPN and connection, or independent @@ -73,8 +74,10 @@ And in crates of their own: | crate | responsibility | |---|---| -| `tsunagi-awg` | ephemeral pairwise Amnezia masking, private negotiation and independently encrypted UDP carriers; reuses the end-to-end WireGuard engine | -| `tsunagi-wg-quic` | the `wg-quic` protocol: its keys, its announcement, its tunnels | +| `tsunagi-wireguard` | independent ordinary WG and masked UDP hop engines, private pair negotiation and local candidates | +| `tsunagi-tcp-tls` | TLS hop encryption and bounded reachability announcements; no inner IP cipher | +| `tsunagi-awg` | ephemeral pairwise Amnezia masking, private negotiation and independently encrypted UDP carriers; shares the independent UDP hop engine with `wg` | +| `tsunagi-wg-quic` | the single internal WireGuard cipher, keys and opaque announcement (historical crate name) | | `tsunagi` | the command line agent, built as the `tsng` executable: the only place that owns a runtime, a logger and signals | The core library is the package `tsunagi-core`; its library target is still called `tsunagi`, so code says `use tsunagi::…`. @@ -85,7 +88,7 @@ they compare the *wire* version, which moves only when the bytes do, so two peers on different releases keep working. Abstractions exist only where something is really substituted or really needs -isolating for tests: `NetworkDiscovery`, `IpPlugin`, `PacketTransport` / +isolating for tests: `NetworkDiscovery`, one `IpPlugin`, external `DataCarrier`, `PacketTransport` / `PacketLink` (so a protocol's carrier can change), `TunFactory` (which is what lets the whole data plane be tested without privileges), `InterfaceProvisioner` and `DnsPublisher` (which are where the platforms differ). Everything else is a @@ -161,8 +164,10 @@ Values that cannot honestly be attributed to one network stay at the endpoint level. A value iroh does not report is `None`. An iroh address is an address for iroh. It is not assumed usable by any other -protocol; a future WireGuard plugin gathers its own reachability data and ships -it through the control plane as an opaque announcement. +protocol; each external carrier gathers its own reachability data and ships it through +the control plane as an opaque announcement or private pair negotiation. +Local iroh candidates and introductions exclude the agent's own overlay, so +transport traffic cannot recursively tunnel through itself. ### Relays diff --git a/docs/protocol.md b/docs/protocol.md index 6f283aa..2490ed6 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -15,9 +15,9 @@ Upgrading the crate or bumping the control protocol must never change an existing `NetworkId`. Control version 8 adds signed, multi-writer VIP field snapshots to version 7's -node retirements. Data ALPN remains `tsunagi/data/5`. All members must upgrade +node retirements. Data ALPN is `tsunagi/data/6` for the single internal WireGuard channel. All members must upgrade together. Network ids, derivation labels, handshake encoding and existing record -signatures are unchanged. Local CLI/tray IPC is version 22; restart the agent +signatures are unchanged. Local CLI/tray IPC is version 23; restart the agent with matching clients. SQLite schema 10 adds `vip_fields`, preserving identity, network configuration, signed node records and retirement proofs. See [virtual services](vip.md), [state sync](sync-model.md) and [AWG](amnezia.md). @@ -145,7 +145,7 @@ event does not report a network id. ## The data plane protocol IP plugin packets never travel on a control connection. They use their own -ALPN, `tsunagi/data/5`, on their own iroh connection: +ALPN, `tsunagi/data/6`, on their own iroh connection: ```text initiator -> responder : (the same membership handshake as above) @@ -192,7 +192,7 @@ and version 5 added the end-to-end protocol to it: | source peer | 32 | original endpoint public key | | destination peer | 32 | final endpoint public key | | flow id | 8 | big endian opaque stable hash supplied by the plugin | -| protocol | 8 | big endian FNV-1a of the end-to-end protocol id (`wg`, `tcp-tls`, ...) | +| protocol | 8 | big endian FNV-1a of the internal protocol id (`wireguard-e2e`) | | payload | remaining | end-to-end encrypted plugin bytes | The fixed header is 82 bytes. Zero/oversized hop limits, unknown versions, diff --git a/docs/routing.md b/docs/routing.md index 0f71c33..10904a4 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -16,8 +16,8 @@ the default 1280-byte interface MTU also works on small QUIC paths. Each authenticated control session advertises that peer's own direct data links. There is one table for each network, whatever protocols run over it: a -link is a way to reach a neighbour, and a protocol is an end-to-end tunnel to a -peer. The envelope names the protocol, so the two are independent. A neighbour +link is a way to reach a neighbour, while the single internal protocol is an +end-to-end tunnel to a peer. The envelope names the protocol, so the two are independent. A neighbour reached over several kinds of transport (UDP, TCP, QUIC datagrams) is sent to over the best open one, in the order the protocols were configured, and over the next if that one fails or cannot carry the frame. Every authenticated @@ -28,14 +28,16 @@ Link arrival, closure and changed announcements rebuild routes immediately. Unchanged announcements refresh their age without republishing the forwarding snapshot. -Which protocol a pair of members runs their tunnel in is settled without -asking: the member with the lower endpoint id decides, by the order it -announces its protocols in, among those both have enabled at the same wire -version. Both ends read the same answer from what they already hold. The -tunnel is created as soon as the peer is reachable by any path, and it does not -change when a direct link comes or goes underneath it: as soon as a direct -link comes up it is shorter than the path through others, and traffic moves to -it with nothing to redo. The user sees only a lower round-trip time. +Every pair uses one internal WireGuard channel, matched by its fixed opaque +capability and independent of the selected external protocols. The tunnel is +created when a member and its key are known; its logical link survives temporary +loss of all physical paths. Carrier selection changes, automatic fallback and +shorter routes keep the same keys, interface and counters. The lower endpoint +id still chooses the mutually enabled external carrier in its preference order. +A relayed AWG first hop can lead to a QUIC next hop: the user's choice governs +this device's adjacent links, while the inner ciphertext stays unchanged. +An existing TCP connection continues after transient loss if its own timeout +has not expired; a sufficiently long outage can still terminate it. Breadth-first search computes the shortest directed paths and every equal-cost first hop in sorted order. An actual direct link always wins. The first row is @@ -124,5 +126,5 @@ counts sends without storing frames. Results are CPU forwarding cost, **not** end-to-end network latency or VPN throughput; encryption, fragmentation, sockets, congestion and scheduling contribute separately. -Wire compatibility: control ALPN `tsunagi/ctrl/8`, data ALPN `tsunagi/data/5`. +Wire compatibility: control ALPN `tsunagi/ctrl/8`, data ALPN `tsunagi/data/6`. Upgrade every participant together; saved identities and network state persist. diff --git a/docs/testing.md b/docs/testing.md index 036fc49..46e2fcb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -185,7 +185,8 @@ validation. Tests are offline; public STUN/DPI/NAT effectiveness needs separate real-network validation. Runtime protocol selection tests switch real AWG and iroh/QUIC carriers and -end-to-end WireGuard tunnels from either endpoint-id role, carry full-size +external policies from either endpoint-id role while preserving the one internal +WireGuard tunnel, key, handshake and counter epoch, carry full-size packets, keep another active network's tunnels unchanged, disable and re-enable the data plane without losing control sessions, and preserve selection after restart. CLI tests use a real agent process and control socket to exercise @@ -225,8 +226,8 @@ to test the entire mesh inside these namespaces. Devices-window tests render the egui grid at multiple widths with its real scroll area. They check stretching and shrinking without an expanding first -column, separate handshake/control RTT values, unknown measurements and short -end-to-end hints that do not substitute the carrier protocol. This does not +column, control RTT without internal handshake/protocol clutter, unknown measurements +and data readiness/error hints. This does not establish native desktop rendering on every platform. @@ -252,3 +253,22 @@ widths, click a diagram backend, and distinguish the new-flow selection from traffic on an existing binding. Sampling checks cover idle open flows, stale observations, counter resets, recreated listeners and removal of cached histories. These are headless egui checks, not native desktop screenshots. + +`tsunagi-wg-quic/tests/carriers.rs` uses the same internal WireGuard plugin over +real ordinary WG UDP and TCP/TLS, with loopback address selection and memory +TUNs. Exact original IP byte/packet counts exclude health probes and encryption +overhead. The AWG suite connects AWG-only and QUIC-only endpoints through a +mixed relay. One real smoltcp TCP connection stays established through relay → +direct QUIC → a dropped packet with all local carriers off → AWG relay; both +middle TUNs and endpoint keys are checked. This exercises a real userspace TCP +stack, not a host SSH server. Legacy key migration tests cover missing keys, +canonical precedence, idempotence and damaged source files. + +Candidate tests exercise the vendored iroh local address filter on real bound +endpoints and exclude overlay subnets from introductions. A renaming memory TUN checks +that carriers receive the actual interface name, as required for macOS utun. +A missing native WG transport is refused rather than silently carried by QUIC. Tray traffic tests +cover a joining peer with 100 GB of prior counters, departed peers, removed +networks and new counter epochs, including resets hidden by growth in the other +direction. These distinguish rates from absolute volumes; they do not establish +the cause of an earlier unrecorded high absolute counter on a real host. diff --git a/docs/threat-model.md b/docs/threat-model.md index 583f86c..cb6bf66 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -130,7 +130,7 @@ AWG profiles, PSKs and outer keys are private to one authenticated pair attempt, not shared network state. Address candidates are unverified until authenticated outer WireGuard data arrives. Header protection and structured signature packets reduce fixed wire markers; they do not guarantee admission by DPI or hide traffic -analysis. iroh control availability remains a prerequisite. Encrypted end-to-end -delivery probes detect sustained carrier loss, never confer membership and pass -the same signed source/destination admission as ordinary IP traffic. See +analysis. iroh control availability remains a prerequisite. Authenticated physical-link +delivery probes detect sustained loss on that carrier, independently of a +working end-to-end relay path. They never confer membership or enter the TUN. See [amnezia.md](amnezia.md) for limits and tested scope. diff --git a/docs/wireguard.md b/docs/wireguard.md index ad08f58..dcd4c0d 100644 --- a/docs/wireguard.md +++ b/docs/wireguard.md @@ -1,6 +1,6 @@ # The WireGuard data plane -WireGuard is the first IP plugin. It carries user traffic between +WireGuard is the single internal IP cipher. It carries user traffic between participants while the control plane keeps doing its own job: deciding who is in the network and carrying each participant's opaque announcement. @@ -46,20 +46,26 @@ peer by the agent and runs a WireGuard tunnel over it: source address checked <--Tunn.decapsulate-- ciphertext <- transport ``` -Reachability — hole punching, relay fallback — belongs to the transport, which -today is iroh. That is the whole reason the plugin's announcement says *who* it -is and never *where* it is: there is no address for a peer to advertise, get -wrong, or lie about. +Reachability belongs to external carriers: ordinary WireGuard UDP (`wg`), +Amnezia-masked UDP (`awg`), iroh QUIC (`wg-quic`), or TCP/TLS (`tcp-tls`). The +user's ordered choice applies only to those physical data links. The internal +capability `wireguard-e2e` v4 is fixed and hidden from the CLI/tray controls. +All carriers move the same inner encrypted bytes, including through relays. +The encryption plugin's announcement identifies a key, never an IP endpoint. -**Two peers behind NAT work exactly as well as iroh does.** iroh hole punches a -direct path when it can and falls back to a relay when it cannot; the tunnel -rides on whichever it got. There is no separate STUN, no separate hole punching -and no second set of NAT problems to solve for WireGuard. +A relay removes its adjacent carrier's encryption to read routing metadata, +then sends the inner ciphertext on another carrier. It cannot decrypt the +endpoint IP packet. WireGuard session keys, counters and the interface survive +carrier changes and direct/relay changes. TCP inside the overlay retransmits +after transient loss; ordinary application and WireGuard timeouts still apply. +The UDP modes have their own bounded STUN/pair discovery and can fail on NATs +that iroh can traverse; selecting only AWG does not permit QUIC fallback. ## Checking it from outside `tsng status` asks a running agent over its local control socket and prints -what it sees, including whether each tunnel has actually handshaken. See +the external transport or relay route, data readiness and plaintext counters. +An unavailable channel includes a reason such as no common permitted transport. See [../README.md](../README.md#checking-that-it-works). ## Deterministic overlay addressing @@ -207,26 +213,40 @@ QUIC DATAGRAM frames themselves cannot fragment. Reassembly is bounded and expires incomplete packets; one lost fragment loses one packet, without blocking unrelated traffic. The logical data payload limit is 64 KiB, with the relay envelope subtracted before it reaches the plugin. -The new framing uses data ALPN `tsunagi/data/5`; both ends and intermediate +The new framing uses data ALPN `tsunagi/data/6`; both ends and intermediate peers need the updated binary. Network identities and saved state do not change. See [protocol.md](protocol.md#the-data-plane-protocol) for the wire format. -## Lifecycle +## Lifecycle and counters -* A network is activated → the plugin loads or creates its key for that - network, derives the interface name, and creates the packet interface. If - that fails — no privileges, for instance — the key and the announcement still - work and the interface is retried on the next reconcile. -* A peer announces its key → recorded. -* A data link to that peer arrives → recorded. -* Reconciliation starts a tunnel for every peer that has **both**, and removes - tunnels for peers that lost either. -* A network is deactivated, or the agent shuts down → the interface and every - tunnel go away. The key stays, so coming back keeps the same overlay address. +The agent owns one interface across every network. On network activation the +internal plugin loads a network-specific key, announces it and reconciles +peer tunnels. It owns no interface or host route. Local IP ingress resolves +the destination owner and sends only through this one plugin. Source admission +is enforced after decryption by the agent, not by trusting a remote address. -There is no external configuration file and no command line tool, so unlike a -kernel-WireGuard setup there is nothing outside this process for anybody to -edit. Reconciliation is purely "do the running tunnels match what is known". +A carrier policy change retires physical links while retaining the logical +peer link and the internal tunnel. A peer's control membership ending removes +its tunnel; network deactivation removes that network's channels. Its persistent +key remains. The interface lives until agent shutdown, independently of carrier +availability and individual network lifetimes. + +The historical `wg-quic/wireguard.sqlite` remains the canonical key store. +Missing keys migrate from `wg` then `awg` without replacing canonical keys or +modifying the source files. Corrupt legacy databases are hard errors. Old +`-o keepalive=…`, `-o wg:keepalive=…` and `mtu` settings configure the shared +internal channel; contradictory values are rejected. Interface MTU is selected +once with `--mtu`, independent of carrier selection. + +User traffic counters count original IP bytes once when the encrypted frame is +accepted by the logical transport, and once when decrypted on receipt. They do +not promise remote delivery. They exclude carrier overhead, handshakes, +keepalives, relay transit and carrier delivery probes. Packets queued before a +handshake retain their exact original lengths. Each new tunnel gets an opaque +counter epoch so tray deltas cannot cross a restart, including when its new +counter already exceeds the old one. A carrier switch does not change that +epoch. Network rates sum per-peer deltas; peers arriving with prior traffic +never add that history to the current rate. ## Using it @@ -238,71 +258,21 @@ tsng join --network lab --secret "$SECRET" See the two-machine walkthrough in [../README.md](../README.md#trying-it-on-two-machines). -From the library: +From the library, configure one `WireguardPlugin` with +`AgentConfig::with_tunnel`, the agent's `with_interface`, and independent +`with_carrier` adapters. UDP/TLS carriers also register their physical +`PacketTransport` using `with_custom_transport`. Their own accept tasks pass +real links to `Agent::install_inbound_link`. The `carriers.rs` integration test +is a rootless example for ordinary WG and TLS; `tsunagi-awg/tests/pairs.rs` +covers mixed transports and relay transitions. -```rust,no_run -use std::sync::Arc; -use tsunagi::config::{AgentConfig, StoragePaths, TransportPolicy}; -use tsunagi::dataplane::IpPlugin; -use tsunagi::dataplane::wireguard::{MemoryTunFactory, WireguardConfig, WireguardPlugin}; -use tsunagi::identity::{NetworkName, NetworkSecret}; -use tsunagi::{Agent, Result}; - -#[tokio::main] -async fn main() -> Result<()> { - let paths = StoragePaths::user_default()?; - - // MemoryTunFactory needs no privileges; swap in ManagedTunFactory for a - // real interface. - let plugin = WireguardPlugin::open( - WireguardConfig::new(paths.state_dir.join("wireguard")), - Arc::new(MemoryTunFactory::new()), - ) - .await - .expect("wireguard plugin"); - - let agent = Agent::spawn( - AgentConfig::new(paths) - .with_transport(TransportPolicy::N0Defaults) - .with_plugin(plugin.clone() as Arc), - ) - .await?; - - let network = agent - .join_network(&NetworkName::new("lab")?, &NetworkSecret::generate()) - .await?; - - if let Some(view) = plugin.overview(network) { - println!("{} on {}", view.interface, view.overlay_address); - } - agent.shutdown().await; - Ok(()) -} -``` - -## Limits and future work - -* **Full mesh only.** Every member runs a tunnel to every other member. - Routing through an intermediate participant is not implemented. -* **Nothing frees an address yet.** The release record exists and merges, but - no command emits one. -* **A snapshot grows with the number of members ever seen**, and is capped per - message rather than compacted. -* **No routes, DNS or firewall rules.** The plugin creates its interface and - nothing else. Anything beyond the overlay `/64` is the operator's business. -* **Membership is session-scoped.** A peer leaves the overlay when its control - session ends; surviving a long absence is the same future work. -* **Userspace costs CPU.** Kernel WireGuard is faster. A kernel backend could - return behind the same boundary, but it would give up transport-provided NAT - traversal unless paired with a local proxy. -* **The interface is managed, not prepared.** On Linux the agent creates the - TUN interface and configures it over netlink, in process. See - *Provisioning* below. -* **No provisioner exists for macOS or Windows yet.** There the agent attaches - to an interface prepared by hand and says so. -* **The system interface path is not exercised by the default suite**, because - it needs privileges. Everything else about the data plane is. +## Limits +The default suite exercises real cryptography, sockets, handshakes, storage and +IP routing using memory TUNs. It does not establish public NAT traversal, DPI +effectiveness, actual host TUN performance, or native GUI behavior on every OS. +Userspace encryption costs CPU. Reassembly and packet queues are bounded and +may drop packets; host TCP handles retransmission within its own timeouts. ## Provisioning the interface @@ -313,7 +283,7 @@ should carry and no others — observes what is actually on the host, and applies the difference. Running it twice changes nothing the second time. The decision of *what* to change is -`dataplane::wireguard::provision::plan_changes`: pure, platform-independent +`overlay::provision::plan_changes`: pure, platform-independent and unit-tested on every platform. Only the execution is behind `InterfaceProvisioner`, which has three implementations — netlink on Linux, a `MockProvisioner` over a pretend host for the tests, and one that refuses with @@ -342,13 +312,12 @@ addresses with them. The two refusals are the interesting part: -* **A link that is not a TUN is never touched.** The name is derived from the - network id, so colliding with a real device is unlikely rather than - impossible, and deleting somebody's bridge is not a recoverable mistake. +* **A link that is not a TUN is never touched.** The interface name is supplied locally, so colliding with a real device is + possible, and deleting somebody's bridge is not a recoverable mistake. * **A TUN another process holds open is never deleted.** Carrier is the signal: a TUN has it exactly while something is attached. An attached one is a working overlay, almost certainly a second agent on this host, and it is - told to use a different `--wg-prefix` instead. + told to use a different `--interface` instead. ### Privilege @@ -374,6 +343,6 @@ construction. ### What it cannot be told to do Nothing here takes a name, an address or a command from the network. The -interface name is derived from the network id, the addresses come from the -local plugin and the signed allocation records, and no external program is +interface name comes from local configuration, the addresses come from the +signed allocation records, and no external program is executed at any point — there is no `ip`, no shell and no `PATH` involved. diff --git a/vendor/iroh/TSUNAGI-PATCH.md b/vendor/iroh/TSUNAGI-PATCH.md index 0af35d5..512f59e 100644 --- a/vendor/iroh/TSUNAGI-PATCH.md +++ b/vendor/iroh/TSUNAGI-PATCH.md @@ -41,3 +41,20 @@ failures, exercise success draining, and bound length and capacity under 10,000 distinct candidates. They fail with the original unconditional enqueue. These are queue algorithm tests; the normal integration suite continues to use real iroh endpoints, membership handshakes, SQLite and encryption. + +## Excluding overlay candidates + +`Builder::addr_filter` filters discovery publication only. That is insufficient +for an overlay: iroh still put the overlay interface address in the local +candidate watch used for path creation, allowing its own transport packets to +re-enter the overlay. Production logs contained verified data paths via +`10.13.37.*` overlay addresses while their underlay links cycled. + +The separate optional `Builder::direct_addr_filter` applies an `AddrFilter` +before storing local direct addresses. Default behavior is unchanged. Tsunagi +uses its existing local-interface exclusion for both this filter and publication, +including the actual kernel-assigned interface name after creation. Control authentication and +wire formats are unchanged. The rootless real-endpoint regression is +`crates/tsunagi-core/tests/iroh_candidates.rs`; overlay subnet filtering for +introductions is also covered in the core. This prevents the recursive path; +it does not prove the historical cause of any particular high byte counter. diff --git a/vendor/iroh/src/endpoint.rs b/vendor/iroh/src/endpoint.rs index cf158f9..df46822 100644 --- a/vendor/iroh/src/endpoint.rs +++ b/vendor/iroh/src/endpoint.rs @@ -138,6 +138,7 @@ pub struct Builder { /// Default address filter applied to all address lookup services added via /// [`Builder::address_lookup`]. addr_filter: Option, + direct_addr_filter: Option, proxy_url: Option, ca_tls_config: Option, #[cfg(not(wasm_browser))] @@ -206,6 +207,7 @@ impl Builder { address_lookup: Default::default(), address_lookup_user_data: Default::default(), addr_filter: None, + direct_addr_filter: None, proxy_url: None, ca_tls_config: None, #[cfg(not(wasm_browser))] @@ -271,6 +273,7 @@ impl Builder { }); let sock_opts = socket::Options { + direct_addr_filter: self.direct_addr_filter, transports: self.transports, secret_key, address_lookup_user_data: self.address_lookup_user_data, @@ -626,6 +629,13 @@ impl Builder { self } + /// Filters local direct candidates before endpoint addresses and hole-punch hints + /// are published. Unlike `addr_filter`, this applies to every candidate source. + pub fn direct_addr_filter(mut self, filter: AddrFilter) -> Self { + self.direct_addr_filter = Some(filter); + self + } + /// Clears the address filter, allowing all addresses to be published. /// /// This removes any filter previously set via [`Self::addr_filter`], including diff --git a/vendor/iroh/src/socket.rs b/vendor/iroh/src/socket.rs index 5c2f6ca..ed6aeb8 100644 --- a/vendor/iroh/src/socket.rs +++ b/vendor/iroh/src/socket.rs @@ -160,6 +160,7 @@ impl From> for RemoteStateActorStoppe /// Contains options for `Socket::listen`. #[derive(derive_more::Debug)] pub(crate) struct Options { + pub(crate) direct_addr_filter: Option, /// The configuration for the different transports. pub(crate) transports: Vec, @@ -349,6 +350,7 @@ pub(crate) struct Socket { // - Networking Info /// Our discovered direct addresses. direct_addrs: DiscoveredDirectAddrs, + direct_addr_filter: Option, /// Our latest net-report net_report: Watchable<(Option, UpdateReason)>, /// If the last net_report report, reports IPv6 to be available. @@ -508,7 +510,15 @@ impl Socket { /// /// If the direct addresses have changed from the previous set, they are published to /// the address lookup system. - fn store_direct_addresses(&self, addrs: BTreeSet) { + fn store_direct_addresses(&self, mut addrs: BTreeSet) { + if let Some(filter) = &self.direct_addr_filter { + let original = addrs + .iter() + .map(|addr| TransportAddr::Ip(addr.addr)) + .collect::>(); + let retained = filter.apply(&original); + addrs.retain(|addr| retained.contains(&TransportAddr::Ip(addr.addr))); + } let updated = self.direct_addrs.update(addrs); if updated { self.publish_my_addr(); @@ -872,6 +882,7 @@ impl EndpointInner { let span = tracing::Span::current(); let Options { + direct_addr_filter, secret_key, transports: transport_configs, address_lookup_user_data, @@ -991,6 +1002,7 @@ impl EndpointInner { address_lookup_user_data: RwLock::new(address_lookup_user_data), configured_addrs: RwLock::new(configured_addrs), direct_addrs, + direct_addr_filter, net_report: Watchable::new((None, UpdateReason::None)), #[cfg(not(wasm_browser))] dns_resolver: dns_resolver.clone(), @@ -2179,6 +2191,7 @@ mod tests { net_report_config: Default::default(), static_config, configured_addrs: Default::default(), + direct_addr_filter: None, } } @@ -2595,6 +2608,7 @@ mod tests { net_report_config: Default::default(), static_config, configured_addrs: Default::default(), + direct_addr_filter: None, }; let sock = EndpointInner::bind(opts).await?; Ok(sock)