diff --git a/Cargo.lock b/Cargo.lock index 7631e5f..822e269 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6246,6 +6246,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "tsunagi-awg", "tsunagi-core", "tsunagi-tcp-tls", "tsunagi-wg-quic", @@ -6253,6 +6254,26 @@ dependencies = [ "windows-service", ] +[[package]] +name = "tsunagi-awg" +version = "0.1.0" +dependencies = [ + "boringtun", + "bytes", + "chacha20 0.9.1", + "iroh", + "postcard", + "rand 0.10.3", + "serde", + "tempfile", + "tokio", + "tracing", + "tsunagi-core", + "tsunagi-wg-quic", + "tsunagi-wireguard", + "zeroize", +] + [[package]] name = "tsunagi-core" version = "0.1.0" diff --git a/README.md b/README.md index 539f322..d7cd191 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,11 @@ The tray does the same with a checkbox. An exit node sees everything you send through it, so use only members you trust. [Details and per-OS notes](docs/exit-node.md). +The default data transport order is `wg,awg,wg-quic,tcp-tls`. AWG negotiates a +fresh masking profile and outer keys for each pair connection attempt, including +reentry after fallback. Sustained data loss triggers switching independently of +the iroh control connection. [Configuration and limits](docs/amnezia.md). + ## Platforms | Platform | Build | CLI `tsng` | Tray | Exit node: offer / use | Packages | @@ -184,6 +189,7 @@ The [threat model](docs/threat-model.md) says what this does and doesn't protect [Architecture](docs/architecture.md) · [Protocol](docs/protocol.md) · [WireGuard data plane](docs/wireguard.md) · +[Pairwise Amnezia transport](docs/amnezia.md) · [Routing](docs/routing.md) · [Exit nodes](docs/exit-node.md) · [Threat model](docs/threat-model.md) · diff --git a/crates/tsunagi-awg/Cargo.toml b/crates/tsunagi-awg/Cargo.toml new file mode 100644 index 0000000..7522090 --- /dev/null +++ b/crates/tsunagi-awg/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "tsunagi-awg" +version = "0.1.0" +description = "Ephemeral pairwise Amnezia transport for Tsunagi." +edition.workspace = true +rust-version.workspace = true +license = "WTFPL AND Apache-2.0" +repository.workspace = true + +[dependencies] +tsunagi-core = { path = "../tsunagi-core" } +tsunagi-wg-quic = { path = "../tsunagi-wg-quic" } +tsunagi-wireguard = { path = "../tsunagi-wireguard" } +iroh.workspace = true +tokio = { workspace = true, features = ["net", "time", "sync", "macros"] } +serde.workspace = true +postcard.workspace = true +bytes.workspace = true +tracing.workspace = true +rand = "0.10" +zeroize = { version = "1.9", features = ["derive"] } +boringtun = { version = "0.7.1", default-features = false } +chacha20 = "0.9" + +[dev-dependencies] +tsunagi-core = { path = "../tsunagi-core", features = ["testing"] } +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/tsunagi-awg/LICENSE-AmneziaWG-RS b/crates/tsunagi-awg/LICENSE-AmneziaWG-RS new file mode 100644 index 0000000..bc588c9 --- /dev/null +++ b/crates/tsunagi-awg/LICENSE-AmneziaWG-RS @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Friderika Coulson (NanoBee) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/tsunagi-awg/UPSTREAM.md b/crates/tsunagi-awg/UPSTREAM.md new file mode 100644 index 0000000..d5a16c3 --- /dev/null +++ b/crates/tsunagi-awg/UPSTREAM.md @@ -0,0 +1,21 @@ +# Amnezia packet engine + +`src/masking.rs` is adapted from `src/core/amnezia.rs` in +https://github.com/RunaRoo/AmneziaWG-RS at +`51579a7d6e8882d1d8d6d00822f09e18a9423b73`, under Apache-2.0. +The upstream license is retained in `LICENSE-AmneziaWG-RS`. + +The adaptation replaces configuration-file loading with bounded typed pair +profiles, updates RNG calls to rand 0.10, removes an unnecessary unwrap, and +does not embed upstream's Node, global executors, OS interface operations or +detached tasks. Noise/WireGuard is supplied by boringtun, already used by +Tsunagi. Each pair owns one socket and one cancellable packet-engine task. + +This is Tsunagi AWG wire version 1, not a claim of compatibility with external +Amnezia servers. The masking engine substitutes message headers outside the +standard WireGuard state machine and restores them before Noise validation. +There are no remote file paths, shell hooks, or persisted pair profiles. + +The adapted masking source remains Apache-2.0. Newly written Tsunagi code +follows the repository's WTFPL. This package declares both licenses in Cargo +metadata; the workspace license alone does not describe its copied source. diff --git a/crates/tsunagi-awg/src/lib.rs b/crates/tsunagi-awg/src/lib.rs new file mode 100644 index 0000000..54f842a --- /dev/null +++ b/crates/tsunagi-awg/src/lib.rs @@ -0,0 +1,16 @@ +//! Pairwise, ephemeral Amnezia packet transport with independent end-to-end tunnels. +//! All negotiation travels over authenticated Tsunagi control sessions. +mod masking; +mod plugin; +pub mod profile; +mod session; +mod stun; +mod transport; + +pub use plugin::AwgPlugin; +pub use transport::{AwgTransport, PairOverview}; + +/// Protocol identifier and default second-choice direct transport. +pub const AWG_PROTOCOL: &str = "awg"; +/// Tsunagi AWG wire version, independent of software releases. +pub const AWG_WIRE_VERSION: u16 = 1; diff --git a/crates/tsunagi-awg/src/masking.rs b/crates/tsunagi-awg/src/masking.rs new file mode 100644 index 0000000..7911726 --- /dev/null +++ b/crates/tsunagi-awg/src/masking.rs @@ -0,0 +1,336 @@ +// Adapted from RunaRoo/AmneziaWG-RS, commit 51579a7d6e8882d1d8d6d00822f09e18a9423b73. +// Apache-2.0; see LICENSE-AmneziaWG-RS and UPSTREAM.md. +#![allow(dead_code, missing_docs)] +// AmneziaWG configuration, header protection, and traffic shaping. + +use bytes::{Bytes, BytesMut}; +use chacha20::{ + ChaCha20, + cipher::{KeyIvInit, StreamCipher}, +}; +use rand::RngExt; + +pub const HEADER_PROTECTION_NONCE_SIZE: usize = 12; +pub const DEFAULT_UDP_WINDOW: usize = 500; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Range { + pub min: u32, + pub max: u32, +} + +impl Range { + pub fn contains(&self, val: u32) -> bool { + val >= self.min && val <= self.max + } + + pub fn random(&self) -> u32 { + if self.min >= self.max { + self.min + } else { + rand::rng().random_range(self.min..=self.max) + } + } + + pub fn is_zero(&self) -> bool { + self.min == 0 && self.max == 0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AwgKind { + Initiation, + Response, + Cookie, + Transport, +} + +impl AwgKind { + pub fn fixed_size(self) -> usize { + match self { + Self::Initiation => 148, + Self::Response => 92, + Self::Cookie => 64, + Self::Transport => 32, + } + } + + fn standard_type(self) -> u32 { + match self { + Self::Initiation => 1, + Self::Response => 2, + Self::Cookie => 3, + Self::Transport => 4, + } + } +} + +#[derive(Debug)] +pub struct DecodedPacket { + pub kind: AwgKind, + pub message: Bytes, + pub wire_len: usize, +} + +pub struct AmneziaConfig { + pub jc: u32, + pub jmin: usize, + pub jmax: usize, + pub s1: usize, + pub s2: usize, + pub s3: usize, + pub s4: usize, + pub h1: Range, + pub h2: Range, + pub h3: Range, + pub h4: Range, + pub i1: String, + pub i2: String, + pub i3: String, + pub i4: String, + pub i5: String, + pub header_protection_key: Option<[u8; 32]>, + pub content_padding_addition: Range, + pub random_trailers: bool, + pub mtu: usize, +} + +impl AmneziaConfig { + pub fn random_header(&self, range: &Range) -> u32 { + range.random() + } + + pub fn random_padding(&self, len: usize) -> Vec { + let mut padding = vec![0u8; len]; + if len > 0 { + rand::fill(&mut padding); + } + padding + } + + fn shape(&self, kind: AwgKind) -> (usize, Range) { + match kind { + AwgKind::Initiation => (self.s1, self.h1), + AwgKind::Response => (self.s2, self.h2), + AwgKind::Cookie => (self.s3, self.h3), + AwgKind::Transport => (self.s4, self.h4), + } + } + + pub fn encode(&self, kind: AwgKind, payload: Bytes, udp_window: usize) -> Bytes { + let (padding_len, _) = self.shape(kind); + let trailer_len = if kind != AwgKind::Transport && self.random_trailers { + let base = padding_len.saturating_add(payload.len()); + let room = udp_window.max(DEFAULT_UDP_WINDOW).saturating_sub(base); + if room == 0 { + 0 + } else { + rand::rng().random_range(0..room) + } + } else { + 0 + }; + + let mut out = BytesMut::zeroed(padding_len + payload.len() + trailer_len); + if padding_len > 0 { + rand::fill(&mut out[..padding_len]); + } + out[padding_len..padding_len + payload.len()].copy_from_slice(&payload); + if trailer_len > 0 { + rand::fill(&mut out[padding_len + payload.len()..]); + } + + if self.header_protection_key.is_some() { + let protected_len = if kind == AwgKind::Transport { + 16 + } else { + kind.fixed_size() + }; + self.apply_header_cipher(&mut out, padding_len, protected_len); + } + + out.freeze() + } + + pub fn decode(&self, datagram: &Bytes) -> Option { + if datagram.len() < 4 { + return None; + } + + for kind in [ + AwgKind::Initiation, + AwgKind::Response, + AwgKind::Cookie, + AwgKind::Transport, + ] { + let (padding_len, headers) = self.shape(kind); + let minimum = padding_len + kind.fixed_size(); + let size_ok = match kind { + AwgKind::Transport | AwgKind::Initiation => datagram.len() >= minimum, + AwgKind::Response | AwgKind::Cookie if self.random_trailers => { + datagram.len() >= minimum + } + AwgKind::Response | AwgKind::Cookie => datagram.len() == minimum, + }; + if !size_ok || datagram.len() < padding_len + 4 { + continue; + } + + let protected_header = + u32::from_le_bytes(datagram[padding_len..padding_len + 4].try_into().ok()?); + let candidate = if self.header_protection_key.is_some() { + protected_header ^ self.header_type_hash(datagram)? + } else { + protected_header + }; + if !headers.contains(candidate) { + continue; + } + + let message_len = match kind { + AwgKind::Transport | AwgKind::Initiation => datagram.len() - padding_len, + AwgKind::Response | AwgKind::Cookie => kind.fixed_size(), + }; + let mut payload = BytesMut::from(&datagram[padding_len..padding_len + message_len]); + if self.header_protection_key.is_some() { + let protected_len = if kind == AwgKind::Transport { + 16 + } else { + kind.fixed_size() + }; + self.apply_header_cipher_with_nonce( + &mut payload, + 0, + protected_len, + &datagram[..HEADER_PROTECTION_NONCE_SIZE], + ); + } + + return Some(DecodedPacket { + kind, + message: payload.freeze(), + wire_len: datagram.len(), + }); + } + + None + } + + pub fn prepare_transport_plaintext(&self, packet: Bytes, udp_window: usize) -> Bytes { + if packet.is_empty() { + return packet; + } + + let requested = if !self.content_padding_addition.is_zero() { + self.content_padding_addition.random() as usize + } else if self.random_trailers { + let wire_without_padding = self.s4 + AwgKind::Transport.fixed_size() + packet.len(); + let room = udp_window + .max(DEFAULT_UDP_WINDOW) + .saturating_sub(wire_without_padding); + if room == 0 { + 0 + } else { + rand::rng().random_range(0..room) + } + } else { + let aligned = (packet.len() + 15) & !15; + aligned.saturating_sub(packet.len()) + }; + + let padding = self.clamp_padding_to_mtu(packet.len(), requested); + if padding == 0 { + return packet; + } + + let mut out = BytesMut::with_capacity(packet.len() + padding); + out.extend_from_slice(&packet); + out.resize(packet.len() + padding, 0); + out.freeze() + } + + pub fn transport_content_length(&self, plaintext: &[u8]) -> Option { + if plaintext.is_empty() || plaintext[0] == 0 { + return Some(0); + } + match plaintext[0] >> 4 { + 4 if plaintext.len() >= 20 => { + let total = u16::from_be_bytes([plaintext[2], plaintext[3]]) as usize; + (total >= 20 && total <= plaintext.len()).then_some(total) + } + 6 if plaintext.len() >= 40 => { + let payload = u16::from_be_bytes([plaintext[4], plaintext[5]]) as usize; + let total = 40 + payload; + (total <= plaintext.len()).then_some(total) + } + _ => None, + } + } + + fn clamp_padding_to_mtu(&self, packet_len: usize, requested: usize) -> usize { + if self.mtu == 0 { + return requested; + } + let normalized = if packet_len > self.mtu { + packet_len % self.mtu + } else { + packet_len + }; + requested.min(self.mtu.saturating_sub(normalized)) + } + + fn header_type_hash(&self, datagram: &Bytes) -> Option { + if datagram.len() < HEADER_PROTECTION_NONCE_SIZE { + return None; + } + let mut hash = [0u8; 4]; + self.apply_keystream(&mut hash, &datagram[..HEADER_PROTECTION_NONCE_SIZE]); + Some(u32::from_le_bytes(hash)) + } + + fn apply_header_cipher(&self, packet: &mut [u8], offset: usize, len: usize) { + if packet.len() < HEADER_PROTECTION_NONCE_SIZE || packet.len() < offset + len { + return; + } + let nonce: [u8; HEADER_PROTECTION_NONCE_SIZE] = + match packet[..HEADER_PROTECTION_NONCE_SIZE].try_into() { + Ok(nonce) => nonce, + Err(_) => return, + }; + self.apply_header_cipher_with_nonce(packet, offset, len, &nonce); + } + + fn apply_header_cipher_with_nonce( + &self, + packet: &mut [u8], + offset: usize, + len: usize, + nonce: &[u8], + ) { + if packet.len() < offset + len { + return; + } + self.apply_keystream(&mut packet[offset..offset + len], nonce); + } + + fn apply_keystream(&self, data: &mut [u8], nonce: &[u8]) { + let Some(key) = self.header_protection_key.as_ref() else { + return; + }; + if nonce.len() != HEADER_PROTECTION_NONCE_SIZE { + return; + } + let Ok(nonce) = <&[u8; HEADER_PROTECTION_NONCE_SIZE]>::try_from(nonce) else { + return; + }; + let mut cipher = ChaCha20::new(key.into(), nonce.into()); + cipher.apply_keystream(data); + } +} + +impl Drop for AmneziaConfig { + fn drop(&mut self) { + use zeroize::Zeroize; + self.header_protection_key.zeroize(); + } +} diff --git a/crates/tsunagi-awg/src/plugin.rs b/crates/tsunagi-awg/src/plugin.rs new file mode 100644 index 0000000..6733e8e --- /dev/null +++ b/crates/tsunagi-awg/src/plugin.rs @@ -0,0 +1,132 @@ +//! End-to-end encryption remains independent of physical Amnezia pair links. +use crate::{AWG_PROTOCOL, AWG_WIRE_VERSION, AwgTransport}; +use iroh::EndpointId; +use std::sync::Arc; +use tsunagi::dataplane::transport::SharedLink; +use tsunagi::dataplane::{IpPlugin, PluginCapability, PluginContext, PluginError, ProtocolOption}; +use tsunagi::state::Ipv4Range; +use tsunagi::{BoxFuture, NetworkId}; +use tsunagi_wg_quic::{NetworkOverview, WireguardConfig, WireguardPlugin}; + +/// AWG capability and an independent end-to-end WireGuard engine. +/// Pair transport keys/profiles never enter its durable end-to-end key store. +#[derive(Debug)] +pub struct AwgPlugin { + inner: Arc, + transport: Arc, +} +impl AwgPlugin { + /// Uses standard tunnel MTU/keepalive options; pair ports and profiles are ephemeral. + pub const OPTIONS: &'static [ProtocolOption] = &[ + ProtocolOption { + key: "keepalive", + value: "SECONDS", + help: "end-to-end tunnel keepalive; 0 disables it", + default: Some("25"), + }, + ProtocolOption { + key: "mtu", + value: "BYTES", + help: "maximum overlay packet size", + default: Some("1280"), + }, + ProtocolOption { + key: "stun", + value: "HOST:PORT|off", + help: "UDP public mapping discovery; disabled in local reach mode", + default: Some("stun.l.google.com:19302"), + }, + ]; + /// Opens the end-to-end engine, retaining separate pair transport state. + pub async fn open( + mut config: WireguardConfig, + transport: Arc, + ) -> Result, PluginError> { + config.protocol = AWG_PROTOCOL; + Ok(Arc::new(Self { + inner: WireguardPlugin::open(config).await?, + transport, + })) + } + /// Reports end-to-end tunnel health and traffic counters. + pub fn overview(&self, network: NetworkId) -> Option { + self.inner.overview(network) + } +} +impl IpPlugin for AwgPlugin { + fn protocol_id(&self) -> &str { + AWG_PROTOCOL + } + fn protocol_version(&self) -> u16 { + AWG_WIRE_VERSION + } + fn options(&self) -> &'static [ProtocolOption] { + Self::OPTIONS + } + fn attach(&self, context: PluginContext) { + self.inner.attach(context); + } + fn local_capability( + &self, + network: NetworkId, + ) -> Result, PluginError> { + self.inner.local_capability(network).map(|cap| { + cap.map(|mut cap| { + cap.protocol = AWG_PROTOCOL.into(); + cap.version = AWG_WIRE_VERSION; + cap + }) + }) + } + fn on_network_activated(&self, network: NetworkId) { + self.inner.on_network_activated(network); + } + fn on_peer_capability( + &self, + network: NetworkId, + peer: EndpointId, + capability: &PluginCapability, + ) -> Result<(), PluginError> { + if capability.protocol != AWG_PROTOCOL + || capability.version != AWG_WIRE_VERSION + || !capability.enabled + { + return Err(PluginError::Rejected("unsupported AWG capability".into())); + } + self.inner.on_peer_capability(network, peer, capability)?; + self.transport.allow_peer(network, peer); + Ok(()) + } + fn on_address_allocation( + &self, + network: NetworkId, + range: Ipv4Range, + allocations: &[(EndpointId, std::net::Ipv4Addr)], + ) { + self.inner + .on_address_allocation(network, range, allocations); + } + fn carry(&self, network: NetworkId, peer: EndpointId, packet: bytes::Bytes) -> bool { + self.inner.carry(network, peer, packet) + } + fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) { + self.inner.on_peer_link(network, peer, link); + } + fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) { + self.transport.forget_peer(network, peer); + self.inner.on_peer_gone(network, peer); + } + fn on_network_deactivated(&self, network: NetworkId) { + self.transport.stop_network(network); + self.inner.on_network_deactivated(network); + } + fn on_network_forgotten(&self, network: NetworkId) { + self.inner.on_network_forgotten(network); + } + fn shutdown(&self) -> BoxFuture<'_, ()> { + Box::pin(async move { + self.transport.shutdown(); + self.inner.shutdown().await; + }) + } +} diff --git a/crates/tsunagi-awg/src/profile.rs b/crates/tsunagi-awg/src/profile.rs new file mode 100644 index 0000000..9ef8284 --- /dev/null +++ b/crates/tsunagi-awg/src/profile.rs @@ -0,0 +1,226 @@ +//! Bounded profiles, regenerated for every pair connection attempt. +use crate::masking::{AmneziaConfig, AwgKind, Range}; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; + +/// An ephemeral masking profile. Secret fields are redacted and erased on drop. +#[derive(Clone, Serialize, Deserialize, zeroize::ZeroizeOnDrop)] +pub struct PairProfile { + id: [u8; 16], + prefix: u16, + headers: [[u32; 2]; 4], + header_key: [u8; 32], + pub(crate) psk: [u8; 32], + junk_count: u8, + junk_min: u16, + junk_max: u16, + padding: u16, + signature: u8, +} +impl std::fmt::Debug for PairProfile { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("PairProfile([REDACTED])") + } +} +impl PairProfile { + /// Generates a new safe profile; no deterministic seed or durable state. + pub fn generate() -> Self { + let headers = std::array::from_fn(|i| { + let base = (i as u32) * 0x4000_0000 + rand::random_range(1024..0x3fff_0000); + [base, base + rand::random_range(64..1024)] + }); + Self { + id: rand::random(), + prefix: rand::random_range(12..49), + headers, + header_key: rand::random(), + psk: rand::random(), + junk_count: rand::random_range(4..9), + junk_min: rand::random_range(24..65), + junk_max: rand::random_range(80..161), + padding: rand::random_range(16..65), + signature: rand::random_range(0..2), + } + } + /// Public attempt identifier, useful for distinguishing retries without revealing secrets. + pub fn id(&self) -> [u8; 16] { + self.id + } + /// Header key, for packet engines and protocol tests; never log this value. + pub fn header_key(&self) -> &[u8; 32] { + &self.header_key + } + /// Rejects invalid remote profiles before allocating packets or entering the codec. + pub fn validate(&self) -> Result<(), &'static str> { + if !(12..=48).contains(&self.prefix) + || !(4..=8).contains(&self.junk_count) + || self.junk_min < 24 + || self.junk_min > self.junk_max + || self.junk_max > 160 + || self.padding > 64 + || self.signature > 1 + || self.header_key == [0; 32] + || self.psk == [0; 32] + { + return Err("invalid AWG profile bounds"); + } + for (i, [lo, hi]) in self.headers.iter().copied().enumerate() { + if lo < 5 || lo > hi || hi - lo > 1024 { + return Err("invalid AWG header range"); + } + for [other_lo, other_hi] in &self.headers[..i] { + if lo <= *other_hi && *other_lo <= hi { + return Err("overlapping AWG header ranges"); + } + } + } + Ok(()) + } + fn config(&self) -> AmneziaConfig { + let range = |i: usize| Range { + min: self.headers[i][0], + max: self.headers[i][1], + }; + AmneziaConfig { + jc: self.junk_count.into(), + jmin: self.junk_min.into(), + jmax: self.junk_max.into(), + s1: self.prefix.into(), + s2: self.prefix.into(), + s3: self.prefix.into(), + s4: self.prefix.into(), + h1: range(0), + h2: range(1), + h3: range(2), + h4: range(3), + i1: String::new(), + i2: String::new(), + i3: String::new(), + i4: String::new(), + i5: String::new(), + header_protection_key: Some(self.header_key), + content_padding_addition: Range { + min: 0, + max: self.padding.into(), + }, + random_trailers: true, + mtu: 1050, + } + } + /// Masks one standard WireGuard message; invalid types and oversized packets are refused. + pub fn encode(&self, raw: &[u8]) -> Option { + let kind = match raw.get(..4)? { + [1, 0, 0, 0] => AwgKind::Initiation, + [2, 0, 0, 0] => AwgKind::Response, + [3, 0, 0, 0] => AwgKind::Cookie, + [4, 0, 0, 0] => AwgKind::Transport, + _ => return None, + }; + if raw.len() < kind.fixed_size() || raw.len() > 1050 { + return None; + } + let config = self.config(); + let range = match kind { + AwgKind::Initiation => config.h1, + AwgKind::Response => config.h2, + AwgKind::Cookie => config.h3, + AwgKind::Transport => config.h4, + }; + let mut payload = raw.to_vec(); + payload[..4].copy_from_slice(&range.random().to_le_bytes()); + Some(config.encode(kind, Bytes::from(payload), 1200)) + } + /// Unmasks a packet and restores standard headers before WireGuard validation. + pub fn decode(&self, wire: &[u8]) -> Option> { + if wire.len() > 1200 { + return None; + } + let decoded = self.config().decode(&Bytes::copy_from_slice(wire))?; + let (kind, len) = match decoded.kind { + AwgKind::Initiation => (1, 148), + AwgKind::Response => (2, 92), + AwgKind::Cookie => (3, 64), + AwgKind::Transport => (4, decoded.message.len()), + }; + let mut message = decoded.message.get(..len)?.to_vec(); + message[..4].copy_from_slice(&(kind as u32).to_le_bytes()); + Some(message) + } + pub(crate) fn pad(&self, packet: Bytes) -> Bytes { + self.config().prepare_transport_plaintext(packet, 1200) + } + /// Generates structured signature packets with fresh fields, never static captures. + /// These target the other Tsunagi peer, not DNS or SIP services. + pub fn signature_packets(&self) -> Vec> { + if self.signature == 0 { + // A complete DNS query. Counts, label lengths and qtype remain valid. + let label = format!("n{:016x}", rand::random::()); + let mut packet = Vec::from(rand::random::().to_be_bytes()); + packet.extend_from_slice(&[1, 0, 0, 1, 0, 0, 0, 0, 0, 0]); + packet.push(label.len() as u8); + packet.extend_from_slice(label.as_bytes()); + packet.extend_from_slice(b"\x07example\x03net\0\0"); + packet.push(if rand::random() { 1 } else { 28 }); + packet.extend_from_slice(&[0, 1]); + vec![packet] + } else { + let id = rand::random::(); + vec![format!("OPTIONS sip:n{id:x}@example.net SIP/2.0\r\nVia: SIP/2.0/UDP example.net;branch=z9hG4bK{id:x}\r\nMax-Forwards: 70\r\nFrom: ;tag={id:x}\r\nTo: \r\nCall-ID: {id:x}@example.net\r\nCSeq: 1 OPTIONS\r\nContent-Length: 0\r\n\r\n").into_bytes()] + } + } + pub(crate) fn junk_packets(&self) -> Vec> { + (0..self.junk_count) + .map(|_| { + let mut packet = vec![0; rand::random_range(self.junk_min..=self.junk_max).into()]; + rand::fill(&mut packet[..]); + packet + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[test] + fn unsafe_peer_profiles_are_rejected_before_the_codec() { + let valid = PairProfile::generate(); + let mut bad = valid.clone(); + bad.prefix = 11; + assert!(bad.validate().is_err()); + let mut bad = valid.clone(); + bad.headers[1] = bad.headers[0]; + assert!(bad.validate().is_err()); + let mut bad = valid.clone(); + bad.junk_max = u16::MAX; + assert!(bad.validate().is_err()); + let mut bad = valid.clone(); + bad.padding = u16::MAX; + assert!(bad.validate().is_err()); + let mut bad = valid.clone(); + bad.psk = [0; 32]; + assert!(bad.validate().is_err()); + let mut bad = valid.clone(); + bad.signature = 2; + assert!(bad.validate().is_err()); + } + #[test] + fn signature_templates_have_consistent_framing_and_fresh_fields() { + let mut profile = PairProfile::generate(); + profile.signature = 0; + let dns = profile.signature_packets().remove(0); + assert_eq!(&dns[2..12], &[1, 0, 0, 1, 0, 0, 0, 0, 0, 0]); + let mut at = 12; + while dns[at] != 0 { + at += 1 + usize::from(dns[at]); + } + assert_eq!(at + 5, dns.len()); + assert_eq!(&dns[dns.len() - 2..], &[0, 1]); + profile.signature = 1; + let sip = profile.signature_packets().remove(0); + assert!(sip.starts_with(b"OPTIONS sip:")); + assert!(sip.ends_with(b"Content-Length: 0\r\n\r\n")); + assert_ne!(sip, profile.signature_packets().remove(0)); + } +} diff --git a/crates/tsunagi-awg/src/session.rs b/crates/tsunagi-awg/src/session.rs new file mode 100644 index 0000000..fbf1b81 --- /dev/null +++ b/crates/tsunagi-awg/src/session.rs @@ -0,0 +1,402 @@ +//! One cancellable actor and UDP socket per pair. All payload and health frames +//! are inside the Amnezia-masked authenticated WireGuard tunnel. +use crate::profile::PairProfile; +use boringtun::noise::{Tunn, TunnResult}; +use bytes::Bytes; +use iroh::EndpointId; +use std::collections::{HashMap, VecDeque}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use tokio::net::UdpSocket; +use tokio::sync::{mpsc, watch}; +use tsunagi::dataplane::transport::{PacketLink, TransportError}; +use tsunagi::{BoxFuture, NetworkId}; + +pub(crate) const MAX_PAYLOAD: usize = tsunagi::config::MAX_DATA_DATAGRAM; +const CHUNK: usize = 700; +const QUEUE: usize = 64; +const WINDOW: Duration = Duration::from_secs(20); + +pub(crate) struct PairLink { + network: NetworkId, + peer: EndpointId, + pub(crate) attempt: [u8; 16], + tx: mpsc::Sender, + rx: tokio::sync::Mutex>, + stop: watch::Sender, + closed: AtomicBool, + remote: Mutex>, + reason: Mutex>, +} +impl std::fmt::Debug for PairLink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AwgPair") + .field("network", &self.network) + .field("peer", &self.peer) + .finish_non_exhaustive() + } +} +impl PairLink { + pub(crate) fn has_authenticated_path(&self) -> bool { + match self.remote.lock() { + Ok(remote) => remote.is_some(), + Err(_) => false, + } + } +} +impl PacketLink for PairLink { + fn network(&self) -> NetworkId { + self.network + } + fn peer(&self) -> EndpointId { + self.peer + } + fn max_datagram_size(&self) -> usize { + MAX_PAYLOAD + } + fn send(&self, payload: Bytes) -> Result<(), TransportError> { + if self.is_closed() { + return Err(TransportError::Closed); + } + if payload.len() > MAX_PAYLOAD { + return Err(TransportError::TooLarge { + size: payload.len(), + limit: MAX_PAYLOAD, + }); + } + self.tx + .try_send(payload) + .map_err(|_| TransportError::Other("AWG send queue full or closed".into())) + } + fn recv(&self) -> BoxFuture<'_, Option> { + Box::pin(async move { + let mut stop = self.stop.subscribe(); + if *stop.borrow() { + return None; + } + let mut rx = self.rx.lock().await; + tokio::select! {biased; _=stop.changed()=>None,packet=rx.recv()=>packet} + }) + } + fn closed(&self) -> BoxFuture<'_, ()> { + Box::pin(async move { + let mut stop = self.stop.subscribe(); + if !*stop.borrow() { + let _ = stop.changed().await; + } + }) + } + fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) || *self.stop.borrow() + } + fn close(&self) { + self.stop.send_replace(true); + } + fn failure_reason(&self) -> Option { + match self.reason.lock() { + Ok(reason) => reason.clone(), + Err(_) => None, + } + } + fn path_description(&self) -> String { + match self.remote.lock() { + Ok(addr) => match *addr { + Some(addr) => format!("awg UDP via {addr}"), + None => "awg UDP, establishing".into(), + }, + Err(_) => "awg UDP".into(), + } + } +} +impl Drop for PairLink { + fn drop(&mut self) { + self.stop.send_replace(true); + } +} + +pub(crate) fn start( + network: NetworkId, + peer: EndpointId, + socket: UdpSocket, + tunn: Tunn, + profile: PairProfile, + candidates: Vec, + parent: watch::Receiver, +) -> Arc { + let (tx, rx) = mpsc::channel(QUEUE); + let (in_tx, in_rx) = mpsc::channel(QUEUE); + let (stop, _) = watch::channel(false); + let link = Arc::new(PairLink { + network, + peer, + attempt: profile.id(), + tx, + rx: tokio::sync::Mutex::new(in_rx), + stop: stop.clone(), + closed: AtomicBool::new(false), + remote: Mutex::new(None), + reason: Mutex::new(None), + }); + let weak = Arc::downgrade(&link); + tokio::spawn(async move { + let mut engine = Engine { + stop: stop.subscribe(), + socket, + tunn, + profile, + candidates, + remote: None, + inbox: in_tx, + rx, + counter: rand::random(), + sequence: 0, + pending: VecDeque::new(), + samples: VecDeque::new(), + assemblies: HashMap::new(), + started: Instant::now(), + last_authenticated: Instant::now(), + }; + engine.run(parent, &weak).await; + if let Some(link) = weak.upgrade() { + link.closed.store(true, Ordering::Release); + link.stop.send_replace(true); + } + }); + link +} + +struct Assembly { + created: Instant, + count: u16, + chunks: Vec>, +} +struct Engine { + stop: watch::Receiver, + socket: UdpSocket, + tunn: Tunn, + profile: PairProfile, + candidates: Vec, + remote: Option, + inbox: mpsc::Sender, + rx: mpsc::Receiver, + counter: u64, + sequence: u64, + pending: VecDeque<(u64, Instant)>, + samples: VecDeque, + assemblies: HashMap, + started: Instant, + last_authenticated: Instant, +} +enum ResultOwned { + Wire(Vec), + Plain(Vec), + Done, +} +fn own(result: TunnResult<'_>) -> ResultOwned { + match result { + TunnResult::WriteToNetwork(p) => ResultOwned::Wire(p.to_vec()), + TunnResult::WriteToTunnelV4(p, _) => ResultOwned::Plain(p.to_vec()), + _ => ResultOwned::Done, + } +} +fn ip(payload: &[u8]) -> Bytes { + let mut packet = vec![0; 20]; + packet[0] = 0x45; + packet[8] = 64; + packet[9] = 253; + packet[12..16].copy_from_slice(&Ipv4Addr::new(192, 0, 2, 1).octets()); + packet[16..20].copy_from_slice(&Ipv4Addr::new(192, 0, 2, 2).octets()); + packet[2..4].copy_from_slice(&((20 + payload.len()) as u16).to_be_bytes()); + let sum = packet + .as_chunks::<2>() + .0 + .iter() + .map(|b| u32::from(u16::from_be_bytes([b[0], b[1]]))) + .sum::(); + let sum = (sum & 0xffff) + (sum >> 16); + packet[10..12].copy_from_slice(&(!(sum as u16)).to_be_bytes()); + packet.extend_from_slice(payload); + Bytes::from(packet) +} +impl Engine { + async fn run(&mut self, mut stop: watch::Receiver, link: &std::sync::Weak) { + if *self.stop.borrow() || *stop.borrow() { + return; + } + let mut buffer = [0; 1201]; + let mut scratch = [0; 4096]; + let mut timer = tokio::time::interval(Duration::from_millis(250)); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_probe = Instant::now() - Duration::from_secs(2); + loop { + tokio::select! { + biased; + _=stop.changed()=>break, + _=self.stop.changed()=>break, + _=timer.tick()=>{ + let result=own(self.tunn.update_timers(&mut scratch));self.process(result,None).await; + if last_probe.elapsed()>=Duration::from_secs(1) { + last_probe=Instant::now();self.sequence=self.sequence.wrapping_add(1); + self.pending.push_back((self.sequence,Instant::now())); + let mut probe=vec![1];probe.extend_from_slice(&self.sequence.to_be_bytes()); + probe.resize(713,0);rand::fill(&mut probe[9..]);self.encrypt(&probe).await; + } + while self.pending.front().is_some_and(|(_,at)|at.elapsed()>=Duration::from_secs(3)) { + self.pending.pop_front();self.samples.push_back(false); + } + while self.samples.len()>20 {self.samples.pop_front();} + self.assemblies.retain(|_,a|a.created.elapsed()WINDOW && (self.last_authenticated.elapsed()>Duration::from_secs(10) + || (self.samples.len()>=15&&self.samples.iter().filter(|ok|!**ok).count()*100>=self.samples.len()*60)) { + if let Some(link)=link.upgrade() && let Ok(mut reason)=link.reason.lock() {*reason=Some("AWG data delivery checks: persistent loss or timeout".into());} + break; + } + } + packet=self.socket.recv_from(&mut buffer)=>{ + if let Ok((len,source))=packet && let Some(message)=self.profile.decode(&buffer[..len]) { + let result=own(self.tunn.decapsulate(None,&message,&mut scratch)); + if matches!(result,ResultOwned::Plain(_)) { + self.remote=Some(source);self.last_authenticated=Instant::now(); + if let Some(link)=link.upgrade() && let Ok(mut remote)=link.remote.lock(){*remote=Some(source);} + } + self.process(result,Some(source)).await; + for _ in 0..16 { + let result=own(self.tunn.decapsulate(None,&[],&mut scratch)); + if matches!(result,ResultOwned::Done){break;} + self.process(result,Some(source)).await; + } + } + } + packet=self.rx.recv()=>{ + let Some(packet)=packet else{break;}; + let count=packet.len().max(1).div_ceil(CHUNK) as u16; + self.counter=self.counter.wrapping_add(1); + for (index,chunk) in packet.chunks(CHUNK).enumerate() { + let mut payload=vec![3];payload.extend_from_slice(&self.counter.to_be_bytes()); + payload.extend_from_slice(&(index as u16).to_be_bytes());payload.extend_from_slice(&count.to_be_bytes()); + payload.extend_from_slice(chunk);self.encrypt(&payload).await; + // Large logical datagrams must let local readers and other + // pairs drain their bounded socket/command queues. + if (index + 1).is_multiple_of(16) {tokio::task::yield_now().await;} + } + } + + } + } + } + async fn encrypt(&mut self, payload: &[u8]) { + let packet = self.profile.pad(ip(payload)); + let mut scratch = [0; 4096]; + let result = own(self.tunn.encapsulate(&packet, &mut scratch)); + self.process(result, None).await; + } + async fn wire(&self, raw: &[u8], target: Option) { + let Some(wire) = self.profile.encode(raw) else { + return; + }; + let targets: Vec<_> = target + .or(self.remote) + .map(|a| vec![a]) + .unwrap_or_else(|| self.candidates.clone()); + for addr in targets { + if raw.first() == Some(&1) { + for signature in self.profile.signature_packets() { + let _ = self.send_wire(&signature, addr).await; + } + for junk in self.profile.junk_packets() { + let _ = self.send_wire(&junk, addr).await; + } + } + let _ = self.send_wire(&wire, addr).await; + } + } + async fn send_wire(&self, bytes: &[u8], address: SocketAddr) { + let mut stop = self.stop.clone(); + if *stop.borrow() { + return; + } + tokio::select! {biased; _=stop.changed()=>{}, _=tokio::time::timeout(Duration::from_millis(100),self.socket.send_to(bytes,address))=>{}} + } + async fn process(&mut self, result: ResultOwned, target: Option) { + match result { + ResultOwned::Wire(raw) => self.wire(&raw, target).await, + ResultOwned::Plain(packet) => { + let Some(payload) = packet.get(20..) else { + return; + }; + let Some(seq) = payload + .get(1..9) + .and_then(|b| b.try_into().ok()) + .map(u64::from_be_bytes) + else { + return; + }; + match payload.first() { + Some(1) => { + let mut pong = vec![2]; + pong.extend_from_slice(&seq.to_be_bytes()); + pong.resize(713, 0); + let packet = self.profile.pad(ip(&pong)); + let mut scratch = [0; 4096]; + if let ResultOwned::Wire(raw) = + own(self.tunn.encapsulate(&packet, &mut scratch)) + { + self.wire(&raw, target).await; + } + } + Some(2) => { + if let Some(index) = + self.pending.iter().position(|(pending, _)| *pending == seq) + { + self.pending.remove(index); + self.samples.push_back(true); + } + } + Some(3) => self.fragment(seq, payload), + _ => {} + } + } + ResultOwned::Done => {} + } + } + fn fragment(&mut self, id: u64, payload: &[u8]) { + let Some(header) = payload.get(9..13) else { + return; + }; + let index = u16::from_be_bytes([header[0], header[1]]); + let count = u16::from_be_bytes([header[2], header[3]]); + let chunk = &payload[13..]; + if count == 0 + || usize::from(count) > MAX_PAYLOAD.div_ceil(CHUNK) + || index >= count + || chunk.len() > CHUNK + { + return; + } + if !self.assemblies.contains_key(&id) && self.assemblies.len() >= 8 { + return; + } + let assembly = self.assemblies.entry(id).or_insert_with(|| Assembly { + created: Instant::now(), + count, + chunks: vec![None; count.into()], + }); + if assembly.count != count { + return; + } + assembly.chunks[usize::from(index)] = Some(Bytes::copy_from_slice(chunk)); + if assembly.chunks.iter().all(Option::is_some) + && let Some(assembly) = self.assemblies.remove(&id) + { + let mut packet = Vec::new(); + for chunk in assembly.chunks.into_iter().flatten() { + packet.extend_from_slice(&chunk); + } + if packet.len() <= MAX_PAYLOAD { + let _ = self.inbox.try_send(Bytes::from(packet)); + } + } + } +} diff --git a/crates/tsunagi-awg/src/stun.rs b/crates/tsunagi-awg/src/stun.rs new file mode 100644 index 0000000..81d9c93 --- /dev/null +++ b/crates/tsunagi-awg/src/stun.rs @@ -0,0 +1,101 @@ +//! Optional, bounded server-reflexive discovery on the pair's own UDP socket. +//! A STUN answer is a candidate only; WireGuard authentication verifies a path. +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::Duration; +use tokio::net::UdpSocket; +const COOKIE: [u8; 4] = [0x21, 0x12, 0xa4, 0x42]; +pub(crate) async fn mapped(socket: &UdpSocket, server: &str) -> Option { + tokio::time::timeout(Duration::from_millis(1200), async { + let address = tokio::net::lookup_host(server) + .await + .ok()? + .find(SocketAddr::is_ipv4)?; + let transaction: [u8; 12] = rand::random(); + let mut request = [0; 20]; + request[..2].copy_from_slice(&[0, 1]); + request[4..8].copy_from_slice(&COOKIE); + request[8..].copy_from_slice(&transaction); + socket.send_to(&request, address).await.ok()?; + let mut bytes = [0; 1024]; + loop { + let (len, source) = socket.recv_from(&mut bytes).await.ok()?; + if source == address + && let Some(mapped) = parse(&bytes[..len], &transaction) + { + return Some(mapped); + } + } + }) + .await + .ok() + .flatten() +} +fn parse(bytes: &[u8], transaction: &[u8; 12]) -> Option { + if bytes.len() < 20 + || bytes[..2] != [1, 1] + || bytes[4..8] != COOKIE + || &bytes[8..20] != transaction + { + return None; + } + let declared = u16::from_be_bytes(bytes[2..4].try_into().ok()?) as usize; + if !declared.is_multiple_of(4) || declared != bytes.len() - 20 { + return None; + } + let mut at = 20; + while at + 4 <= bytes.len() { + let kind = u16::from_be_bytes(bytes[at..at + 2].try_into().ok()?); + let len = u16::from_be_bytes(bytes[at + 2..at + 4].try_into().ok()?) as usize; + at += 4; + let attribute = bytes.get(at..at.checked_add(len)?)?; + if kind == 0x0020 && attribute.len() == 8 && attribute[0..2] == [0, 1] { + let port = u16::from_be_bytes([attribute[2] ^ COOKIE[0], attribute[3] ^ COOKIE[1]]); + let ip = Ipv4Addr::new( + attribute[4] ^ COOKIE[0], + attribute[5] ^ COOKIE[1], + attribute[6] ^ COOKIE[2], + attribute[7] ^ COOKIE[3], + ); + if port == 0 || ip.is_unspecified() || ip.is_multicast() || ip.is_broadcast() { + return None; + } + return Some(SocketAddr::new(IpAddr::V4(ip), port)); + } + at = at.checked_add((len + 3) & !3)?; + } + None +} +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[tokio::test] + async fn mapping_uses_the_actual_socket_and_rejects_unrelated_or_truncated_answers() { + let server = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let address = server.local_addr().unwrap(); + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let expected = socket.local_addr().unwrap(); + let task = tokio::spawn(async move { + let mut request = [0; 128]; + let (len, source) = server.recv_from(&mut request).await.unwrap(); + assert_eq!(len, 20); + assert_eq!(source, expected); + let tx: [u8; 12] = request[8..20].try_into().unwrap(); + let mut reply = request[..20].to_vec(); + reply[..4].copy_from_slice(&[1, 1, 0, 12]); + reply.extend_from_slice(&[0, 0x20, 0, 8, 0, 1]); + let port = source.port().to_be_bytes(); + reply.extend_from_slice(&[port[0] ^ COOKIE[0], port[1] ^ COOKIE[1]]); + for (i, b) in Ipv4Addr::LOCALHOST.octets().into_iter().enumerate() { + reply.push(b ^ COOKIE[i]); + } + assert!(parse(&reply, &[0; 12]).is_none()); + for len in 0..reply.len() { + assert!(parse(&reply[..len], &tx).is_none()); + } + server.send_to(&reply, source).await.unwrap(); + }); + assert_eq!(mapped(&socket, &address.to_string()).await, Some(expected)); + task.await.unwrap(); + } +} diff --git a/crates/tsunagi-awg/src/transport.rs b/crates/tsunagi-awg/src/transport.rs new file mode 100644 index 0000000..491e601 --- /dev/null +++ b/crates/tsunagi-awg/src/transport.rs @@ -0,0 +1,720 @@ +//! Private pair negotiation. No secret profiles in capabilities or durable state. +use crate::{ + AWG_PROTOCOL, AWG_WIRE_VERSION, + profile::PairProfile, + session::{self, PairLink}, +}; +use boringtun::{ + noise::Tunn, + x25519::{PublicKey, StaticSecret}, +}; +use iroh::EndpointId; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::time::Duration; +use tokio::net::UdpSocket; +use tokio::sync::{mpsc, oneshot, watch}; +use tsunagi::dataplane::transport::TransportPayload; +use tsunagi::dataplane::transport::{ + ControlSender, InboundLink, PacketLink, PacketTransport, SharedLink, TransportError, +}; +use tsunagi::{BoxFuture, NetworkId}; + +type Pair = (NetworkId, EndpointId); +#[derive(Serialize, Deserialize)] +enum Negotiation { + Offer { + version: u16, + profile: PairProfile, + key: [u8; 32], + #[serde(deserialize_with = "bounded_addresses")] + addresses: Vec, + }, + Answer { + attempt: [u8; 16], + key: [u8; 32], + #[serde(deserialize_with = "bounded_addresses")] + addresses: Vec, + }, +} +fn bounded_addresses<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Vec; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("at most 32 UDP candidates") + } + fn visit_seq>( + self, + mut sequence: A, + ) -> Result { + if sequence.size_hint().is_some_and(|n| n > 32) { + return Err(serde::de::Error::custom("too many UDP candidates")); + } + let mut addresses = Vec::with_capacity(sequence.size_hint().unwrap_or(0).min(32)); + while let Some(address) = sequence.next_element()? { + if addresses.len() == 32 { + return Err(serde::de::Error::custom("too many UDP candidates")); + } + addresses.push(address); + } + Ok(addresses) + } + } + deserializer.deserialize_seq(Visitor) +} +struct Pending { + profile: PairProfile, + key: StaticSecret, + socket: UdpSocket, + reply: oneshot::Sender>, +} +struct Inner { + stopped: watch::Sender, + local: EndpointId, + bind: Ipv4Addr, + stun: Option, + busy: Mutex>, + control: OnceLock>, + allowed: Mutex>, + pending: Mutex>, + links: Mutex>>, + inbound: mpsc::Sender, + port_by_pair: Mutex>, +} +/// Public diagnostics for one pair. No key or masking parameters are exposed. +#[derive(Debug, Clone)] +pub struct PairOverview { + /// Other endpoint's authenticated identity. + pub peer: EndpointId, + /// Fresh connection-attempt identifier. + pub attempt: [u8; 16], + /// Last known UDP port; None if diagnostics race with pair retirement. + pub local_port: Option, + /// Actual transport path, if established. + pub path: String, + /// Whether this attempt has been retired. + pub closed: bool, +} +/// A bounded set of independently configured pair engines. +pub struct AwgTransport { + inner: Arc, + commands: mpsc::Sender<(NetworkId, EndpointId, Negotiation)>, + incoming: tokio::sync::Mutex>, +} +impl std::fmt::Debug for AwgTransport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AwgTransport") + .field("local", &self.inner.local) + .finish_non_exhaustive() + } +} +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + match mutex.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + } +} +fn error(text: &str) -> TransportError { + TransportError::Other(text.into()) +} +async fn addresses(inner: &Inner, socket: &UdpSocket) -> Result, TransportError> { + let local = socket + .local_addr() + .map_err(|_| error("AWG socket address unavailable"))?; + let mut candidates = if inner.bind.is_loopback() { + vec![local] + } else { + tsunagi_wireguard::local_ip_candidates() + .into_iter() + .filter(|ip| ip.is_ipv4()) + .take(31) + .map(|ip| SocketAddr::new(ip, local.port())) + .collect() + }; + if let Some(server) = &inner.stun + && let Some(mapped) = crate::stun::mapped(socket, server).await + && !candidates.contains(&mapped) + { + candidates.push(mapped); + } + if candidates.is_empty() { + return Err(error("AWG has no UDP address candidates")); + } + Ok(candidates) +} +fn validate_endpoint(key: &[u8; 32], addresses: &[SocketAddr]) -> Result<(), TransportError> { + if *key == [0; 32] + || addresses.is_empty() + || addresses.len() > 32 + || addresses.iter().any(|address| { + address.port() == 0 + || !address.is_ipv4() + || address.ip().is_unspecified() + || address.ip().is_multicast() + || matches!(address.ip(), std::net::IpAddr::V4(ip) if ip.is_broadcast()) + }) + { + return Err(error("invalid AWG endpoint")); + } + Ok(()) +} + +impl AwgTransport { + /// Starts the coordinator. Production binds wildcard IPv4; tests use loopback. + pub fn new(local: EndpointId, bind: Ipv4Addr) -> Arc { + Self::new_with_stun(local, bind, None) + } + /// Enables an optional STUN server (host:port) for this pair's own socket. + /// Local-only callers pass None; failure still permits local UDP candidates. + pub fn new_with_stun(local: EndpointId, bind: Ipv4Addr, stun: Option) -> Arc { + let (commands, mut rx) = mpsc::channel(64); + let (inbound, incoming) = mpsc::channel(64); + let (stopped, _) = watch::channel(false); + let inner = Arc::new(Inner { + stopped, + local, + bind, + stun, + busy: Mutex::default(), + control: OnceLock::new(), + allowed: Mutex::default(), + pending: Mutex::default(), + links: Mutex::default(), + inbound, + port_by_pair: Mutex::default(), + }); + let weak = Arc::downgrade(&inner); + let mut stopped = inner.stopped.subscribe(); + tokio::spawn(async move { + while let Some((network, peer, message)) = + tokio::select! { biased; _=stopped.changed()=>None, message=rx.recv()=>message } + { + let Some(inner) = weak.upgrade() else { + break; + }; + if !lock(&inner.allowed).contains(&(network, peer)) + || !lock(&inner.busy).insert((network, peer)) + { + continue; + } + tokio::spawn(async move { + let mut stop = inner.stopped.subscribe(); + tokio::select! {biased; _=stop.changed()=>{}, _=async { + if let Err(err)=receive(&inner,(network,peer),message).await {tracing::debug!(%err,"AWG pair negotiation rejected");} + }=>{}} + lock(&inner.busy).remove(&(network, peer)); + }); + } + }); + Arc::new(Self { + inner, + commands, + incoming: tokio::sync::Mutex::new(incoming), + }) + } + /// Registers a peer after its authenticated capability has been validated. + pub fn allow_peer(&self, network: NetworkId, peer: EndpointId) { + let mut allowed = lock(&self.inner.allowed); + if !*self.inner.stopped.borrow() && allowed.len() < 128 { + allowed.insert((network, peer)); + } + } + /// Retires a pair when its network membership session goes away. + pub fn forget_peer(&self, network: NetworkId, peer: EndpointId) { + let pair = (network, peer); + lock(&self.inner.allowed).remove(&pair); + lock(&self.inner.pending).remove(&pair); + lock(&self.inner.port_by_pair).remove(&pair); + if let Some(link) = lock(&self.inner.links).remove(&pair) { + link.close(); + } + } + /// Stops all pair engines in this network. + pub fn stop_network(&self, network: NetworkId) { + let peers: Vec<_> = lock(&self.inner.allowed) + .iter() + .filter(|(n, _)| *n == network) + .map(|(_, p)| *p) + .collect(); + for peer in peers { + self.forget_peer(network, peer); + } + } + /// Stops all sockets and coordinator work without waiting for remote peers. + pub fn shutdown(&self) { + self.inner.stopped.send_replace(true); + lock(&self.inner.allowed).clear(); + lock(&self.inner.pending).clear(); + for (_, link) in lock(&self.inner.links).drain() { + link.close(); + } + lock(&self.inner.port_by_pair).clear(); + } + /// Receives a successfully negotiated and authenticated inbound pair link. + pub async fn next_inbound(&self) -> Option { + let mut stopped = self.inner.stopped.subscribe(); + if *stopped.borrow() { + return None; + } + let mut incoming = self.incoming.lock().await; + tokio::select! {biased; _=stopped.changed()=>None, link=incoming.recv()=>link} + } + /// Takes a public snapshot of this network's pair attempts. + pub fn overview(&self, network: NetworkId) -> Vec { + let links = lock(&self.inner.links); + let ports = lock(&self.inner.port_by_pair); + links + .iter() + .filter(|((n, _), _)| *n == network) + .map(|((_, peer), link)| PairOverview { + peer: *peer, + attempt: link.attempt, + local_port: ports.get(&(network, *peer)).copied(), + path: link.path_description(), + closed: link.is_closed(), + }) + .collect() + } +} +impl PacketTransport for AwgTransport { + fn name(&self) -> &str { + AWG_PROTOCOL + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn attach_control(&self, control: Arc) { + let _ = self.inner.control.set(control); + } + fn on_control( + &self, + network: NetworkId, + peer: EndpointId, + protocol: &str, + payload: &[u8], + ) -> Result<(), TransportError> { + if protocol != AWG_PROTOCOL || !lock(&self.inner.allowed).contains(&(network, peer)) { + return Err(TransportError::Declined(protocol.into())); + } + if payload.len() > 4096 { + return Err(error("AWG negotiation too large")); + } + let message: Negotiation = + postcard::from_bytes(payload).map_err(|_| error("malformed AWG negotiation"))?; + match &message { + Negotiation::Offer { + version, + profile, + key, + addresses, + } => { + if *version != AWG_WIRE_VERSION || peer.as_bytes() >= self.inner.local.as_bytes() { + return Err(error("invalid AWG offer role/version")); + } + profile.validate().map_err(error)?; + validate_endpoint(key, addresses)?; + } + Negotiation::Answer { key, addresses, .. } => validate_endpoint(key, addresses)?, + } + self.commands + .try_send((network, peer, message)) + .map_err(|_| error("AWG negotiation queue full")) + } + fn open<'a>( + &'a self, + network: NetworkId, + peer: EndpointId, + protocol: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let pair = (network, peer); + if protocol != AWG_PROTOCOL + || self.inner.local.as_bytes() >= peer.as_bytes() + || !lock(&self.inner.allowed).contains(&pair) + { + return Err(TransportError::Declined(protocol.into())); + } + let control = self + .inner + .control + .get() + .ok_or_else(|| error("AWG control plane unavailable"))?; + let socket = UdpSocket::bind((self.inner.bind, 0)) + .await + .map_err(|_| error("AWG UDP bind failed"))?; + let profile = PairProfile::generate(); + let attempt = profile.id(); + let key = StaticSecret::from(rand::random::<[u8; 32]>()); + let public = PublicKey::from(&key).to_bytes(); + let message = Negotiation::Offer { + version: AWG_WIRE_VERSION, + profile: profile.clone(), + key: public, + addresses: addresses(&self.inner, &socket).await?, + }; + let (reply, rx) = oneshot::channel(); + if lock(&self.inner.pending).len() >= 128 { + return Err(error("too many AWG pair negotiations")); + } + lock(&self.inner.pending).insert( + pair, + Pending { + profile, + key, + socket, + reply, + }, + ); + // A cancellation guard also retires partially started attempts. + let mut guard = AttemptGuard { + inner: Arc::downgrade(&self.inner), + pair, + attempt, + completed: false, + }; + let bytes = + postcard::to_stdvec(&message).map_err(|_| error("AWG offer encoding failed"))?; + control + .send(network, peer, AWG_PROTOCOL, TransportPayload::new(bytes)) + .await?; + let result = tokio::time::timeout(Duration::from_secs(15), rx) + .await + .map_err(|_| error("AWG negotiation/handshake timed out"))? + .map_err(|_| error("AWG attempt cancelled"))?; + guard.completed = result.is_ok(); + result + }) + } +} +struct AttemptGuard { + inner: Weak, + pair: Pair, + attempt: [u8; 16], + completed: bool, +} +impl Drop for AttemptGuard { + fn drop(&mut self) { + if let Some(inner) = self.inner.upgrade() { + let mut pending = lock(&inner.pending); + if pending + .get(&self.pair) + .is_some_and(|p| p.profile.id() == self.attempt) + { + pending.remove(&self.pair); + } + drop(pending); + if !self.completed + && let Some(link) = lock(&inner.links).get(&self.pair) + && link.attempt == self.attempt + { + link.close(); + } + } + } +} +async fn receive( + inner: &Arc, + pair: Pair, + message: Negotiation, +) -> Result<(), TransportError> { + if !lock(&inner.allowed).contains(&pair) { + return Err(error("AWG peer no longer authenticated")); + } + match message { + Negotiation::Offer { + profile, + key, + addresses: candidates, + .. + } => { + if let Some(link) = lock(&inner.links).get(&pair) { + if link.attempt == profile.id() { + return Err(error("duplicate AWG offer")); + } + link.close(); + } + let socket = UdpSocket::bind((inner.bind, 0)) + .await + .map_err(|_| error("AWG UDP bind failed"))?; + let local_port = socket + .local_addr() + .map_err(|_| error("AWG socket address unavailable"))? + .port(); + let local_key = StaticSecret::from(rand::random::<[u8; 32]>()); + let public = PublicKey::from(&local_key).to_bytes(); + let answer = Negotiation::Answer { + attempt: profile.id(), + key: public, + addresses: addresses(inner, &socket).await?, + }; + let tunn = Tunn::new( + local_key, + PublicKey::from(key), + Some(profile.psk), + Some(1), + rand::random(), + None, + ); + + let link = session::start( + pair.0, + pair.1, + socket, + tunn, + profile, + candidates, + inner.stopped.subscribe(), + ); + if !lock(&inner.allowed).contains(&pair) { + link.close(); + return Err(error("AWG membership ended")); + } + lock(&inner.links).insert(pair, link.clone()); + lock(&inner.port_by_pair).insert(pair, local_port); + let control = inner + .control + .get() + .ok_or_else(|| error("AWG control unavailable"))?; + if let Err(err) = control + .send( + pair.0, + pair.1, + AWG_PROTOCOL, + TransportPayload::new( + postcard::to_stdvec(&answer) + .map_err(|_| error("AWG answer encoding failed"))?, + ), + ) + .await + { + link.close(); + return Err(err); + } + publish_when_up(Arc::downgrade(inner), pair, link, None); + } + Negotiation::Answer { + attempt, + key, + addresses: candidates, + } => { + if !lock(&inner.pending) + .get(&pair) + .is_some_and(|p| p.profile.id() == attempt) + { + return Err(error("stale AWG answer")); + } + let pending = lock(&inner.pending) + .remove(&pair) + .ok_or_else(|| error("AWG attempt cancelled"))?; + let local_port = pending + .socket + .local_addr() + .map_err(|_| error("AWG socket address unavailable"))? + .port(); + let tunn = Tunn::new( + pending.key, + PublicKey::from(key), + Some(pending.profile.psk), + Some(1), + rand::random(), + None, + ); + + let link = session::start( + pair.0, + pair.1, + pending.socket, + tunn, + pending.profile, + candidates, + inner.stopped.subscribe(), + ); + if !lock(&inner.allowed).contains(&pair) { + link.close(); + return Err(error("AWG membership ended")); + } + lock(&inner.links).insert(pair, link.clone()); + lock(&inner.port_by_pair).insert(pair, local_port); + publish_when_up(Arc::downgrade(inner), pair, link, Some(pending.reply)); + } + } + Ok(()) +} +fn publish_when_up( + inner: Weak, + pair: Pair, + link: Arc, + reply: Option>>, +) { + tokio::spawn(async move { + let mut reply = reply; + let result = tokio::time::timeout(Duration::from_secs(14), async { + loop { + if link.is_closed() || reply.as_ref().is_some_and(oneshot::Sender::is_closed) { + return Err(error("AWG pair closed during handshake")); + } + if link.has_authenticated_path() { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await; + if !matches!(result, Ok(Ok(()))) { + link.close(); + if let Some(reply) = reply.take() { + let _ = reply.send(Err(error("AWG authenticated data path timed out"))); + } + return; + } + let Some(inner) = inner.upgrade() else { + link.close(); + return; + }; + if !lock(&inner.allowed).contains(&pair) { + link.close(); + return; + } + if link.is_closed() + || !lock(&inner.links) + .get(&pair) + .is_some_and(|current| current.attempt == link.attempt) + { + link.close(); + return; + } + let shared: SharedLink = link.clone(); + if let Some(reply) = reply { + if reply.send(Ok(shared)).is_err() { + link.close(); + } + } else if inner + .inbound + .try_send(InboundLink { + network: pair.0, + peer: pair.1, + protocol: AWG_PROTOCOL.into(), + link: shared, + }) + .is_err() + { + link.close(); + } + }); +} +impl Drop for AwgTransport { + fn drop(&mut self) { + self.shutdown(); + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[tokio::test] + async fn concurrent_admission_is_bounded_across_networks_and_retirement_releases_slots() { + let identity = tsunagi::identity::DeviceIdentity::generate(); + let transport = AwgTransport::new(identity.endpoint_id(), Ipv4Addr::LOCALHOST); + let peer = tsunagi::identity::DeviceIdentity::generate().endpoint_id(); + std::thread::scope(|scope| { + for thread in 0..8u8 { + let transport = &transport; + scope.spawn(move || { + for item in 0..32u8 { + let mut id = [0; 32]; + id[0] = thread; + id[1] = item; + transport.allow_peer(NetworkId::from_bytes(id), peer); + } + }); + } + }); + assert_eq!(lock(&transport.inner.allowed).len(), 128); + let network = lock(&transport.inner.allowed).iter().next().unwrap().0; + transport.stop_network(network); + let new = NetworkId::from_bytes([255; 32]); + transport.allow_peer(new, peer); + assert!(lock(&transport.inner.allowed).contains(&(new, peer))); + assert_eq!(lock(&transport.inner.allowed).len(), 128); + transport.shutdown(); + assert!(lock(&transport.inner.allowed).is_empty()); + } + #[tokio::test] + async fn private_negotiation_bounds_roles_and_membership_before_starting_engines() { + let mut ids = [ + tsunagi::identity::DeviceIdentity::generate().endpoint_id(), + tsunagi::identity::DeviceIdentity::generate().endpoint_id(), + ]; + ids.sort_by_key(|id| *id.as_bytes()); + let peer = ids[0]; + let local = ids[1]; + let network = NetworkId::from_bytes([7; 32]); + let transport = AwgTransport::new(local, Ipv4Addr::LOCALHOST); + let offer = |version, key, addresses| Negotiation::Offer { + version, + profile: PairProfile::generate(), + key, + addresses, + }; + let address = SocketAddr::from((Ipv4Addr::LOCALHOST, 12345)); + let encode = |message: &Negotiation| postcard::to_stdvec(message).unwrap(); + let valid = encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![address])); + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &valid) + .is_err() + ); + transport.allow_peer(network, peer); + assert!( + transport + .on_control(network, peer, "wrong-protocol", &valid) + .is_err() + ); + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &[0; 4097]) + .is_err() + ); + for bytes in [ + encode(&offer(99, [1; 32], vec![address])), + encode(&offer(AWG_WIRE_VERSION, [0; 32], vec![address])), + encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![])), + encode(&offer(AWG_WIRE_VERSION, [1; 32], vec![address; 33])), + ] { + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &bytes) + .is_err() + ); + } + for len in 0..valid.len() { + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &valid[..len]) + .is_err() + ); + } + let wrong_role = AwgTransport::new(peer, Ipv4Addr::LOCALHOST); + wrong_role.allow_peer(network, local); + assert!( + wrong_role + .on_control(network, local, AWG_PROTOCOL, &valid) + .is_err() + ); + transport.forget_peer(network, peer); + assert!( + transport + .on_control(network, peer, AWG_PROTOCOL, &valid) + .is_err() + ); + assert!(transport.overview(network).is_empty()); + transport.shutdown(); + wrong_role.shutdown(); + } +} diff --git a/crates/tsunagi-awg/tests/pairs.rs b/crates/tsunagi-awg/tests/pairs.rs new file mode 100644 index 0000000..5adc745 --- /dev/null +++ b/crates/tsunagi-awg/tests/pairs.rs @@ -0,0 +1,530 @@ +//! Real membership, UDP sockets, pair negotiation and nested WireGuard encryption. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use bytes::Bytes; +use std::{ + net::Ipv4Addr, + path::Path, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::Duration, +}; +use tempfile::TempDir; +use tsunagi::dataplane::IpPlugin; +use tsunagi::dataplane::transport::{ + ControlSender, PacketLink, PacketTransport, SharedLink, TransportError, +}; +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::overlay::{MemoryTun, MemoryTunFactory}; +use tsunagi::testing::{config_with, network, wait_for_peers, wait_until}; +use tsunagi::{Agent, BoxFuture, NetworkId}; +use tsunagi_awg::{AWG_PROTOCOL, AwgPlugin, AwgTransport}; +use tsunagi_wg_quic::WireguardConfig; + +struct Node { + agent: Agent, + plugin: Arc, + transport: Arc, + tuns: MemoryTunFactory, + loss: Arc, +} +impl Node { + async fn open(root: &Path, discovery: &SharedMemoryDiscovery, interface: &str) -> Self { + Self::open_with_fallback(root, discovery, interface, false).await + } + async fn open_with_fallback( + root: &Path, + discovery: &SharedMemoryDiscovery, + interface: &str, + fallback: bool, + ) -> Self { + Self::open_config(root, discovery, interface, fallback, 1280).await + } + async fn open_config( + root: &Path, + discovery: &SharedMemoryDiscovery, + interface: &str, + fallback: bool, + mtu: u32, + ) -> Self { + let identity = + tsunagi::storage::StateStore::open(tsunagi::StoragePaths::under(root).state_db()) + .unwrap() + .load_or_create_device_identity() + .unwrap(); + let transport = AwgTransport::new(identity.endpoint_id(), Ipv4Addr::LOCALHOST); + let plugin = AwgPlugin::open( + WireguardConfig::new(root.join("awg")) + .with_mtu(mtu) + .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), + transport.clone(), + ) + .await + .unwrap(); + let loss = Arc::new(AtomicBool::new(false)); + let faults = Arc::new(FaultTransport { + inner: transport.clone(), + loss: loss.clone(), + }); + let tuns = MemoryTunFactory::new(); + let mut config = config_with(root, discovery) + .with_interface(Arc::new(tuns.clone()), interface, mtu) + .with_overlay_ipv4_range(Some(tsunagi::state::DEFAULT_IPV4_RANGE)) + .with_plugin(plugin.clone() as Arc) + .with_custom_transport(AWG_PROTOCOL, faults); + if fallback { + let quic = tsunagi_wg_quic::WireguardPlugin::open( + WireguardConfig::new(root.join("wg-quic")) + .with_mtu(mtu) + .with_reconcile(Duration::from_millis(20), Duration::from_millis(250)), + ) + .await + .unwrap(); + config = config.with_plugin(quic); + } + let agent = Agent::spawn(config).await.unwrap(); + let inbound_agent = agent.clone(); + let inbound_transport = transport.clone(); + let inbound_loss = loss.clone(); + tokio::spawn(async move { + while let Some(mut link) = inbound_transport.next_inbound().await { + link.link = impaired(link.link, inbound_loss.clone()); + let _ = inbound_agent.install_inbound_link(link).await; + } + }); + Self { + agent, + plugin, + transport, + tuns, + loss, + } + } + async fn ready(&self, id: NetworkId, n: usize) { + wait_until("AWG outer paths and end-to-end handshakes", || async { + (self + .transport + .overview(id) + .iter() + .filter(|p| !p.closed && !p.path.contains("establishing")) + .count() + == n + && self + .plugin + .overview(id)? + .peers + .iter() + .filter(|p| { + p.tunnel + .as_ref() + .is_some_and(|t| t.health.is_up() && t.path.contains("awg UDP via")) + }) + .count() + == n) + .then_some(()) + }) + .await; + } + fn tun(&self) -> Arc { + self.tuns + .device(&self.agent.overlay().unwrap().interface) + .unwrap() + } + fn addr(&self, id: NetworkId) -> Ipv4Addr { + self.plugin + .overview(id) + .unwrap() + .overlay_address_v4 + .unwrap() + } +} +fn packet(source: Ipv4Addr, dest: Ipv4Addr, len: usize) -> Bytes { + let mut bytes = vec![0; len]; + bytes[0] = 0x45; + bytes[2..4].copy_from_slice(&(len as u16).to_be_bytes()); + bytes[8] = 64; + bytes[9] = 253; + bytes[12..16].copy_from_slice(&source.octets()); + bytes[16..20].copy_from_slice(&dest.octets()); + rand::fill(&mut bytes[20..]); + Bytes::from(bytes) +} +async fn exchange(a: &Node, b: &Node, id: NetworkId) { + for len in [64, 1097, 1280] { + let bytes = packet(a.addr(id), b.addr(id), len); + a.tun().push_from_os(bytes.clone()); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), b.tun().pop_to_os()) + .await + .unwrap() + .unwrap(), + bytes + ); + } +} +#[tokio::test] +async fn real_pairs_carry_full_size_packets_and_restart_with_fresh_profiles() { + tsunagi::testing::init_tracing(); + let discovery = SharedMemoryDiscovery::new(); + let roots = [ + TempDir::new().unwrap(), + TempDir::new().unwrap(), + TempDir::new().unwrap(), + ]; + let a = Node::open(roots[0].path(), &discovery, "awga").await; + let b = Node::open(roots[1].path(), &discovery, "awgb").await; + let c = Node::open(roots[2].path(), &discovery, "awgc").await; + let (name, secret) = network("awg-pairs"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + c.agent.join_network(&name, &secret).await.unwrap(); + for node in [&a, &b, &c] { + wait_for_peers(&node.agent, id, 2).await; + node.ready(id, 2).await; + } + let ids: std::collections::HashSet<_> = [&a, &b, &c] + .into_iter() + .flat_map(|n| n.transport.overview(id).into_iter().map(|p| p.attempt)) + .collect(); + assert_eq!( + ids.len(), + 3, + "one independent profile for each of three pairs" + ); + exchange(&a, &b, id).await; + exchange(&b, &a, id).await; + exchange(&a, &c, id).await; + let old_ports: Vec<_> = a + .transport + .overview(id) + .into_iter() + .filter_map(|p| p.local_port) + .collect(); + a.agent.shutdown().await; + for port in old_ports { + wait_until("old pair socket released", || async { + tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, port)) + .await + .ok() + }) + .await; + } + let restarted = Node::open(roots[0].path(), &discovery, "awga").await; + restarted.ready(id, 2).await; + assert!( + restarted + .transport + .overview(id) + .iter() + .all(|p| !ids.contains(&p.attempt)) + ); + exchange(&restarted, &b, id).await; + exchange(&c, &restarted, id).await; + restarted.agent.shutdown().await; + b.agent.shutdown().await; + c.agent.shutdown().await; + assert!( + tokio::time::timeout(Duration::from_secs(1), restarted.transport.next_inbound()) + .await + .unwrap() + .is_none() + ); +} + +// Drops datagrams at the link boundary. Membership, control, sockets, crypto, +// addressing and storage remain real. This substitutes a lossy physical path. +#[derive(Debug)] +struct FaultTransport { + inner: Arc, + loss: Arc, +} +impl PacketTransport for FaultTransport { + fn name(&self) -> &str { + AWG_PROTOCOL + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn attach_control(&self, sender: Arc) { + self.inner.attach_control(sender); + } + fn on_control( + &self, + network: NetworkId, + peer: iroh::EndpointId, + protocol: &str, + bytes: &[u8], + ) -> Result<(), TransportError> { + self.inner.on_control(network, peer, protocol, bytes) + } + fn open<'a>( + &'a self, + network: NetworkId, + peer: iroh::EndpointId, + protocol: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Ok(impaired( + self.inner.open(network, peer, protocol).await?, + self.loss.clone(), + )) + }) + } +} +fn impaired(raw: SharedLink, loss: Arc) -> SharedLink { + Arc::new(FaultLink { + raw, + loss, + sequence: AtomicU64::new(0), + }) +} +#[derive(Debug)] +struct FaultLink { + raw: SharedLink, + loss: Arc, + sequence: AtomicU64, +} +impl PacketLink for FaultLink { + fn network(&self) -> NetworkId { + self.raw.network() + } + fn peer(&self) -> iroh::EndpointId { + self.raw.peer() + } + fn max_datagram_size(&self) -> usize { + self.raw.max_datagram_size() + } + fn send(&self, bytes: Bytes) -> Result<(), TransportError> { + if self.loss.load(Ordering::Relaxed) + && !self + .sequence + .fetch_add(1, Ordering::Relaxed) + .is_multiple_of(5) + { + Ok(()) + } else { + self.raw.send(bytes) + } + } + fn recv(&self) -> BoxFuture<'_, Option> { + self.raw.recv() + } + fn closed(&self) -> BoxFuture<'_, ()> { + self.raw.closed() + } + fn is_closed(&self) -> bool { + self.raw.is_closed() + } + fn close(&self) { + self.raw.close(); + } + fn path_description(&self) -> String { + self.raw.path_description() + } +} +#[tokio::test] +async fn persistent_runtime_loss_retires_the_pair_and_negotiates_fresh_parameters() { + let discovery = SharedMemoryDiscovery::new(); + let roots = [TempDir::new().unwrap(), TempDir::new().unwrap()]; + let a = Node::open(roots[0].path(), &discovery, "lossa").await; + let b = Node::open(roots[1].path(), &discovery, "lossb").await; + let (name, secret) = network("awg-loss"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + a.ready(id, 1).await; + b.ready(id, 1).await; + exchange(&a, &b, id).await; + let old = a.transport.overview(id)[0].attempt; + let mut events = a.agent.subscribe(); + a.loss.store(true, Ordering::Relaxed); + tsunagi::testing::wait_event(&mut events, |event| match event { + tsunagi::Event::DataLinkDown { + network, reason, .. + } if *network == id && reason.contains("delivery checks") => Some(()), + _ => None, + }) + .await; + a.loss.store(false, Ordering::Relaxed); + wait_until("fresh AWG profile after runtime loss", || async { + a.transport + .overview(id) + .iter() + .any(|p| p.attempt != old && !p.closed && !p.path.contains("establishing")) + .then_some(()) + }) + .await; + a.ready(id, 1).await; + b.ready(id, 1).await; + let initiator = if a.agent.endpoint_id().as_bytes() < b.agent.endpoint_id().as_bytes() { + &a + } else { + &b + }; + let received = initiator.plugin.overview(id).unwrap().peers[0] + .tunnel + .as_ref() + .unwrap() + .stats + .rx_packets; + let attempt = initiator.transport.overview(id)[0].attempt; + wait_until( + "fresh pair delivery checks answered after installation", + || async { + let view = initiator.plugin.overview(id)?; + (view.peers.first()?.tunnel.as_ref()?.stats.rx_packets >= received + 3).then_some(()) + }, + ) + .await; + assert_eq!(initiator.transport.overview(id)[0].attempt, attempt); + wait_for_peers(&a.agent, id, 1).await; + exchange(&a, &b, id).await; + exchange(&b, &a, id).await; + a.agent.shutdown().await; + b.agent.shutdown().await; +} + +#[tokio::test] +async fn runtime_loss_falls_back_to_real_iroh_without_interrupting_control_or_ip() { + let discovery = SharedMemoryDiscovery::new(); + let roots = [TempDir::new().unwrap(), TempDir::new().unwrap()]; + let a = Node::open_with_fallback(roots[0].path(), &discovery, "fallba", true).await; + let b = Node::open_with_fallback(roots[1].path(), &discovery, "fallbb", true).await; + let (name, secret) = network("awg-fallback"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + a.ready(id, 1).await; + b.ready(id, 1).await; + exchange(&a, &b, id).await; + let mut events = a.agent.subscribe(); + a.loss.store(true, Ordering::Relaxed); + tsunagi::testing::wait_event(&mut events, |event| match event { + tsunagi::Event::DataLinkDown { + network, reason, .. + } if *network == id && reason.contains("delivery checks") => Some(()), + _ => None, + }) + .await; + for node in [&a, &b] { + wait_until("fallback carrier is wg-quic",||async { + let status=node.agent.status().await.ok()?; + let network=status.networks.iter().find(|n|n.network_id==id)?; + (network.members.iter().any(|m|matches!(&m.data_path,tsunagi::agent::DataPath::Direct{transport} if transport=="wg-quic"))).then_some(()) + }).await; + wait_for_peers(&node.agent, id, 1).await; + } + exchange(&a, &b, id).await; + exchange(&b, &a, id).await; + a.agent.shutdown().await; + b.agent.shutdown().await; +} + +#[tokio::test] +async fn opaque_pair_carrier_fragments_its_full_advertised_capacity() { + let discovery = SharedMemoryDiscovery::new(); + let roots = [TempDir::new().unwrap(), TempDir::new().unwrap()]; + let mut nodes = Vec::new(); + for root in &roots { + let identity = tsunagi::storage::StateStore::open( + tsunagi::StoragePaths::under(root.path()).state_db(), + ) + .unwrap() + .load_or_create_device_identity() + .unwrap(); + let transport = AwgTransport::new(identity.endpoint_id(), Ipv4Addr::LOCALHOST); + let agent = Agent::spawn( + config_with(root.path(), &discovery) + .with_custom_transport(AWG_PROTOCOL, transport.clone()), + ) + .await + .unwrap(); + nodes.push((agent, transport)); + } + let (name, secret) = network("awg-capacity"); + let id = nodes[0].0.join_network(&name, &secret).await.unwrap(); + nodes[1].0.join_network(&name, &secret).await.unwrap(); + for (i, (agent, transport)) in nodes.iter().enumerate() { + wait_for_peers(agent, id, 1).await; + // This fixture tests only the transport contract: register the member + // after the real core has authenticated it, without an IP plugin/TUN. + transport.allow_peer(id, nodes[1 - i].0.endpoint_id()); + } + nodes.sort_by_key(|(agent, _)| *agent.endpoint_id().as_bytes()); + let outgoing = nodes[0] + .1 + .open(id, nodes[1].0.endpoint_id(), AWG_PROTOCOL) + .await + .unwrap(); + let incoming = tokio::time::timeout(Duration::from_secs(5), nodes[1].1.next_inbound()) + .await + .unwrap() + .unwrap() + .link; + for (from, to) in [(&outgoing, &incoming), (&incoming, &outgoing)] { + let mut bytes = vec![0; from.max_datagram_size()]; + rand::fill(&mut bytes[..]); + let bytes = Bytes::from(bytes); + from.send(bytes.clone()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), to.recv()) + .await + .unwrap() + .unwrap(), + bytes + ); + assert!(matches!( + from.send(Bytes::from(vec![0; from.max_datagram_size() + 1])), + Err(TransportError::TooLarge { .. }) + )); + } + outgoing.close(); + incoming.close(); + for (agent, transport) in nodes { + transport.shutdown(); + agent.shutdown().await; + } +} + +#[tokio::test] +async fn encrypted_health_checks_fit_a_smaller_peers_configured_interface_mtu() { + let discovery = SharedMemoryDiscovery::new(); + let roots = [TempDir::new().unwrap(), TempDir::new().unwrap()]; + let a = Node::open_config(roots[0].path(), &discovery, "smalla", false, 1280).await; + let b = Node::open_config(roots[1].path(), &discovery, "smallb", false, 576).await; + let (name, secret) = network("awg-small-mtu"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + a.ready(id, 1).await; + b.ready(id, 1).await; + let bytes = packet(a.addr(id), b.addr(id), 576); + a.tun().push_from_os(bytes.clone()); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), b.tun().pop_to_os()) + .await + .unwrap() + .unwrap(), + bytes + ); + wait_until( + "encrypted delivery checks answered with a small MTU", + || async { + let view = a.plugin.overview(id)?; + let peer = view.peers.first()?.tunnel.as_ref()?; + (peer.stats.rx_packets >= 3).then_some(()) + }, + ) + .await; + for node in [&a, &b] { + assert_eq!( + node.plugin.overview(id).unwrap().peers[0] + .tunnel + .as_ref() + .unwrap() + .stats + .dropped_oversize, + 0 + ); + } + a.agent.shutdown().await; + b.agent.shutdown().await; +} diff --git a/crates/tsunagi-awg/tests/profiles.rs b/crates/tsunagi-awg/tests/profiles.rs new file mode 100644 index 0000000..c158ea7 --- /dev/null +++ b/crates/tsunagi-awg/tests/profiles.rs @@ -0,0 +1,36 @@ +//! Fresh and bounded masking profiles, and rejection of foreign/truncated wire packets. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use tsunagi_awg::profile::PairProfile; + +#[test] +fn profiles_are_fresh_bounded_and_redacted() { + let profiles: Vec<_> = (0..128).map(|_| PairProfile::generate()).collect(); + for (i, profile) in profiles.iter().enumerate() { + profile.validate().unwrap(); + assert!(!format!("{profile:?}").contains(&format!("{:?}", profile.header_key()))); + for other in &profiles[..i] { + assert_ne!(profile.header_key(), other.header_key()); + assert_ne!(profile.id(), other.id()); + } + for packet in profile.signature_packets() { + assert!(packet.len() < 1200); + } + } +} + +#[test] +fn masked_packets_round_trip_and_reject_other_profiles_and_truncation() { + let profile = PairProfile::generate(); + let other = PairProfile::generate(); + for (kind, size) in [(1, 148), (2, 92), (3, 64), (4, 800)] { + let mut message = vec![0x7b; size]; + message[..4].copy_from_slice(&(kind as u32).to_le_bytes()); + let wire = profile.encode(&message).unwrap(); + assert_ne!(&wire[..4], &message[..4]); + assert_eq!(profile.decode(&wire).unwrap(), message); + assert!(other.decode(&wire).is_none()); + for length in 0..44 { + assert!(profile.decode(&wire[..length]).is_none()); + } + } +} diff --git a/crates/tsunagi-core/src/agent/mod.rs b/crates/tsunagi-core/src/agent/mod.rs index 848edb2..d70526f 100644 --- a/crates/tsunagi-core/src/agent/mod.rs +++ b/crates/tsunagi-core/src/agent/mod.rs @@ -169,6 +169,21 @@ impl PacketSink for Sink { let Some(inner) = self.0.upgrade() else { return; }; + if let Some(payload) = crate::dataplane::transport::health::ip_payload(&packet) { + if let Some(header) = crate::overlay::IpHeader::parse(&packet) + && let (std::net::IpAddr::V4(source), std::net::IpAddr::V4(destination)) = + (header.source(), header.destination()) + && inner.routes.may_send_from(network, peer, source) + && inner.routes.is_local_destination(network, destination) + && let Some(handle) = inner.networks.read().await.get(&network) + { + let _ = handle.commands.try_send(NetCommand::HealthProbe { + peer, + packet: bytes::Bytes::copy_from_slice(payload), + }); + } + return; + } let Some(interface) = inner.interface.get() else { return; }; @@ -187,6 +202,35 @@ impl PacketSink for Sink { #[derive(Debug)] struct TransportCtx(Weak); +impl crate::dataplane::transport::ControlSender for TransportCtx { + fn send<'a>( + &'a self, + network: NetworkId, + peer: EndpointId, + protocol: &'a str, + payload: crate::proto::message::TransportPayload, + ) -> crate::BoxFuture<'a, std::result::Result<(), crate::dataplane::transport::TransportError>> + { + Box::pin(async move { + let inner = self + .0 + .upgrade() + .ok_or(crate::dataplane::transport::TransportError::Closed)?; + Agent { inner } + .send( + network, + peer, + crate::proto::message::ControlMessage::Transport { + protocol: protocol.into(), + payload, + }, + ) + .await + .map_err(|err| crate::dataplane::transport::TransportError::Other(err.to_string())) + }) + } +} + impl TransportContext for TransportCtx { fn snapshot(&self) -> crate::BoxFuture<'_, HashMap> { Box::pin(async move { @@ -302,6 +346,7 @@ impl Agent { inner.config.custom_transports.clone(), )) }; + transport.attach_control(Arc::new(TransportCtx(Arc::downgrade(&inner)))); let _ = inner.transport.set(transport); if let CacheOutcome::Reset(reason) = inner.storage.cache_outcome().clone() { diff --git a/crates/tsunagi-core/src/agent/network.rs b/crates/tsunagi-core/src/agent/network.rs index 9440d80..9e0f791 100644 --- a/crates/tsunagi-core/src/agent/network.rs +++ b/crates/tsunagi-core/src/agent/network.rs @@ -15,6 +15,7 @@ use tokio::sync::{broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use crate::config::{DiscoveryPolicy, Limits, ReconnectPolicy}; +use crate::dataplane::transport::health::{HealthLink, ProbeRoute, Settings as HealthSettings}; use crate::dataplane::transport::{InboundLink, PacketLink, PacketTransport, SharedLink}; use crate::dataplane::{PluginCapability, SharedPlugin}; use crate::discovery::worker::DiscoveryWorker; @@ -59,6 +60,10 @@ pub(crate) enum NetCommand { }, /// A peer opened a data plane link towards us. InboundLink(Box), + HealthProbe { + peer: EndpointId, + packet: bytes::Bytes, + }, Status { reply: oneshot::Sender>, }, @@ -123,6 +128,7 @@ impl std::fmt::Debug for NetCommand { NetCommand::SetHostname(_) => f.write_str("SetHostname"), NetCommand::Release { .. } => f.write_str("Release"), NetCommand::PluginError { protocol, .. } => write!(f, "PluginError({protocol})"), + NetCommand::HealthProbe { peer, .. } => write!(f, "HealthProbe({})", peer.fmt_short()), NetCommand::ProtocolFailure { peer, protocol, .. } => { write!(f, "ProtocolFailure({}, {protocol})", peer.fmt_short()) } @@ -261,6 +267,12 @@ struct ProtocolDialState { const DATA_LINK_BASE_BACKOFF: Duration = Duration::from_secs(3); const DATA_LINK_MAX_BACKOFF: Duration = Duration::from_secs(60); +fn next_protocol_index(supported: &[String], last_failed: Option<&str>) -> usize { + last_failed + .and_then(|failed| supported.iter().position(|p| p == failed)) + .map_or(0, |i| (i + 1) % supported.len().max(1)) +} + fn data_link_backoff(failures: u32) -> Duration { if failures == 0 { return Duration::ZERO; @@ -292,7 +304,9 @@ struct Runtime { /// /// The direct ones. What a protocol holds is the hub's link for that /// peer, which outlives any particular path. - links: HashMap<(EndpointId, String), SharedLink>, + links: HashMap<(EndpointId, String), Arc>, + pending_health: HashMap<(EndpointId, String), HealthSettings>, + last_failed_protocol: HashMap, /// Every data link of this network, and what is reached through what. hub: Arc, /// Which peers each peer says it has a live link with, and when it @@ -404,6 +418,8 @@ impl Runtime { dial_results_tx, dial_results_rx, links: HashMap::new(), + pending_health: HashMap::new(), + last_failed_protocol: HashMap::new(), hub, reachable: HashMap::new(), local_capabilities: Vec::new(), @@ -491,7 +507,12 @@ impl Runtime { // any more, and a sender blocked on a full queue would stall shutdown. self.session_events_rx.close(); self.hub.close(); + for link in self.links.values() { + link.close(); + } self.links.clear(); + self.pending_health.clear(); + self.last_failed_protocol.clear(); self.protocol_dial_states.clear(); let peers: Vec = self.sessions.keys().copied().collect(); for peer in peers { @@ -595,12 +616,20 @@ impl Runtime { reason, }); } + NetCommand::HealthProbe { peer, packet } => { + for ((p, _), link) in &self.links { + if *p == peer { + link.receive_probe(packet.clone()); + } + } + } NetCommand::ProtocolFailure { peer, protocol, reason, } => { - if self.links.remove(&(peer, protocol.clone())).is_some() { + if let Some(link) = self.links.remove(&(peer, protocol.clone())) { + link.close(); self.hub.clear_direct(peer, &protocol); } self.metrics.data_link_failures += 1; @@ -655,7 +684,7 @@ impl Runtime { peer: peer.fmt_short().to_string(), })?; - match session.outbound.try_send(encoded) { + match session.outbound.try_send(zeroize::Zeroizing::new(encoded)) { Ok(()) => { session.messages_sent += 1; session.bytes_sent += bytes; @@ -1423,6 +1452,7 @@ impl Runtime { } fn record_protocol_failure(&mut self, peer: EndpointId, protocol: &str) { + self.last_failed_protocol.insert(peer, protocol.to_string()); let key = (peer, protocol.to_string()); let entry = self .protocol_dial_states @@ -1461,7 +1491,17 @@ impl Runtime { } } for (peer, protocol) in dead { - self.links.remove(&(peer, protocol.clone())); + let reason = self + .links + .remove(&(peer, protocol.clone())) + .map(|link| { + let reason = link + .failure_reason() + .unwrap_or_else(|| "link closed".into()); + link.close(); + reason + }) + .unwrap_or_else(|| "link closed".into()); // Only the direct path is gone. Whether the peer is still // reachable through somebody is decided below; the protocol's // link stays either way. @@ -1471,7 +1511,7 @@ impl Runtime { network: self.network_id, peer, protocol, - reason: "link closed".into(), + reason, }); } @@ -1512,12 +1552,21 @@ impl Runtime { continue; } - let chosen = supported.iter().find(|proto| { - match self.protocol_dial_states.get(&(peer, (*proto).clone())) { - Some(state) => state.cooldown_until <= now, - None => true, - } - }); + let start = next_protocol_index( + &supported, + self.last_failed_protocol.get(&peer).map(String::as_str), + ); + let chosen = supported + .iter() + .cycle() + .skip(start) + .take(supported.len()) + .find( + |proto| match self.protocol_dial_states.get(&(peer, (*proto).clone())) { + Some(state) => state.cooldown_until <= now, + None => true, + }, + ); if let Some(proto) = chosen { wanted.push((peer, proto.clone())); @@ -1831,7 +1880,11 @@ impl Runtime { } fn install_link(&mut self, inbound: InboundLink) { - if self.no_direct_data(inbound.peer) { + if self.no_direct_data(inbound.peer) + || inbound.link.is_closed() + || !self.sessions.contains_key(&inbound.peer) + { + inbound.link.close(); return; } self.record_protocol_success(inbound.peer, &inbound.protocol); @@ -1842,6 +1895,14 @@ impl Runtime { /// Hands a link to the plugin that owns its protocol. fn adopt_link(&mut self, peer: EndpointId, protocol: String, link: SharedLink) { + if link.is_closed() + || link.network() != self.network_id + || link.peer() != peer + || !self.sessions.contains_key(&peer) + { + link.close(); + return; + } if !self .params .plugins @@ -1851,10 +1912,42 @@ impl Runtime { return; } + let key = (peer, protocol.clone()); + let settings = if self.local_id.as_bytes() < peer.as_bytes() { + let settings = HealthSettings::fresh(); + if let Ok(bytes) = postcard::to_stdvec(&(protocol.clone(), settings.clone())) { + let _ = self.send_to( + peer, + ControlMessage::Transport { + protocol: "link-health".into(), + payload: crate::proto::message::TransportPayload::new(bytes), + }, + ); + } + Some(settings) + } else { + self.pending_health.remove(&key) + }; + let route = self + .sessions + .get(&peer) + .and_then(|s| self.tunnel_protocol(s)) + .and_then(|name| self.params.plugins.iter().find(|p| p.protocol_id() == name)) + .map(|plugin| ProbeRoute { + plugin: plugin.clone(), + routes: self.params.routes.clone(), + network: self.network_id, + peer, + max_probe_size: self.params.interface.as_ref().map_or(540, |interface| { + (interface.mtu() as usize).saturating_sub(36) + }), + }); + let link = HealthLink::new(link, self.local_id, settings, route); let path = link.path_description(); let max_datagram = link.max_datagram_size(); - self.links - .insert((peer, protocol.clone()), Arc::clone(&link)); + if let Some(old) = self.links.insert(key, Arc::clone(&link)) { + old.close(); + } // What this is, is a way to reach the peer. Which protocol's // tunnel runs over it is decided in `update_paths`, and that tunnel // is handed the hub's link, not this one: the same object for as @@ -1878,6 +1971,8 @@ impl Runtime { self.hub.remove_peer(peer); self.reachable.remove(&peer); self.protocol_dial_states.retain(|(p, _), _| *p != peer); + self.last_failed_protocol.remove(&peer); + self.pending_health.retain(|(p, _), _| *p != peer); let keys: Vec<(EndpointId, String)> = self .links .keys() @@ -1885,7 +1980,9 @@ impl Runtime { .cloned() .collect(); for key in keys { - self.links.remove(&key); + if let Some(link) = self.links.remove(&key) { + link.close(); + } self.emit(Event::DataLinkDown { network: self.network_id, peer, @@ -2134,6 +2231,50 @@ impl Runtime { fn dispatch_message(&mut self, peer: EndpointId, message: ControlMessage) { match &message { + ControlMessage::Transport { protocol, payload } => { + if protocol == "link-health" { + // Only the elected initiator supplies a profile, for an enabled, + // mutually supported physical protocol in this authenticated session. + if peer.as_bytes() >= self.local_id.as_bytes() { + return; + } + let Ok((name, settings)) = + postcard::from_bytes::<(String, HealthSettings)>(payload.as_bytes()) + else { + return; + }; + if name.len() > crate::dataplane::MAX_PROTOCOL_ID_LEN + || !self.served_protocols().iter().any(|(p, v)| { + p == &name + && self.sessions.get(&peer).is_some_and(|s| { + s.capabilities + .iter() + .any(|c| c.enabled && c.protocol == name && c.version == *v) + }) + }) + { + return; + } + let key = (peer, name); + // Negotiation and inbound installation are separate queues. + // Keep the newest settings even if the previous physical + // link is still in the map when this control frame arrives. + if let Some(link) = self.links.get(&key) + && !link.is_closed() + { + link.configure(settings.clone()); + } + self.pending_health.insert(key, settings); + return; + } + if let Some(transport) = &self.params.transport + && let Err(err) = + transport.on_control(self.network_id, peer, protocol, payload.as_bytes()) + { + tracing::debug!(%err, "transport negotiation rejected"); + } + return; + } ControlMessage::Announce(announcement) => { let capabilities = announcement.capabilities.clone(); if let Some(session) = self.sessions.get_mut(&peer) { @@ -2652,3 +2793,17 @@ mod relay_host_tests { assert_eq!(relay_host("https://bad host/").as_deref(), None); } } + +#[cfg(test)] +mod fallback_order_tests { + use super::next_protocol_index; + #[test] + fn degradation_advances_the_cycle_even_after_earlier_cooldowns_expire() { + let protocols = ["wg", "awg", "wg-quic", "tcp-tls"].map(str::to_owned); + assert_eq!(next_protocol_index(&protocols, None), 0); + for (i, p) in protocols.iter().enumerate() { + assert_eq!(next_protocol_index(&protocols, Some(p)), (i + 1) % 4); + } + assert_eq!(next_protocol_index(&protocols, Some("removed")), 0); + } +} diff --git a/crates/tsunagi-core/src/agent/session.rs b/crates/tsunagi-core/src/agent/session.rs index 4543dd2..79ebe0f 100644 --- a/crates/tsunagi-core/src/agent/session.rs +++ b/crates/tsunagi-core/src/agent/session.rs @@ -70,7 +70,7 @@ pub(crate) struct Session { pub(crate) conn: Connection, /// Already encoded frame payloads, so the runtime can account for the exact /// number of control bytes it queues. - pub(crate) outbound: mpsc::Sender>, + pub(crate) outbound: mpsc::Sender>>, pub(crate) hostname: Option, pub(crate) capabilities: Vec, pub(crate) broadcast: bool, @@ -244,7 +244,7 @@ pub(crate) fn spawn( async fn writer_task( mut send: SendStream, - mut outbound: mpsc::Receiver>, + mut outbound: mpsc::Receiver>>, limits: Arc, shutdown: Shutdown, parent: Shutdown, @@ -331,6 +331,7 @@ async fn reader_task( } }; + let payload = zeroize::Zeroizing::new(payload); let bytes = payload.len(); let envelope: Envelope = match decode(&payload) { Ok(envelope) => envelope, diff --git a/crates/tsunagi-core/src/config.rs b/crates/tsunagi-core/src/config.rs index 9e3149b..9fe9df4 100644 --- a/crates/tsunagi-core/src/config.rs +++ b/crates/tsunagi-core/src/config.rs @@ -166,6 +166,8 @@ pub struct Limits { pub max_capabilities: usize, /// Largest opaque plugin payload, in bytes. pub max_capability_data_len: usize, + /// Largest private transport negotiation payload (also hard-capped at 4096 bytes). + pub max_transport_payload_len: usize, /// Largest accepted echo payload in a ping/pong exchange, in bytes. pub max_echo_payload_len: usize, /// Largest accepted free-text reason string, in bytes. @@ -204,6 +206,7 @@ impl Default for Limits { max_frame_len: 64 * 1024, max_hostname_len: 255, max_capabilities: 16, + max_transport_payload_len: 4096, max_capability_data_len: 4 * 1024, max_echo_payload_len: 4 * 1024, max_reason_len: 256, diff --git a/crates/tsunagi-core/src/dataplane/transport/health.rs b/crates/tsunagi-core/src/dataplane/transport/health.rs new file mode 100644 index 0000000..e2a63ef --- /dev/null +++ b/crates/tsunagi-core/src/dataplane/transport/health.rs @@ -0,0 +1,350 @@ +//! Authenticated probes of the actual data link, independent of control RTT. +//! Keys are negotiated privately for each physical link. No fixed wire magic. +use super::{PacketLink, SharedLink, TransportError}; +use crate::dataplane::SharedPlugin; +use crate::overlay::RoutingTable; +use crate::{BoxFuture, NetworkId}; +use bytes::Bytes; +use hmac::{Hmac, KeyInit, Mac}; +use iroh::EndpointId; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use tokio::sync::{mpsc, watch}; + +#[derive(Clone, Serialize, Deserialize, zeroize::ZeroizeOnDrop)] +pub(crate) struct Settings { + marker: [u8; 16], + key: [u8; 32], +} +impl Settings { + pub(crate) fn fresh() -> Self { + Self { + marker: rand::random(), + key: rand::random(), + } + } +} +impl std::fmt::Debug for Settings { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HealthSettings([REDACTED])") + } +} +struct Samples { + pending: VecDeque<(u64, Instant)>, + recent: VecDeque, +} +impl Samples { + fn new() -> Self { + Self { + pending: VecDeque::new(), + recent: VecDeque::new(), + } + } + fn sent(&mut self, seq: u64, now: Instant) { + self.pending.push_back((seq, now)); + } + fn acknowledge(&mut self, seq: u64) { + if let Some(i) = self.pending.iter().position(|(n, _)| *n == seq) { + self.pending.remove(i); + self.recent.push_back(true); + } + } + fn expire(&mut self, now: Instant) { + while self + .pending + .front() + .is_some_and(|(_, at)| now.duration_since(*at) >= Duration::from_secs(3)) + { + self.pending.pop_front(); + self.recent.push_back(false); + } + while self.recent.len() > 20 { + self.recent.pop_front(); + } + } + fn degraded(&self) -> bool { + self.recent.len() >= 15 + && self.recent.iter().filter(|ok| !**ok).count() * 100 >= self.recent.len() * 60 + } +} +// This prefix is inside end-to-end encryption. It never appears on the UDP wire. +const IP_MAGIC: &[u8; 16] = b"tsunagi-health01"; +pub(crate) fn ip_payload(packet: &[u8]) -> Option<&[u8]> { + (packet.len() >= 36 && packet[0] == 0x45 && packet[9] == 253 && &packet[20..36] == IP_MAGIC) + .then(|| &packet[36..]) +} +/// Inject probes through the same encrypted tunnel as application IP packets. +/// This matters for pure WG: raw UDP probes could pass DPI while WG is blocked. +pub(crate) struct ProbeRoute { + pub(crate) plugin: SharedPlugin, + pub(crate) routes: Arc, + pub(crate) network: NetworkId, + pub(crate) peer: EndpointId, + pub(crate) max_probe_size: usize, +} +impl ProbeRoute { + fn send(&self, payload: Bytes) -> bool { + let Some((source, destination)) = self.routes.probe_addresses(self.network, self.peer) + else { + return false; + }; + let mut packet = vec![0; 36]; + packet[0] = 0x45; + packet[8] = 64; + packet[9] = 253; + packet[12..16].copy_from_slice(&source.octets()); + packet[16..20].copy_from_slice(&destination.octets()); + packet[20..36].copy_from_slice(IP_MAGIC); + let length = (packet.len() + payload.len()) as u16; + packet[2..4].copy_from_slice(&length.to_be_bytes()); + let sum = packet[..20] + .as_chunks::<2>() + .0 + .iter() + .map(|b| u32::from(u16::from_be_bytes(*b))) + .sum::(); + let sum = (sum & 0xffff) + (sum >> 16); + packet[10..12].copy_from_slice(&(!(sum as u16)).to_be_bytes()); + packet.extend_from_slice(&payload); + self.plugin + .carry(self.network, self.peer, Bytes::from(packet)) + } +} +/// A monitored physical link. It forwards application bytes unchanged. +pub(crate) struct HealthLink { + probes: mpsc::Sender, + raw: SharedLink, + settings: Mutex>, + rx: tokio::sync::Mutex>, + stop: watch::Sender, + reason: Mutex>, +} +impl std::fmt::Debug for HealthLink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HealthLink") + .field("raw", &self.raw) + .finish_non_exhaustive() + } +} +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + match m.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + } +} +impl HealthLink { + pub(crate) fn new( + raw: SharedLink, + local: EndpointId, + settings: Option, + route: Option, + ) -> Arc { + let (tx, rx) = mpsc::channel(64); + let (probe_tx, mut probe_rx) = mpsc::channel(64); + let (stop, _) = watch::channel(false); + let link = Arc::new(Self { + probes: probe_tx, + raw: raw.clone(), + settings: Mutex::new(settings), + rx: tokio::sync::Mutex::new(rx), + stop, + reason: Mutex::new(None), + }); + let weak = Arc::downgrade(&link); + let mut stopped = link.stop.subscribe(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut samples = Samples::new(); + let mut sequence = rand::random::(); + let mut since: Option = None; + let mut marker = None; + loop { + tokio::select! { + biased; + _=stopped.changed()=>break, + _=interval.tick()=>{ + let Some(link)=weak.upgrade()else{break;};let Some(settings)=lock(&link.settings).clone()else{continue;}; + if marker != Some(settings.marker) { marker = Some(settings.marker); samples = Samples::new(); since = None; } + let start=*since.get_or_insert_with(Instant::now);let now=Instant::now();samples.expire(now); + if now.duration_since(start)>=Duration::from_secs(20) && samples.degraded() { + *lock(&link.reason)=Some("data delivery checks: at least 60% loss over the last 20 probes".into());break; + } + sequence=sequence.wrapping_add(1); + let size=route.as_ref().map_or(raw.max_datagram_size(),|route|route.max_probe_size); + let packet=probe(&settings,local,1,sequence,size.min(1000)); + if let Some(route)=&route { if route.send(packet) {samples.sent(sequence,now);} } + else { let _=raw.send(packet); samples.sent(sequence,now); } + } + packet=raw.recv()=>{ + let Some(packet)=packet else{break;};let Some(link)=weak.upgrade()else{break;}; + let settings=lock(&link.settings).clone(); + if route.is_none() && let Some(settings)=settings && packet.starts_with(&settings.marker) { + if let Some((kind,seq))=verify(&settings,raw.peer(),&packet) { + if kind==1 {let _=raw.send(probe(&settings,local,2,seq,packet.len()));} + else if kind==2 {samples.acknowledge(seq);} + } + } else {let _=tx.try_send(packet);} + } + packet=probe_rx.recv(),if route.is_some()=>{ + let Some(packet)=packet else {break;};let Some(link)=weak.upgrade()else{break;}; + let Some(settings)=lock(&link.settings).clone()else{continue;}; + if let Some((kind,seq))=verify(&settings,raw.peer(),&packet) { + if kind==1 { if let Some(route)=&route { route.send(probe(&settings,local,2,seq,packet.len().min(route.max_probe_size))); } } + else if kind==2 {samples.acknowledge(seq);} + } + } + + } + } + raw.close(); + if let Some(link) = weak.upgrade() { + link.stop.send_replace(true); + } + }); + link + } + pub(crate) fn receive_probe(&self, packet: Bytes) { + let _ = self.probes.try_send(packet); + } + pub(crate) fn configure(&self, settings: Settings) { + *lock(&self.settings) = Some(settings); + } +} +fn probe(settings: &Settings, sender: EndpointId, kind: u8, seq: u64, size: usize) -> Bytes { + let mut bytes = vec![0; size.max(57)]; + bytes[..16].copy_from_slice(&settings.marker); + bytes[16] = kind; + bytes[17..25].copy_from_slice(&seq.to_be_bytes()); + rand::fill(&mut bytes[25..size.max(57) - 32]); + let tail = bytes.len() - 32; + if let Ok(mut mac) = as KeyInit>::new_from_slice(&settings.key) { + mac.update(sender.as_bytes()); + mac.update(&bytes[..tail]); + bytes[tail..].copy_from_slice(&mac.finalize().into_bytes()); + } + Bytes::from(bytes) +} +fn verify(settings: &Settings, sender: EndpointId, bytes: &[u8]) -> Option<(u8, u64)> { + if bytes.len() < 57 || !bytes.starts_with(&settings.marker) { + return None; + } + let mut mac = as KeyInit>::new_from_slice(&settings.key).ok()?; + let tail = bytes.len() - 32; + mac.update(sender.as_bytes()); + mac.update(&bytes[..tail]); + mac.verify_slice(&bytes[tail..]).ok()?; + Some(( + bytes[16], + u64::from_be_bytes(bytes.get(17..25)?.try_into().ok()?), + )) +} +impl PacketLink for HealthLink { + fn network(&self) -> NetworkId { + self.raw.network() + } + fn peer(&self) -> EndpointId { + self.raw.peer() + } + fn max_datagram_size(&self) -> usize { + self.raw.max_datagram_size() + } + fn send(&self, payload: Bytes) -> Result<(), TransportError> { + if self.is_closed() { + Err(TransportError::Closed) + } else { + self.raw.send(payload) + } + } + fn send_flow( + &self, + payload: Bytes, + flow: super::super::routing::FlowId, + ) -> Result<(), TransportError> { + if self.is_closed() { + Err(TransportError::Closed) + } else { + self.raw.send_flow(payload, flow) + } + } + fn recv(&self) -> BoxFuture<'_, Option> { + Box::pin(async move { + let mut stop = self.stop.subscribe(); + if *stop.borrow() { + return None; + } + let mut rx = self.rx.lock().await; + tokio::select! {biased;_=stop.changed()=>None,packet=rx.recv()=>packet} + }) + } + fn closed(&self) -> BoxFuture<'_, ()> { + Box::pin(async move { + let mut stop = self.stop.subscribe(); + if !*stop.borrow() { + let _ = stop.changed().await; + } + }) + } + fn is_closed(&self) -> bool { + *self.stop.borrow() || self.raw.is_closed() + } + fn close(&self) { + self.stop.send_replace(true); + self.raw.close(); + } + fn failure_reason(&self) -> Option { + lock(&self.reason) + .clone() + .or_else(|| self.raw.failure_reason()) + } + fn path_description(&self) -> String { + self.raw.path_description() + } +} +impl Drop for HealthLink { + fn drop(&mut self) { + self.stop.send_replace(true); + self.raw.close(); + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[test] + fn no_false_failure_before_window_but_persistent_loss_is_detected() { + let mut samples = Samples::new(); + let now = Instant::now(); + for seq in 0..20 { + samples.sent(seq, now); + if seq % 5 == 0 { + samples.acknowledge(seq); + } + } + assert!(!samples.degraded()); + samples.expire(now + Duration::from_secs(4)); + assert!(samples.degraded()); + for seq in 20..40 { + samples.sent(seq, now); + samples.acknowledge(seq); + } + samples.expire(now + Duration::from_secs(4)); + assert!(!samples.degraded()); + } + #[test] + fn a_forged_or_reflected_probe_cannot_claim_delivery() { + let a = crate::identity::DeviceIdentity::generate().endpoint_id(); + let b = crate::identity::DeviceIdentity::generate().endpoint_id(); + let settings = Settings::fresh(); + let packet = probe(&settings, a, 1, 42, 1100); + assert_eq!(verify(&settings, a, &packet), Some((1, 42))); + assert!(verify(&settings, b, &packet).is_none()); + let mut forged = packet.to_vec(); + forged[16] = 2; + assert!(verify(&settings, a, &forged).is_none()); + } +} diff --git a/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs b/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs index 8f997f5..4cbd90f 100644 --- a/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs +++ b/crates/tsunagi-core/src/dataplane/transport/iroh_link.rs @@ -97,6 +97,9 @@ impl IrohLink { } impl PacketLink for IrohLink { + fn close(&self) { + self.conn.close(0u32.into(), b"data link retired"); + } fn network(&self) -> NetworkId { self.network } diff --git a/crates/tsunagi-core/src/dataplane/transport/mod.rs b/crates/tsunagi-core/src/dataplane/transport/mod.rs index 73e96d2..708fd30 100644 --- a/crates/tsunagi-core/src/dataplane/transport/mod.rs +++ b/crates/tsunagi-core/src/dataplane/transport/mod.rs @@ -30,12 +30,14 @@ //! one plugin protocol. mod fragments; +pub(crate) mod health; pub mod iroh_link; use std::sync::Arc; use bytes::Bytes; use iroh::EndpointId; +use serde::{Deserialize, Serialize}; use crate::BoxFuture; use crate::identity::NetworkId; @@ -112,6 +114,14 @@ pub trait PacketLink: Send + Sync + std::fmt::Debug + 'static { /// keeping a task parked on [`PacketLink::closed`]. fn is_closed(&self) -> bool; + /// Explicitly retires a physical link. Implementations should wake readers. + fn close(&self) {} + + /// Observed reason for retiring this link, if available. + fn failure_reason(&self) -> Option { + None + } + /// A short description of the path in use, for diagnostics. /// /// Reports what the transport actually knows. It must not invent a value. @@ -121,6 +131,76 @@ pub trait PacketLink: Send + Sync + std::fmt::Debug + 'static { /// A shared handle to a link. pub type SharedLink = std::sync::Arc; +/// Maximum opaque pair-negotiation payload. +pub const MAX_TRANSPORT_PAYLOAD: usize = 4096; + +/// Private transport bytes, never included in Debug output. +#[derive(Clone, PartialEq, Eq, Serialize, zeroize::ZeroizeOnDrop)] +pub struct TransportPayload(Vec); + +impl<'de> Deserialize<'de> for TransportPayload { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = TransportPayload; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "at most {MAX_TRANSPORT_PAYLOAD} private transport bytes") + } + fn visit_seq>( + self, + mut sequence: A, + ) -> std::result::Result { + if sequence + .size_hint() + .is_some_and(|size| size > MAX_TRANSPORT_PAYLOAD) + { + return Err(serde::de::Error::custom("transport payload too large")); + } + let mut payload = TransportPayload(Vec::with_capacity( + sequence.size_hint().unwrap_or(0).min(MAX_TRANSPORT_PAYLOAD), + )); + while let Some(byte) = sequence.next_element::()? { + if payload.0.len() == MAX_TRANSPORT_PAYLOAD { + return Err(serde::de::Error::custom("transport payload too large")); + } + payload.0.push(byte); + } + Ok(payload) + } + } + deserializer.deserialize_seq(Visitor) + } +} +impl TransportPayload { + /// Wraps bytes for one authenticated control session. + pub fn new(bytes: Vec) -> Self { + Self(bytes) + } + /// Borrows private bytes; callers must not log them. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} +impl std::fmt::Debug for TransportPayload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("TransportPayload([REDACTED])") + } +} + +/// Private control-plane sender for transports. Contains no network secret. +pub trait ControlSender: Send + Sync + std::fmt::Debug + 'static { + /// Queues a bounded message to one authenticated member of one network. + fn send<'a>( + &'a self, + network: NetworkId, + peer: EndpointId, + protocol: &'a str, + payload: TransportPayload, + ) -> BoxFuture<'a, Result<(), TransportError>>; +} + /// An inbound link a peer opened towards us. #[derive(Debug)] pub struct InboundLink { @@ -139,6 +219,18 @@ pub struct InboundLink { /// The agent owns one of these and hands links to plugins; plugins never call /// it directly. pub trait PacketTransport: Send + Sync + std::fmt::Debug + 'static { + /// Supplies a private sender before network activation. + fn attach_control(&self, _control: Arc) {} + /// Validates and queues private negotiation received from an authenticated member. + fn on_control( + &self, + _network: NetworkId, + _peer: EndpointId, + _protocol: &str, + _payload: &[u8], + ) -> Result<(), TransportError> { + Err(TransportError::Declined(self.name().into())) + } /// A short name used in diagnostics. fn name(&self) -> &str; @@ -184,6 +276,24 @@ impl MultiPacketTransport { } impl PacketTransport for MultiPacketTransport { + fn attach_control(&self, control: Arc) { + self.default.attach_control(control.clone()); + for transport in self.by_protocol.values() { + transport.attach_control(control.clone()); + } + } + fn on_control( + &self, + network: NetworkId, + peer: EndpointId, + protocol: &str, + payload: &[u8], + ) -> Result<(), TransportError> { + match self.by_protocol.get(protocol) { + Some(transport) => transport.on_control(network, peer, protocol, payload), + None => Err(TransportError::Declined(protocol.into())), + } + } fn name(&self) -> &str { "multi" } diff --git a/crates/tsunagi-core/src/overlay/router.rs b/crates/tsunagi-core/src/overlay/router.rs index a894c5d..93d110a 100644 --- a/crates/tsunagi-core/src/overlay/router.rs +++ b/crates/tsunagi-core/src/overlay/router.rs @@ -384,6 +384,24 @@ impl RoutingTable { conflicts } + /// Current signed allocations used for encrypted data delivery checks. + pub(crate) fn probe_addresses( + &self, + network: NetworkId, + peer: EndpointId, + ) -> Option<(Ipv4Addr, Ipv4Addr)> { + let table = self.read(); + let routes = table.get(&network)?; + Some(( + routes.local?, + routes + .peers + .iter() + .find(|(_, holder)| *holder == peer) + .map(|(address, _)| *address)?, + )) + } + /// How many networks the table covers. pub fn len(&self) -> usize { self.read().len() diff --git a/crates/tsunagi-core/src/proto/message.rs b/crates/tsunagi-core/src/proto/message.rs index 653ced1..a6acc2b 100644 --- a/crates/tsunagi-core/src/proto/message.rs +++ b/crates/tsunagi-core/src/proto/message.rs @@ -20,7 +20,7 @@ use crate::error::ProtocolError; /// The version in the ALPN is the wire-compatibility version of the control /// protocol. It is independent of the network identity scheme version, so /// bumping it must not change any existing [`crate::NetworkId`]. -pub const ALPN: &[u8] = b"tsunagi/ctrl/5"; +pub const ALPN: &[u8] = b"tsunagi/ctrl/6"; /// ALPN of the tsunagi data plane. /// @@ -43,7 +43,9 @@ pub const MAX_DATA_PROTOCOL_LEN: usize = 32; pub const MAX_SIGNATURE_LEN: usize = 64; /// Control protocol version carried inside the handshake. -pub const PROTOCOL_VERSION: u16 = 5; +pub const PROTOCOL_VERSION: u16 = 6; + +pub use crate::dataplane::transport::{MAX_TRANSPORT_PAYLOAD, TransportPayload}; /// First message of the handshake, sent by the initiator. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -114,6 +116,13 @@ pub struct DataOpenAck { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum ControlMessage { + /// Private transport negotiation; never persisted or rebroadcast. + Transport { + /// Transport protocol id; the payload stays opaque to the core. + protocol: String, + /// Secret negotiation data, redacted in diagnostics. + payload: TransportPayload, + }, /// Hostname and capability announcement. Announce(Announcement), /// A small request used to verify that the exchange works. @@ -264,6 +273,17 @@ pub fn validate_capability( /// network, the other networks or the agent. pub fn validate(message: &ControlMessage, limits: &Limits) -> Result<(), ProtocolError> { match message { + ControlMessage::Transport { protocol, payload } => { + if protocol.is_empty() { + return Err(ProtocolError::Malformed("empty transport protocol")); + } + check_len("transport.protocol", protocol.len(), MAX_PROTOCOL_ID_LEN)?; + check_len( + "transport.payload", + payload.as_bytes().len(), + limits.max_transport_payload_len.min(MAX_TRANSPORT_PAYLOAD), + )?; + } ControlMessage::Peers { peers } => { check_len("peers", peers.len(), limits.max_state_records)?; for hint in peers { @@ -320,6 +340,7 @@ pub fn validate(message: &ControlMessage, limits: &Limits) -> Result<(), Protoco /// A short, stable label for a message kind, for metrics and diagnostics. pub fn kind(message: &ControlMessage) -> &'static str { match message { + ControlMessage::Transport { .. } => "transport", ControlMessage::Announce(_) => "announce", ControlMessage::Ping { .. } => "ping", ControlMessage::Pong { .. } => "pong", diff --git a/crates/tsunagi-core/tests/transport_control.rs b/crates/tsunagi-core/tests/transport_control.rs new file mode 100644 index 0000000..2bd1954 --- /dev/null +++ b/crates/tsunagi-core/tests/transport_control.rs @@ -0,0 +1,130 @@ +//! Opaque transport negotiation stays inside authenticated network sessions. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::sync::{Arc, Mutex}; +use tsunagi::dataplane::transport::{ControlSender, PacketTransport, SharedLink, TransportError}; +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::proto::message::{ControlMessage, TransportPayload}; +use tsunagi::testing::{TestAgent, network, wait_for_peers, wait_until}; +use tsunagi::{BoxFuture, NetworkId}; + +#[derive(Debug, Default)] +struct Negotiator { + control: Mutex>>, + received: Mutex)>>, +} +impl PacketTransport for Negotiator { + fn name(&self) -> &str { + "negotiator" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn attach_control(&self, control: Arc) { + *self.control.lock().unwrap() = Some(control); + } + fn on_control( + &self, + network: NetworkId, + peer: iroh::EndpointId, + _: &str, + payload: &[u8], + ) -> Result<(), TransportError> { + self.received + .lock() + .unwrap() + .push((network, peer, payload.to_vec())); + Ok(()) + } + fn open<'a>( + &'a self, + _: NetworkId, + _: iroh::EndpointId, + _: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async { Err(TransportError::Closed) }) + } +} + +#[tokio::test] +async fn negotiated_secrets_are_redacted_scoped_and_delivered_over_real_control() { + let discovery = SharedMemoryDiscovery::new(); + let a_transport = Arc::new(Negotiator::default()); + let b_transport = Arc::new(Negotiator::default()); + let a = TestAgent::spawn_with( + |c| c.with_custom_transport("negotiator", a_transport.clone()), + &discovery, + ) + .await + .unwrap(); + let b = TestAgent::spawn_with( + |c| c.with_custom_transport("negotiator", b_transport.clone()), + &discovery, + ) + .await + .unwrap(); + let (name, secret) = network("private-negotiation"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&a.agent, id, 1).await; + let message = ControlMessage::Transport { + protocol: "negotiator".into(), + payload: TransportPayload::new(b"never-export-this-secret".to_vec()), + }; + assert!(!format!("{message:?}").contains("never-export")); + let sender = a_transport.control.lock().unwrap().clone().unwrap(); + sender + .send( + id, + b.agent.endpoint_id(), + "negotiator", + TransportPayload::new(b"never-export-this-secret".to_vec()), + ) + .await + .unwrap(); + wait_until("private transport message", || async { + (!b_transport.received.lock().unwrap().is_empty()).then_some(()) + }) + .await; + assert_eq!( + b_transport.received.lock().unwrap()[0], + ( + id, + a.agent.endpoint_id(), + b"never-export-this-secret".to_vec() + ) + ); + let unknown = NetworkId::from_bytes([0xa5; 32]); + assert!( + sender + .send( + unknown, + b.agent.endpoint_id(), + "negotiator", + TransportPayload::new(vec![1]) + ) + .await + .is_err() + ); + a.agent.shutdown().await; + b.agent.shutdown().await; +} + +#[test] +fn private_bytes_are_bounded_on_decode_and_by_local_policy() { + let limits = tsunagi::Limits { + max_transport_payload_len: 4, + ..Default::default() + }; + let message = |len| ControlMessage::Transport { + protocol: "awg".into(), + payload: TransportPayload::new(vec![42; len]), + }; + assert!(tsunagi::proto::message::validate(&message(4), &limits).is_ok()); + assert!(tsunagi::proto::message::validate(&message(5), &limits).is_err()); + let bytes = postcard::to_stdvec(&TransportPayload::new(vec![1; 4097])).unwrap(); + assert!(postcard::from_bytes::(&bytes).is_err()); + let bytes = postcard::to_stdvec(&message(4)).unwrap(); + let decoded: ControlMessage = postcard::from_bytes(&bytes).unwrap(); + assert_eq!(decoded, message(4)); +} diff --git a/crates/tsunagi-tcp-tls/src/transport.rs b/crates/tsunagi-tcp-tls/src/transport.rs index c9d6fce..0d469df 100644 --- a/crates/tsunagi-tcp-tls/src/transport.rs +++ b/crates/tsunagi-tcp-tls/src/transport.rs @@ -115,6 +115,7 @@ pub struct TcpTlsLink { peer: EndpointId, tx: mpsc::Sender, rx: tokio::sync::Mutex>, + tasks: [tokio::task::AbortHandle; 2], closed_notify: Arc, is_closed: Arc, remote_addr: String, @@ -136,7 +137,7 @@ impl TcpTlsLink { let closed_flag_w = Arc::clone(&is_closed); let closed_notify_w = Arc::clone(&closed_notify); - tokio::spawn(async move { + let writer_task = tokio::spawn(async move { while let Some(msg) = rx_out.recv().await { if msg.len() > MAX_FRAME_SIZE { continue; @@ -158,7 +159,7 @@ impl TcpTlsLink { let closed_flag_r = Arc::clone(&is_closed); let closed_notify_r = Arc::clone(&closed_notify); - tokio::spawn(async move { + let reader_task = tokio::spawn(async move { let mut len_buf = [0u8; 2]; loop { if reader.read_exact(&mut len_buf).await.is_err() { @@ -185,6 +186,7 @@ impl TcpTlsLink { peer, tx: tx_out, rx: tokio::sync::Mutex::new(rx_in), + tasks: [writer_task.abort_handle(), reader_task.abort_handle()], closed_notify, is_closed, remote_addr, @@ -192,7 +194,19 @@ impl TcpTlsLink { } } +impl Drop for TcpTlsLink { + fn drop(&mut self) { + self.close(); + } +} impl PacketLink for TcpTlsLink { + fn close(&self) { + self.is_closed.store(true, Ordering::Release); + self.closed_notify.notify_waiters(); + for task in &self.tasks { + task.abort(); + } + } fn network(&self) -> NetworkId { self.network } diff --git a/crates/tsunagi-wg-quic/src/plugin.rs b/crates/tsunagi-wg-quic/src/plugin.rs index a9f9068..3f6f695 100644 --- a/crates/tsunagi-wg-quic/src/plugin.rs +++ b/crates/tsunagi-wg-quic/src/plugin.rs @@ -79,6 +79,8 @@ pub const WIREGUARD_OVERHEAD: u32 = 32; /// Configuration of the WireGuard plugin. #[derive(Debug, Clone)] pub struct WireguardConfig { + /// Diagnostic protocol id when this end-to-end engine is reused by another carrier. + pub protocol: &'static str, /// Directory for the plugin's own key store. Separate from agent state. pub state_dir: PathBuf, /// WireGuard keepalive, which keeps tunnels and their links warm. @@ -100,6 +102,7 @@ impl WireguardConfig { /// Creates a configuration rooted at `state_dir` with sensible defaults. pub fn new(state_dir: impl Into) -> Self { Self { + protocol: WIREGUARD_PROTOCOL, state_dir: state_dir.into(), keepalive: Some(25), mtu: DEFAULT_MTU, @@ -394,7 +397,7 @@ impl Worker { fn report(&self, network: NetworkId, reason: impl std::fmt::Display) { tracing::warn!(network = %network.fmt_short(), %reason, "wireguard plugin error"); if let Some(context) = self.context.get() { - context.report_error(network, WIREGUARD_PROTOCOL, reason.to_string()); + context.report_error(network, self.config.protocol, reason.to_string()); } } @@ -647,7 +650,7 @@ async fn run(worker: Arc, mut commands: mpsc::Receiver) { impl IpPlugin for WireguardPlugin { fn protocol_id(&self) -> &str { - WIREGUARD_PROTOCOL + self.worker.config.protocol } fn protocol_version(&self) -> u16 { @@ -686,7 +689,7 @@ impl IpPlugin for WireguardPlugin { // Identity only. Where to send packets is the transport's business. let announcement = WgAnnouncement::new(network, &state.key.public()); Ok(Some(PluginCapability { - protocol: WIREGUARD_PROTOCOL.to_string(), + protocol: self.worker.config.protocol.to_string(), version: crate::announcement::ANNOUNCEMENT_VERSION, enabled: true, data: announcement.encode()?, diff --git a/crates/tsunagi-wireguard/src/transport.rs b/crates/tsunagi-wireguard/src/transport.rs index adcfc61..bbb3929 100644 --- a/crates/tsunagi-wireguard/src/transport.rs +++ b/crates/tsunagi-wireguard/src/transport.rs @@ -231,6 +231,9 @@ impl WireguardLink { } impl PacketLink for WireguardLink { + fn close(&self) { + WireguardLink::close(self); + } fn network(&self) -> NetworkId { self.network } diff --git a/crates/tsunagi/Cargo.toml b/crates/tsunagi/Cargo.toml index 461e603..a4bea6c 100644 --- a/crates/tsunagi/Cargo.toml +++ b/crates/tsunagi/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" [dependencies] tsunagi-core = { path = "../tsunagi-core", version = "0.1.0", features = ["tun-device", "dns-publish"] } tsunagi-wireguard = { path = "../tsunagi-wireguard", version = "0.1.0" } +tsunagi-awg = { path = "../tsunagi-awg", version = "0.1.0" } tsunagi-wg-quic = { path = "../tsunagi-wg-quic", version = "0.1.0" } tsunagi-tcp-tls = { path = "../tsunagi-tcp-tls", version = "0.1.0" } iroh.workspace = true diff --git a/crates/tsunagi/src/main.rs b/crates/tsunagi/src/main.rs index 1911611..953d917 100644 --- a/crates/tsunagi/src/main.rs +++ b/crates/tsunagi/src/main.rs @@ -26,6 +26,7 @@ use tsunagi::iroh_types::{EndpointAddr, EndpointId}; use tsunagi::overlay::{MemoryTunFactory, TunFactory}; use tsunagi::state::Ipv4Range; use tsunagi::{Agent, NetworkId}; +use tsunagi_awg::{AWG_PROTOCOL, AWG_WIRE_VERSION, AwgPlugin, AwgTransport}; use tsunagi_tcp_tls::{ TCP_TLS_PROTOCOL, TCP_TLS_VERSION, TcpTlsCodec, TcpTlsConfig, TcpTlsPlugin, TcpTlsTransport, }; @@ -473,7 +474,7 @@ struct UpArgs { long = "protocol", value_name = "LIST", value_delimiter = ',', - default_value = "wg,tcp-tls,wg-quic", + default_value = "wg,awg,wg-quic,tcp-tls", help_heading = "Transport" )] protocols: Vec, @@ -1429,22 +1430,27 @@ const PROTOCOLS: &[ProtocolSpec] = &[ ProtocolSpec { name: WG_PROTOCOL, version: WG_WIRE_VERSION, - summary: "Pure WireGuard direct UDP transport for uncensored networks and maximum performance", + summary: "Pure WireGuard direct UDP transport", options: PureWgPlugin::OPTIONS, }, ProtocolSpec { - name: TCP_TLS_PROTOCOL, - version: TCP_TLS_VERSION, - summary: "Noise E2EE carried over TCP TLS 1.3 (port 443) disguised as HTTPS traffic", - options: TcpTlsCodec::OPTIONS, + name: AWG_PROTOCOL, + version: AWG_WIRE_VERSION, + summary: "Pairwise Amnezia-masked UDP with fresh private profiles on every connection attempt", + options: AwgPlugin::OPTIONS, }, ProtocolSpec { name: tsunagi_wg_quic::WIREGUARD_PROTOCOL, version: tsunagi_wg_quic::ANNOUNCEMENT_VERSION, - summary: "WireGuard's cryptography carried in iroh's QUIC datagrams, so it \ - crosses NAT and survives where plain WireGuard is blocked", + summary: "WireGuard encryption over iroh QUIC datagrams with NAT traversal and relays", options: WireguardPlugin::OPTIONS, }, + ProtocolSpec { + name: TCP_TLS_PROTOCOL, + version: TCP_TLS_VERSION, + summary: "Noise E2EE carried over TCP TLS 1.3", + options: TcpTlsCodec::OPTIONS, + }, ]; /// One `-o` setting, and the protocol it was aimed at. @@ -1534,6 +1540,7 @@ fn show_protocols() -> Result<(), Box> { struct AgentControl { agent: Agent, pure_wg: Option>, + awg: Option>, plugin: Option>, tcp_tls: Option>, /// The resolver, which this owns so it can be switched while running. @@ -1549,6 +1556,7 @@ impl tsunagi::ipc::ReportSource for AgentControl { build_report( &self.agent, self.pure_wg.as_deref(), + self.awg.as_deref(), self.plugin.as_deref(), self.tcp_tls.as_deref(), dns, @@ -4483,6 +4491,8 @@ async fn up(args: UpArgs) -> Result<(), Box> { // The interface belongs to the agent, so it is configured once whatever // was selected to carry traffic over it. let mut wireguard = None; + let mut awg_transport = None; + let mut awg_plugin = None; let mut tcp_tls_transport = None; let mut tcp_tls_plugin = None; let mut pure_wg_plugin: Option> = None; @@ -4500,6 +4510,48 @@ async fn up(args: UpArgs) -> Result<(), Box> { for spec in &wanted { let options = settings_for(spec, &settings); match spec.name { + AWG_PROTOCOL => { + let mut wg = WireguardConfig::new(paths.state_dir.join("awg")); + if let Some(mtu) = args.mtu { + wg = wg.with_mtu(mtu); + } + let tunnel_options: Vec<_> = options + .iter() + .filter(|(key, _)| key != "stun") + .cloned() + .collect(); + let wg = WireguardPlugin::configure(wg, &tunnel_options)?; + let server = options + .iter() + .find(|(key, _)| key == "stun") + .map(|(_, value)| value.as_str()) + .unwrap_or("stun.l.google.com:19302"); + if server.len() > 255 || (server != "off" && !server.contains(':')) { + return Err("awg:stun must be HOST:PORT or off".into()); + } + let stun = (!matches!(args.reach, Reach::Local) && server != "off") + .then(|| server.to_owned()); + let state_db = paths.state_db(); + let identity = tokio::task::spawn_blocking(move || { + tsunagi::storage::StateStore::open(state_db)?.load_or_create_device_identity() + }) + .await??; + let transport = AwgTransport::new_with_stun( + identity.endpoint_id(), + std::net::Ipv4Addr::UNSPECIFIED, + stun, + ); + let plugin = AwgPlugin::open(wg, transport.clone()).await?; + config = config + .with_plugin(plugin.clone() as Arc) + .with_custom_transport( + AWG_PROTOCOL, + transport.clone() + as Arc, + ); + awg_transport = Some(transport); + awg_plugin = Some(plugin); + } tsunagi_wg_quic::WIREGUARD_PROTOCOL => { let mut wg = WireguardConfig::new(paths.state_dir.join("wg-quic")); if let Some(mtu) = args.mtu { @@ -4603,6 +4655,16 @@ async fn up(args: UpArgs) -> Result<(), Box> { // is never dropped without being closed. let mut events = agent.subscribe(); + if let Some(transport) = awg_transport { + let inbound_agent = agent.clone(); + tokio::spawn(async move { + while let Some(inbound) = transport.next_inbound().await { + if let Err(err) = inbound_agent.install_inbound_link(inbound).await { + tracing::debug!(%err, "failed to install inbound awg link"); + } + } + }); + } if let Some(transport) = tcp_tls_transport { let agent_inbound = agent.clone(); tokio::spawn(async move { @@ -4670,12 +4732,14 @@ async fn up(args: UpArgs) -> Result<(), Box> { let control = { let agent = agent.clone(); let pure_wg = pure_wg_plugin.clone(); + let awg = awg_plugin.clone(); let plugin = wireguard.clone(); let tcp_tls = tcp_tls_plugin.clone(); let dns = Arc::clone(&dns); let source: Arc = Arc::new(AgentControl { agent, pure_wg, + awg, plugin, tcp_tls, dns, @@ -4761,6 +4825,7 @@ async fn up(args: UpArgs) -> Result<(), Box> { async fn build_report( agent: &Agent, pure_wg: Option<&PureWgPlugin>, + awg: Option<&AwgPlugin>, wireguard: Option<&WireguardPlugin>, tcp_tls: Option<&TcpTlsPlugin>, dns: Option, @@ -4781,6 +4846,7 @@ async fn build_report( .networks .iter() .map(|network| { + let awg_view = awg.and_then(|plugin| plugin.overview(network.network_id)); let wg_view = wireguard.and_then(|plugin| plugin.overview(network.network_id)); let pure_wg_view = pure_wg.and_then(|plugin| plugin.overview(network.network_id)); let tcp_tls_reports = tcp_tls @@ -4793,14 +4859,18 @@ async fn build_report( let wg_overlay_addr = wg_view .as_ref() .and_then(|v| v.overlay_address_v4) - .or_else(|| pure_wg_view.as_ref().and_then(|v| v.overlay_address_v4)); + .or_else(|| pure_wg_view.as_ref().and_then(|v| v.overlay_address_v4)) + .or_else(|| awg_view.as_ref().and_then(|v| v.overlay_address_v4)); let wg_prefix_len = wg_view .as_ref() .and_then(|v| v.ipv4_range) .or_else(|| pure_wg_view.as_ref().and_then(|v| v.ipv4_range)) + .or_else(|| awg_view.as_ref().and_then(|v| v.ipv4_range)) .map_or(0, |r| r.prefix_len); - let has_any_plugin = - wg_view.is_some() || pure_wg_view.is_some() || !tcp_tls_reports.is_empty(); + let has_any_plugin = wg_view.is_some() + || pure_wg_view.is_some() + || awg_view.is_some() + || !tcp_tls_reports.is_empty(); let overlay = if has_any_plugin && overlay.is_some() { let mut peers: Vec = Vec::new(); @@ -4828,6 +4898,13 @@ async fn build_report( } } } + if let Some(ref view) = awg_view { + for peer in &view.peers { + if !all_peer_ids.contains(&peer.endpoint_id) { + all_peer_ids.push(peer.endpoint_id); + } + } + } for report in &tcp_tls_reports { if !all_peer_ids.contains(&report.peer) { all_peer_ids.push(report.peer); @@ -4838,6 +4915,9 @@ async fn build_report( let pure_wg_match = pure_wg_view .as_ref() .and_then(|v| v.peers.iter().find(|p| p.endpoint_id == peer_id)); + let awg_match = awg_view + .as_ref() + .and_then(|v| v.peers.iter().find(|p| p.endpoint_id == peer_id)); let tcp_tls_match = tcp_tls_reports.iter().find(|r| r.peer == peer_id); let wg_quic_match = wg_view .as_ref() @@ -4875,6 +4955,7 @@ async fn build_report( let pubkey = pure_wg_match .map(|p| p.public_key.to_string()) .or_else(|| wg_quic_match.map(|p| p.public_key.to_string())) + .or_else(|| awg_match.map(|p| p.public_key.to_string())) .unwrap_or_default(); peers.push(OverlayPeerReport { endpoint_id: peer_id.to_string(), @@ -4890,6 +4971,27 @@ async fn build_report( protocol_errors: 0, path: tcp_tls.path.clone(), }); + } else if let Some(awg_peer) = awg_match + && let Some(tunnel) = &awg_peer.tunnel + { + peers.push(OverlayPeerReport { + endpoint_id: peer_id.to_string(), + protocol: AWG_PROTOCOL.into(), + public_key: awg_peer.public_key.to_string(), + address: awg_peer + .overlay_address_v4 + .map(|a| a.to_string()) + .or(member_addr), + handshake_secs_ago: tunnel.health.since_handshake.map(|s| s.as_secs()), + tx_packets: tunnel.stats.tx_packets, + rx_packets: tunnel.stats.rx_packets, + tx_bytes: tunnel.stats.tx_bytes, + rx_bytes: tunnel.stats.rx_bytes, + dropped: tunnel.stats.dropped_wrong_source + + tunnel.stats.dropped_oversize, + protocol_errors: tunnel.stats.protocol_errors, + path: tunnel.path.clone(), + }); } else if let Some(wg_peer) = wg_quic_match && let Some(tunnel) = &wg_peer.tunnel { @@ -4929,6 +5031,24 @@ async fn build_report( protocol_errors: 0, path: "no data link".into(), }); + } else if let Some(awg_peer) = awg_match { + peers.push(OverlayPeerReport { + endpoint_id: peer_id.to_string(), + protocol: AWG_PROTOCOL.into(), + public_key: awg_peer.public_key.to_string(), + address: awg_peer + .overlay_address_v4 + .map(|a| a.to_string()) + .or(member_addr), + handshake_secs_ago: None, + tx_packets: 0, + rx_packets: 0, + tx_bytes: 0, + rx_bytes: 0, + dropped: 0, + protocol_errors: 0, + path: "no data link".into(), + }); } else if let Some(wg_peer) = wg_quic_match { peers.push(OverlayPeerReport { endpoint_id: peer_id.to_string(), @@ -6041,6 +6161,34 @@ mod status_tests { #[cfg(test)] mod network_tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + #[test] + fn default_carriers_include_awg_and_explicit_order_is_preserved() { + use clap::Parser; + for (words, expected) in [ + (vec!["tsng", "up"], vec!["wg", "awg", "wg-quic", "tcp-tls"]), + ( + vec![ + "tsng", + "up", + "--protocol", + "awg,wg-quic", + "-o", + "awg:stun=off", + ], + vec!["awg", "wg-quic"], + ), + ] { + let super::Command::Up(args) = super::Cli::try_parse_from(words).unwrap().command + else { + panic!("expected up"); + }; + assert_eq!(args.protocols, expected); + for name in expected { + assert!(super::PROTOCOLS.iter().any(|spec| spec.name == name)); + } + } + } + #[test] fn dht_defaults_and_local_mode_do_not_require_manual_peers() { use clap::Parser; diff --git a/docs/amnezia.md b/docs/amnezia.md new file mode 100644 index 0000000..1741c2b --- /dev/null +++ b/docs/amnezia.md @@ -0,0 +1,116 @@ +# Pairwise Amnezia transport + +The CLI's default carrier preference is `wg,awg,wg-quic,tcp-tls`. To select +AWG explicitly: + +```sh +tsng up --protocol awg +tsng up --protocol wg,awg,wg-quic,tcp-tls +``` + +`awg` is Tsunagi AWG wire version 1. Its packet masking engine is adapted from +[RunaRoo/AmneziaWG-RS](https://github.com/RunaRoo/AmneziaWG-RS), with the pinned +source and license recorded in +[UPSTREAM.md](../crates/tsunagi-awg/UPSTREAM.md). It uses the existing boringtun +Noise/WireGuard implementation instead of upstream's Node, global executors +and OS interface machinery. Compatibility with ordinary Amnezia servers is +not implemented: this carrier connects authenticated Tsunagi members. + +## Negotiation and lifetime + +There is one independent UDP socket and cancellable packet-engine task per +`(network, peer)` pair. The smaller endpoint id initiates, following the same +rule as other data carriers. Both sides must announce enabled AWG wire version +1. An authenticated iroh control session is required throughout; AWG supplies +no alternate discovery or membership mechanism. + +Every `open` generates new masking parameters, a header-protection key, an +outer WireGuard key pair and a PSK. The initiator sends a private `Offer` with +a random attempt id, its public key, candidate socket addresses and the shared +profile. The responder validates it, creates its own fresh key pair and socket, +and returns an `Answer` naming that attempt. Both send masked handshakes and +encrypted delivery probes to the candidates. Only authenticated WireGuard +**data**, not a STUN response or an incoming UDP address alone, verifies a path. + +Negotiation payloads are bounded at 4096 bytes (a caller can lower the limit), +redacted in Debug, excluded from public message events and erased on drop. +They are never capabilities, signed records, cache entries or exported status. +At most 128 pairs are admitted per transport, with bounded command/data queues, +at most 32 candidates per attempt and finite establishment deadlines. +Stale answers and duplicate offers are rejected; a new attempt replaces and +closes an older pair engine. Cancellation, peer departure, network deactivation +and shutdown retire sockets without waiting for a remote reader. + +A process restart, a failed setup, or a complete fallback cycle returning to AWG +starts fresh negotiation. Normal WireGuard rekeying within a live attempt keeps +that attempt's masking profile. A failed pair does not reconfigure other pairs +or other networks. A full mesh of 15 members uses 14 engines per member and 105 +independent pairs in total; it does not create additional system interfaces. + +## Masking + +Each profile randomizes disjoint H1–H4 header ranges, a 12–48-byte S1–S4 prefix, +a ChaCha20 header-protection key, handshake trailers, junk packet count/lengths, +and extra content padding. Prefix lengths are equal within one profile so +header detection has one offset. A fresh random nonce is in every packet's +prefix; handshake headers and transport headers/counters are protected. +The standard WireGuard header is restored before boringtun authenticates a +message. This preserves boringtun's standard MAC calculations, which is another +reason this is a Tsunagi-specific AWG wire version. + +Before a handshake, an I1 signature packet is generated from either a complete +DNS query or SIP OPTIONS template, with fresh identifiers and consistent length +fields, followed by random junk packets. I2–I5 are unused in this version. +The templates are structural generators, not replayed captures. QUIC imitation +is not selected. Masking can change DPI classification; it is not a promise of +indistinguishability or that any particular firewall will admit it. + +## Packets, NAT and fallback + +The pair engine carries opaque relay envelopes inside authenticated outer WG +packets. It fragments those envelopes into bounded chunks and expires incomplete +assemblies; a logical datagram is bounded at 64 KiB and an outer UDP packet is +at most 1200 bytes. The encapsulated IPv4 headers are internal framing only: no interface or OS address is created for a +pair. Overlay addresses, routing, ownership checks and end-to-end encryption +remain independent. A relay decrypts its own adjacent AWG carrier to forward +opaque end-to-end ciphertext, never the destination's IP packet. + +Local IPv4 candidates and simultaneous encrypted probes provide direct UDP +connectivity. In `direct`/`relay` reach mode the CLI also uses +`stun.l.google.com:19302` to discover a server-reflexive address on **this pair's +socket**, with a 1.2-second bound. A STUN result is only a candidate. Local reach +mode makes no STUN request. Configure or disable it with +`-o awg:stun=HOST:PORT` or `-o awg:stun=off`. Failed discovery still permits local +candidates. IPv6-only paths, symmetric NAT and networks blocking arbitrary UDP +may fail AWG and fall back; there is no AWG-over-QUIC substitution or AWG TURN +relay hidden behind its name. + +Data delivery checks go through the actual end-to-end encrypted tunnel, once +per second, independently of control-plane RTT. Requests and replies bind a +fresh private marker/key to the sender's endpoint id. Unanswered requests expire +after three seconds. After a 20-second warmup, at least 15 completed observations +and at least 60% loss in the last 20 observations retire a carrier. Probes are +consumed locally after source/destination admission and never enter the TUN. +AWG additionally checks encrypted outer delivery throughout the pair lifetime. + +Setup failure or persistent runtime loss advances to the next mutually supported +carrier in the initiator's order. Earlier expired cooldowns do not jump ahead of +untried carriers. Retry cooldowns remain bounded; reaching AWG again negotiates +new parameters. Control sessions and the logical end-to-end tunnel survive a +carrier change. Status names the end-to-end protocol and the actual carrier +separately, so `wg` over `awg` is a valid result. + +Control wire version 6 introduces the private transport message. All members +must upgrade together; device identities, network ids, handshake transcript +encoding, signed state and stored network configuration are unchanged. Data +ALPN remains `tsunagi/data/5`; the relay envelope is unchanged. + +## Verification + +The default rootless, offline tests use real iroh endpoints, membership proofs, +SQLite, UDP sockets, masked WireGuard handshakes, end-to-end encryption and +memory TUNs. They cover independent three-member pairs, full-size bidirectional +packets, restart freshness/socket release, 80% runtime datagram loss, fallback +to real iroh and fresh negotiation on retry, plus malformed profiles/messages, +role checks, probe forgery and STUN response parsing. These tests do not establish +DPI effectiveness, WAN throughput, public NAT traversal or per-pair memory cost. diff --git a/docs/architecture.md b/docs/architecture.md index 7b5c5b5..a1be023 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -20,8 +20,9 @@ network secret. The core moves an opaque, bounded payload and never parses it. a `PacketLink` — an authenticated, unreliable datagram channel to one peer for one protocol — and never learn how it is carried. -The separation between the two planes is **logical, not physical**. Both ride -on iroh, on different ALPNs and different connections. That is deliberate: +The separation between the two planes is **logical, not physical**. The control plane uses +iroh; the data plane can use iroh on its own ALPN and connection, or independent +UDP/TLS carriers. That is deliberate: iroh's whole value is hole punching a direct path between peers behind NAT, with a relay as fallback, and a data plane that refused to use it would have to reimplement all of it. What the separation buys is that `proto` knows nothing @@ -32,6 +33,10 @@ A plugin talks to the core through three narrow hooks — `on_network_activated` a `PluginContext` for re-announcements and error reports, and a bounded `shutdown` — so the core never learns anything protocol-specific. +Private pair negotiation uses a bounded transport sender supplied by the core. +Protocol crates do not encode or parse control messages and never receive the +network secret. No pair profiles enter signed state. + A data plane failure never stops the daemon: the control plane keeps running and the agent stays manageable. @@ -68,6 +73,7 @@ And in crates of their own: | crate | responsibility | |---|---| +| `tsunagi-awg` | ephemeral pairwise Amnezia masking, private negotiation and independently encrypted UDP carriers; reuses the end-to-end WireGuard engine | | `tsunagi-wg-quic` | the `wg-quic` protocol: its keys, its announcement, its tunnels | | `tsunagi` | the command line agent, built as the `tsng` executable: the only place that owns a runtime, a logger and signals | diff --git a/docs/protocol.md b/docs/protocol.md index 7d37910..c0c007d 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -9,11 +9,15 @@ Two versions exist and are independent: - **Identity scheme**, `tsunagi-network-id-v1`. Frozen. Changing it creates a different network space for the same name and secret. -- **Control protocol**, ALPN `tsunagi/ctrl/5`, `PROTOCOL_VERSION = 5`. +- **Control protocol**, ALPN `tsunagi/ctrl/6`, `PROTOCOL_VERSION = 6`. Upgrading the crate or bumping the control protocol must never change an existing `NetworkId`. +Control version 6 adds private transport negotiation for pairwise AWG and data +delivery checks. Data ALPN remains `tsunagi/data/5`. All members must upgrade +together; network ids and stored state are unchanged. See [amnezia.md](amnezia.md). + ## Canonical encoding Everything that is hashed or MAC'd uses length-prefixed fields, written here as @@ -230,6 +234,7 @@ not affect other networks. | message | meaning | |---|---| | `Announce { hostname, capabilities, broadcast }` | this agent's hostname, IP-plugin capabilities and local broadcast participation | +| `Transport { protocol, payload }` | private, bounded per-peer transport negotiation; never state or a public message event | | `Ping { seq, payload }` | small request used to verify the exchange | | `Pong { seq, payload }` | the echoed reply | | `State { records }` | a snapshot of signed records, merged into what the receiver holds | diff --git a/docs/routing.md b/docs/routing.md index d9368c7..d4b2d0a 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -96,6 +96,14 @@ topology change can move a flow; datagrams remain unreliable and unordered, and there is no promise to preserve order across a physical path failure. Logical peer links and end-to-end encryption state survive these changes. +## Carrier health and switching + +Carrier setup failures and persistent encrypted delivery loss advance the +initiator's ordered fallback cycle. Logical end-to-end links survive while +physical links are retired explicitly. The AWG carrier negotiates a fresh +private pair profile on every reentry, without changing topology's durable +state. See [amnezia.md](amnezia.md) for thresholds and bounds. + ## Verification and measurement The default offline suite exercises four real agents, iroh transports and @@ -116,5 +124,5 @@ counts sends without storing frames. Results are CPU forwarding cost, **not** end-to-end network latency or VPN throughput; encryption, fragmentation, sockets, congestion and scheduling contribute separately. -Wire compatibility: control ALPN `tsunagi/ctrl/5`, data ALPN `tsunagi/data/5`. +Wire compatibility: control ALPN `tsunagi/ctrl/6`, data ALPN `tsunagi/data/5`. Upgrade every participant together; saved identities and network state persist. diff --git a/docs/testing.md b/docs/testing.md index 8f65284..7e11cba 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -177,3 +177,10 @@ TSUNAGI_TEST_LOG=tsunagi=debug cargo test --test multi_peer -- --nocapture The library never installs a global subscriber; the harness opts in only when that variable is set. + +The `tsunagi-awg` suite runs real pair negotiation and masked UDP WireGuard +engines with memory TUNs. A loss wrapper drops datagrams at the carrier boundary +without substituting iroh, membership, encryption or SQLite. It checks runtime +loss detection, fallback and fresh profiles, in addition to restart and input +validation. Tests are offline; public STUN/DPI/NAT effectiveness needs separate +real-network validation. diff --git a/docs/threat-model.md b/docs/threat-model.md index f30c32f..d05c8a7 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -26,8 +26,8 @@ Read this before relying on anything here. The protocol is in neither receive nor forge another member's traffic. See [wireguard.md](wireguard.md#address-ownership-is-enforced-not-announced). - **Tunnelled traffic is end-to-end encrypted by WireGuard**, independently of - this crate. The transport underneath is also encrypted by iroh, but the - tunnel's confidentiality does not depend on that. + this crate. Carriers may be iroh, TLS, direct WG or + pairwise AWG; confidentiality does not depend on a particular carrier. - **Resource bounds.** Frame lengths are validated before allocation; strings, lists, queues, concurrent dials and in-flight handshakes are all bounded; handshakes, dials and writes have timeouts. @@ -113,3 +113,14 @@ for key separation, HMAC-SHA256 for the membership proof, constant-time verification, iroh's Ed25519 endpoint keys and QUIC/TLS for the transport, and RFC 5705 TLS exporter output for channel binding. There is no custom encryption layer and no custom PAKE. + +## Pairwise AWG masking + +AWG profiles, PSKs and outer keys are private to one authenticated pair attempt, +not shared network state. Address candidates are unverified until authenticated +outer WireGuard data arrives. Header protection and structured signature packets +reduce fixed wire markers; they do not guarantee admission by DPI or hide traffic +analysis. iroh control availability remains a prerequisite. Encrypted end-to-end +delivery probes detect sustained carrier loss, never confer membership and pass +the same signed source/destination admission as ordinary IP traffic. See +[amnezia.md](amnezia.md) for limits and tested scope.