diff --git a/Cargo.lock b/Cargo.lock index cda99ad..f12f337 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6239,6 +6239,7 @@ dependencies = [ "iroh", "netwatch", "nix 0.31.3", + "serde_json", "simple-dns", "tempfile", "tokio", diff --git a/README.md b/README.md index dd0aac0..6ad0336 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,19 @@ stopping a suspended network cancels its automatic resume. Machines are called `tsunagi-` until you rename them: `tsng id hostname laptop`. +### Virtual services + +```sh +tsng vip create home api +tsng vip port add home api tcp/443 --backend server1:6443 --backend server2:7443 +tsng vip status home api --watch +``` + +Use `api.home:443`. Each client selects the available backend with the lowest +measured data-plane RTT. A VIP can have multiple independent TCP and UDP ports. +Every member may edit it; the tray's **VIP** button opens a draft editor with +explicit Save. [Commands, health, metrics and limits](docs/vip.md). + ### Exit nodes ```sh @@ -227,6 +240,7 @@ The [threat model](docs/threat-model.md) says what this does and doesn't protect [Pairwise Amnezia transport](docs/amnezia.md) · [Routing](docs/routing.md) · [Exit nodes](docs/exit-node.md) · +[Virtual services](docs/vip.md) · [Threat model](docs/threat-model.md) · [Contributing](AGENTS.md) diff --git a/crates/tsunagi-core/src/agent/mod.rs b/crates/tsunagi-core/src/agent/mod.rs index 6b83b05..130a26c 100644 --- a/crates/tsunagi-core/src/agent/mod.rs +++ b/crates/tsunagi-core/src/agent/mod.rs @@ -309,6 +309,28 @@ impl TransportContext for TransportCtx { } impl Agent { + /// Atomically saves or removes a version-checked shared VIP draft. + /// Every authenticated network member may edit any service. + pub async fn save_vip( + &self, + network: NetworkId, + config: crate::vip::VipConfig, + expected: Option, + delete: bool, + ) -> Result> { + let (reply, receive) = oneshot::channel(); + self.command( + network, + NetCommand::SaveVip { + config, + expected, + delete, + reply, + }, + ) + .await?; + receive.await.map_err(|_| Error::Stopped)? + } /// Retires one offline member's observed claim without revoking membership. /// Every member has this right. The version must still match the selection. pub async fn forget_node( @@ -916,6 +938,34 @@ impl Agent { .storage .retirements(network_id, self.inner.limits.max_retirements) .await?; + let vip_entries = self + .inner + .storage + .vip_fields( + network_id, + self.inner + .limits + .max_vips + .saturating_mul(self.inner.limits.max_vip_fields) + .saturating_add(self.inner.limits.max_vip_tombstones), + ) + .await?; + let mut vips = crate::vip::VipSet::default(); + for id in vip_entries + .iter() + .map(|e| e.vip.clone()) + .collect::>() + { + vips.merge( + network_id, + vip_entries + .iter() + .filter(|e| e.vip == id) + .cloned() + .collect(), + &self.inner.limits, + )?; + } let range = self.desired_range(network_id).await?; if let Some(range) = range { let conflicts = self.inner.routes.overlapping_networks(network_id, range); @@ -968,6 +1018,7 @@ impl Agent { .and_then(|stored| stored.exit_via) .and_then(|bytes| EndpointId::from_bytes(&bytes).ok()); let handle = network::spawn(RuntimeParams { + vips, retirements, broadcast, exit_offer, @@ -1215,7 +1266,43 @@ impl Agent { let keys = NetworkKeys::derive(&stored.name, &stored.secret); let range = self.desired_range(stored.network_id).await?; let range_conflicts = self.range_conflicts(stored.network_id, range).await?; + let fields = self + .inner + .storage + .vip_fields( + stored.network_id, + self.inner + .limits + .max_vips + .saturating_mul(self.inner.limits.max_vip_fields) + .saturating_add(self.inner.limits.max_vip_tombstones), + ) + .await?; + let mut service_state = crate::vip::VipSet::default(); + for id in fields + .iter() + .map(|e| e.vip.clone()) + .collect::>() + { + service_state.merge( + stored.network_id, + fields.iter().filter(|e| e.vip == id).cloned().collect(), + &self.inner.limits, + )?; + } + let vips = service_state + .configs() + .into_iter() + .map(|config| crate::vip::VipStatus { + revision: service_state.revision(&config.id), + config, + address: None, + error: Some("network is inactive".into()), + listeners: vec![], + }) + .collect(); networks.push(NetworkStatus { + vips, protocols: self.effective_protocols(stored.protocols.as_ref()), auto_start: stored.auto_start, broadcast: stored.broadcast, diff --git a/crates/tsunagi-core/src/agent/network.rs b/crates/tsunagi-core/src/agent/network.rs index 9725d6f..7b7b15a 100644 --- a/crates/tsunagi-core/src/agent/network.rs +++ b/crates/tsunagi-core/src/agent/network.rs @@ -50,6 +50,12 @@ pub(crate) struct InboundSession { /// Commands accepted by a network runtime. pub(crate) enum NetCommand { + SaveVip { + config: crate::vip::VipConfig, + expected: Option, + delete: bool, + reply: oneshot::Sender>>, + }, ForgetNode { peer: EndpointId, expected_version: u64, @@ -125,6 +131,7 @@ pub(crate) enum NetCommand { impl std::fmt::Debug for NetCommand { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + NetCommand::SaveVip { .. } => f.write_str("SaveVip"), NetCommand::ForgetNode { peer, .. } => write!(f, "ForgetNode({peer})"), NetCommand::Inbound(_) => f.write_str("Inbound"), NetCommand::InboundLink(link) => write!(f, "InboundLink({})", link.protocol), @@ -170,6 +177,7 @@ impl NetworkHandle { /// Everything a network runtime needs to run. pub(crate) struct RuntimeParams { + pub(crate) vips: crate::vip::VipSet, pub(crate) retirements: Vec, pub(crate) broadcast: bool, /// This agent offers itself as an exit node here. @@ -376,6 +384,8 @@ struct Runtime { /// Snapshots received while dispatching, handled on the next loop pass. pending_state: Vec<(EndpointId, Vec)>, pending_retirements: Vec<(EndpointId, Vec)>, + pending_vips: Vec>, + vip_cursor: HashMap, /// Next cleanup batch per live session; prevents queue pressure starving a suffix. retirement_cursor: HashMap, /// The highest version this agent has ever published for this network. @@ -461,6 +471,8 @@ impl Runtime { state: StateSet::new(), pending_state: Vec::new(), pending_retirements: Vec::new(), + pending_vips: Vec::new(), + vip_cursor: HashMap::new(), retirement_cursor: HashMap::new(), own_version: 0, exit_via_was_online: false, @@ -561,6 +573,15 @@ impl Runtime { async fn handle_command(&mut self, command: NetCommand) { match command { + NetCommand::SaveVip { + config, + expected, + delete, + reply, + } => { + let result = self.save_vip(config, expected, delete).await; + let _ = reply.send(result); + } NetCommand::ForgetNode { peer, expected_version, @@ -830,6 +851,7 @@ impl Runtime { // the check is a comparison when nothing has changed. self.ensure_own_claim().await; self.report_hostname_taken(); + self.broadcast_vips(); self.broadcast_retirements(); // The same timer repairs the relay picture: reachability is soft // state, repeated while it holds and forgotten when it stops. @@ -1104,7 +1126,8 @@ impl Runtime { .own_record_version(self.network_id, self.local_id) .await .unwrap_or(0) - .max(self.state.authored_version(&self.local_id)); + .max(self.state.authored_version(&self.local_id)) + .max(self.params.vips.authored_version(self.local_id)); for (id, hostname) in self .params .storage @@ -1178,11 +1201,26 @@ impl Runtime { match self.state.address_of(&self.local_id) { Some(mine) if range.contains(mine) => Some(mine), _ => { - let taken: std::collections::HashSet = holders + let mut taken: std::collections::HashSet = holders .iter() .filter(|(_, holder)| **holder != self.local_id) .map(|(address, _)| *address) .collect(); + for vip in self.params.vips.views( + Some(range), + taken.clone(), + &std::collections::BTreeSet::new(), + &self.params.limits, + ) { + if let Some(address) = vip.address { + taken.insert(address); + } + } + for vip in self.params.routes.vips.statuses(self.network_id) { + if let Some(address) = vip.address { + taken.insert(address); + } + } match allocate( self.network_id, self.local_id, @@ -1460,8 +1498,188 @@ impl Runtime { self.broadcast_state(); } + fn update_vips(&self) { + let taken = self.state.address_holders().into_keys().collect(); + let names = self + .state + .active_records() + .into_iter() + .filter_map(|r| { + r.author_id() + .ok() + .and_then(|p| self.state.hostname_of(&p).map(str::to_owned)) + }) + .collect(); + let views = + self.params + .vips + .views(self.effective_range(), taken, &names, &self.params.limits); + let peers = self + .state + .address_holders() + .into_iter() + .map(|(a, p)| (p, a, self.hub.reachable(p))) + .collect(); + self.params.routes.vips.configure( + self.network_id, + self.local_id, + self.state.address_of(&self.local_id), + views, + peers, + &self.params.limits, + ); + } + async fn save_vip( + &mut self, + config: crate::vip::VipConfig, + expected: Option, + delete: bool, + ) -> Result> { + if self.released || self.shutdown.is_triggered() { + return Err(Error::Stopped); + } + let version = self + .own_version + .checked_add(1) + .filter(|n| *n <= i64::MAX as u64) + .ok_or_else(|| crate::vip::invalid("author counter exhausted"))?; + let entries = if delete { + self.params.vips.delete( + self.network_id, + &self.params.device_secret, + version, + &config.id, + expected.as_deref().unwrap_or(""), + )? + } else { + self.params.vips.edit( + self.network_id, + &self.params.device_secret, + version, + &config, + expected.as_deref(), + &self.params.limits, + )? + }; + let mut candidate = self.params.vips.clone(); + candidate.merge(self.network_id, entries, &self.params.limits)?; + if !delete { + let duplicate = candidate + .configs() + .iter() + .any(|v| v.id != config.id && v.name == config.name) + || self.state.active_records().iter().any(|r| { + r.author_id().ok().and_then(|p| self.state.hostname_of(&p)) + == Some(config.name.as_str()) + }); + if duplicate { + return Err(crate::vip::invalid( + "VIP name is already used by a node or another VIP", + )); + } + } + let snapshot = candidate.entries(&config.id); + self.validate_vip_frame(&snapshot)?; + self.params + .storage + .put_vip_fields( + self.network_id, + snapshot.clone(), + Some((*self.local_id.as_bytes(), version)), + ) + .await?; + self.own_version = version; + self.params.vips = candidate; + self.update_vips(); + let peers: Vec<_> = self.sessions.keys().copied().collect(); + for peer in peers { + let _ = self.send_to( + peer, + ControlMessage::Vips { + entries: snapshot.clone(), + }, + ); + } + Ok(self.params.routes.vips.statuses(self.network_id)) + } + async fn receive_vips(&mut self, entries: Vec) -> Result<()> { + let Some(id) = entries.first().map(|e| e.vip.clone()) else { + return Ok(()); + }; + if entries.iter().any(|e| e.vip != id) { + return Err(crate::vip::invalid("VIP snapshot mixes service identities")); + } + let own_seen = entries + .iter() + .filter(|e| e.author == *self.local_id.as_bytes()) + .map(|e| e.version) + .max() + .unwrap_or(0); + let mut candidate = self.params.vips.clone(); + let changed = candidate.merge(self.network_id, entries, &self.params.limits)?; + if !changed && own_seen <= self.own_version { + return Ok(()); + } + let counter = + (own_seen > self.own_version).then_some((*self.local_id.as_bytes(), own_seen)); + let snapshot = candidate.entries(&id); + self.validate_vip_frame(&snapshot)?; + self.params + .storage + .put_vip_fields(self.network_id, snapshot, counter) + .await?; + self.own_version = self.own_version.max(own_seen); + self.params.vips = candidate; + self.update_vips(); + self.broadcast_vips(); + Ok(()) + } + fn validate_vip_frame(&self, entries: &[crate::vip::VipEntry]) -> Result<()> { + let frame = Envelope { + network_id: *self.network_id.as_bytes(), + message: ControlMessage::Vips { + entries: entries.to_vec(), + }, + }; + let bytes = encode(&frame).map_err(|e| crate::vip::invalid(e.to_string()))?; + if bytes.len() > self.params.limits.max_frame_len { + return Err(crate::vip::invalid( + "VIP snapshot exceeds the control frame limit", + )); + } + Ok(()) + } + fn broadcast_vips(&mut self) { + let ids = self.params.vips.ids(); + if ids.is_empty() { + return; + } + let peers: Vec<_> = self.sessions.keys().copied().collect(); + for peer in peers { + let start = self.vip_cursor.get(&peer).copied().unwrap_or(0) % ids.len(); + for step in 0..ids.len() { + let index = (start + step) % ids.len(); + let entries = self.params.vips.entries(&ids[index]); + if self + .send_to(peer, ControlMessage::Vips { entries }) + .is_err() + { + self.vip_cursor.insert(peer, index); + break; + } + self.vip_cursor.insert(peer, (index + 1) % ids.len()); + } + } + } + /// Handles snapshots collected while dispatching messages. async fn drain_pending_state(&mut self) { + for entries in std::mem::take(&mut self.pending_vips) { + if let Err(err) = self.receive_vips(entries).await { + self.metrics.protocol_violations += 1; + tracing::warn!(%err,"rejecting VIP snapshot"); + } + } for (peer, markers) in std::mem::take(&mut self.pending_retirements) { self.receive_retirements(peer, markers).await; } @@ -1473,6 +1691,7 @@ impl Runtime { /// Sends everything we know to every peer. fn broadcast_state(&mut self) { + self.broadcast_vips(); self.broadcast_retirements(); let mut records = self.state.records(); records.truncate(self.params.limits.max_state_records); @@ -1671,6 +1890,7 @@ impl Runtime { self.params.range_changed.send_replace(()); return; } + self.update_vips(); for plugin in &self.params.plugins { plugin.on_address_allocation(self.network_id, range, &allocations); } @@ -2154,6 +2374,7 @@ impl Runtime { }); } } + self.update_vips(); } fn handle_link_result(&mut self, outcome: LinkOutcome) { @@ -2304,6 +2525,7 @@ impl Runtime { /// Drops every link to a peer. fn drop_links_for(&mut self, peer: EndpointId) { self.retirement_cursor.remove(&peer); + self.vip_cursor.remove(&peer); self.reset_peer_data(peer, "peer session ended"); self.hub.remove_peer(peer); self.reachable.remove(&peer); @@ -2664,6 +2886,7 @@ impl Runtime { tracing::debug!(%err, "could not queue pong"); } } + ControlMessage::Vips { entries } => self.pending_vips.push(entries.clone()), ControlMessage::State { records } => { self.pending_state.push((peer, records.clone())); } @@ -2891,6 +3114,7 @@ impl Runtime { members.dedup_by(|a, b| a.endpoint_id == b.endpoint_id); NetworkStatus { + vips: self.params.routes.vips.statuses(self.network_id), protocols: self .params .plugins diff --git a/crates/tsunagi-core/src/agent/status.rs b/crates/tsunagi-core/src/agent/status.rs index 8c52d3e..aa6b73c 100644 --- a/crates/tsunagi-core/src/agent/status.rs +++ b/crates/tsunagi-core/src/agent/status.rs @@ -225,6 +225,8 @@ pub struct RangeConflict { /// Status of one network. #[derive(Debug, Clone)] pub struct NetworkStatus { + /// Shared service configuration with this client's routing observations. + pub vips: Vec, /// Enabled local data-plane protocols, in preference order. pub protocols: Vec, /// Whether this network is requested to start or resume automatically. diff --git a/crates/tsunagi-core/src/config.rs b/crates/tsunagi-core/src/config.rs index 0ed1afe..ee43ae7 100644 --- a/crates/tsunagi-core/src/config.rs +++ b/crates/tsunagi-core/src/config.rs @@ -158,6 +158,22 @@ pub enum TransportPolicy { /// object where that is possible (notably [`Limits::max_frame_len`]). #[derive(Debug, Clone, PartialEq, Eq)] pub struct Limits { + /// Maximum active VIP identities per network. + pub max_vips: usize, + /// Maximum permanent VIP deletion tombstones per network. + pub max_vip_tombstones: usize, + /// Maximum concurrent active TCP port checks across the agent. + pub max_vip_checks: usize, + /// Maximum signed fields per VIP, including removed fields. + pub max_vip_fields: usize, + /// Maximum listeners per VIP. + pub max_vip_listeners: usize, + /// Maximum backend endpoints per listener. + pub max_vip_backends: usize, + /// Maximum simultaneous local VIP flow bindings. + pub max_vip_flows: usize, + /// Maximum incomplete IPv4 datagrams per network in VIP reassembly. + pub max_vip_fragments: usize, /// Largest accepted control frame payload, in bytes. pub max_frame_len: usize, /// Largest accepted hostname, in bytes. @@ -205,6 +221,14 @@ pub struct Limits { impl Default for Limits { fn default() -> Self { Self { + max_vips: 64, + max_vip_tombstones: 1024, + max_vip_checks: 32, + max_vip_fields: 192, + max_vip_listeners: 16, + max_vip_backends: 32, + max_vip_flows: 4096, + max_vip_fragments: 64, max_frame_len: 64 * 1024, max_hostname_len: 255, max_capabilities: 16, diff --git a/crates/tsunagi-core/src/ipc/mod.rs b/crates/tsunagi-core/src/ipc/mod.rs index a846c7f..ddbd058 100644 --- a/crates/tsunagi-core/src/ipc/mod.rs +++ b/crates/tsunagi-core/src/ipc/mod.rs @@ -171,6 +171,17 @@ pub fn default_control_socket_path() -> PathBuf { #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum Request { + /// Save a complete version-checked service draft or its deletion. + SaveVip { + /// Exact network id. + network_id: String, + /// Entire desired configuration. + config: crate::vip::VipConfig, + /// Revision originally displayed; absent for creation. + expected: Option, + /// Remove the service permanently. + delete: bool, + }, /// Retire the selected offline member's observed claim. ForgetNode { /// Exact public network id. @@ -271,6 +282,9 @@ pub enum Request { impl std::fmt::Debug for Request { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Request::SaveVip { + network_id, config, .. + } => write!(f, "SaveVip {{ {network_id}, {} }}", config.id), Request::ForgetNode { network_id, peer, @@ -317,6 +331,8 @@ impl std::fmt::Debug for Request { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum Response { + /// Accepted configuration with fresh local routing observations. + Vips(Vec), /// An observed offline claim was durably retired. NodeForgotten(ForgottenNodeReport), /// Accepted ordered protocol selection. @@ -358,6 +374,57 @@ pub struct ForgottenNodeReport { pub peers_told: u32, } +/// Saves a whole VIP draft through the running agent. +pub async fn save_vip( + socket: &Path, + network_id: &str, + config: crate::vip::VipConfig, + expected: Option, + delete: bool, +) -> Result> { + match exchange( + socket, + &Request::SaveVip { + network_id: network_id.into(), + config, + expected, + delete, + }, + EXCHANGE_TIMEOUT, + ) + .await? + { + Response::Vips(vips) => Ok(vips), + Response::Error(reason) => Err(Error::Rejected { reason }), + _ => Err(Error::Rejected { + reason: "unexpected VIP response".into(), + }), + } +} +/// Resolves a service by exact name, full id or unique id prefix. +pub fn resolve_vip<'a>( + network: &'a NetworkReport, + selector: &str, +) -> std::result::Result<&'a crate::vip::VipStatus, String> { + let matches: Vec<_> = network + .vips + .iter() + .filter(|v| v.config.name == selector || v.config.id.starts_with(selector)) + .collect(); + match matches.as_slice() { + [vip] => Ok(vip), + [] => Err(format!("no VIP matches `{selector}`")), + _ => Err(format!( + "ambiguous VIP `{selector}`: {}", + matches + .iter() + .map(|v| format!("{} ({})", v.config.name, v.config.id)) + .collect::>() + .join(", ") + )), + } +} + /// Resolves a node by exact hostname, full id, unique id prefix or overlay IPv4. /// Ambiguity is reported with names and full ids and never guessed. pub fn resolve_node<'a>( @@ -642,6 +709,9 @@ pub struct ProtocolReport { /// One network. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct NetworkReport { + /// Shared VIPs and this client's routing observations. + #[serde(default)] + pub vips: Vec, /// Enabled local data-plane protocols, in preference order. pub protocols: Vec, /// Whether the network starts or resumes automatically when its range is free. @@ -905,6 +975,16 @@ impl OverlayPeerReport { /// running and what they can report or change. That is what keeps this module /// free of any knowledge of them. pub trait ReportSource: Send + Sync + 'static { + /// Atomically saves a complete VIP draft using its original revision. + fn save_vip( + &self, + _network: String, + _config: crate::vip::VipConfig, + _expected: Option, + _delete: bool, + ) -> BoxFuture<'_, std::result::Result, String>> { + Box::pin(async { Err("this agent cannot manage VIPs".into()) }) + } /// Persists and announces a version-scoped node cleanup. fn forget_node( &self, @@ -1051,7 +1131,7 @@ pub const EXCHANGE_TIMEOUT: Duration = Duration::from_secs(5); /// /// Bump it whenever [`Request`], [`Response`] or anything they contain /// changes shape. -pub const CONTROL_PROTOCOL: u32 = u32::from_be_bytes([b'T', b'S', b'N', 21]); +pub const CONTROL_PROTOCOL: u32 = u32::from_be_bytes([b'T', b'S', b'N', 22]); /// Reads one request off an accepted stream, answers it, writes the response. /// @@ -1075,6 +1155,15 @@ where } }; let response = match request { + Request::SaveVip { + network_id, + config, + expected, + delete, + } => match source.save_vip(network_id, config, expected, delete).await { + Ok(vips) => Response::Vips(vips), + Err(reason) => Response::Error(reason), + }, Request::ForgetNode { network_id, peer, diff --git a/crates/tsunagi-core/src/lib.rs b/crates/tsunagi-core/src/lib.rs index 4444e5a..1036841 100644 --- a/crates/tsunagi-core/src/lib.rs +++ b/crates/tsunagi-core/src/lib.rs @@ -51,6 +51,7 @@ pub mod storage; pub mod task; #[cfg(feature = "testing")] pub mod testing; +pub mod vip; pub use agent::{Agent, AgentStatus, Event, NetworkStatus, PeerStatus}; pub use config::{AgentConfig, Limits, ReconnectPolicy, StoragePaths, TransportPolicy}; diff --git a/crates/tsunagi-core/src/overlay/interface.rs b/crates/tsunagi-core/src/overlay/interface.rs index 5730a61..ebc0149 100644 --- a/crates/tsunagi-core/src/overlay/interface.rs +++ b/crates/tsunagi-core/src/overlay/interface.rs @@ -145,18 +145,19 @@ impl Tally { } /// The overlay interface. -#[derive(Debug)] pub struct Interface { device: Arc, factory: Arc, name: String, mtu: u32, routes: Arc, + carrier: Arc, tally: Arc, /// What was last applied to the host, so an unchanged table is not /// re-applied on every pass. applied: std::sync::Mutex>, task: std::sync::Mutex>>, + vip_task: std::sync::Mutex>>, sync_lock: Arc>, /// The host rules for this interface, when the factory manages the host. host_rules: Option>, @@ -200,6 +201,14 @@ impl ExitState { } } +impl std::fmt::Debug for Interface { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Interface") + .field("name", &self.name) + .finish_non_exhaustive() + } +} + impl Interface { /// Creates the interface and starts moving packets. pub async fn start( @@ -221,7 +230,29 @@ impl Interface { let name = device.name().to_string(); let tally = Arc::new(Tally::default()); + let vip_task = { + let routes = Arc::clone(&routes); + let carrier = Arc::clone(&carrier); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_millis(250)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut checks = tokio::task::JoinSet::new(); + loop { + tokio::select! { + _=interval.tick()=>{ + let (probes,jobs)=routes.vips.tick(routes.vips.check_capacity().saturating_sub(checks.len())); + for (route,packet) in probes {carrier.carry(route,packet);} + for job in jobs {checks.spawn(job.run());} + }, + Some(result)=checks.join_next(),if !checks.is_empty()=>{ + if let Ok((job,ok))=result {routes.vips.checked(job,ok);} + } + } + } + }) + }; let task = { + let carrier = Arc::clone(&carrier); let device = Arc::clone(&device); let routes = Arc::clone(&routes); let tally = Arc::clone(&tally); @@ -231,6 +262,26 @@ impl Interface { tally.malformed.fetch_add(1, Ordering::Relaxed); continue; }; + match routes.vips.outgoing(packet.clone()) { + crate::vip::router::Decision::Send(route, packet) => { + for part in crate::vip::packet::fragments(packet, mtu) { + if carrier.carry(route, part) { + tally.sent.fetch_add(1, Ordering::Relaxed); + } else { + tally.undeliverable.fetch_add(1, Ordering::Relaxed); + } + } + continue; + } + crate::vip::router::Decision::Local(packet) => { + for part in crate::vip::packet::fragments(packet, mtu) { + let _ = device.send(part).await; + } + continue; + } + crate::vip::router::Decision::Drop => continue, + crate::vip::router::Decision::Pass => {} + } let destination = header.destination(); if let (IpAddr::V4(source), IpAddr::V4(destination)) = (header.source(), destination) @@ -279,9 +330,11 @@ impl Interface { name, mtu, routes, + carrier, tally, applied: std::sync::Mutex::new(Vec::new()), task: std::sync::Mutex::new(Some(task)), + vip_task: std::sync::Mutex::new(Some(vip_task)), sync_lock: Arc::new(tokio::sync::Mutex::new(())), host_rules, rules_report: std::sync::Mutex::new(None), @@ -716,6 +769,15 @@ impl Interface { task.abort(); let _ = task.await; } + let vip_task = self + .vip_task + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + if let Some(task) = vip_task { + task.abort(); + let _ = task.await; + } self.factory.destroy(self.device.name()).await; } @@ -770,6 +832,24 @@ impl Interface { return Err(Rejected::WrongDestination); } }; + // Source ownership was checked before any VIP reverse translation. + let packet = match self.routes.vips.incoming(network, peer, packet.clone()) { + crate::vip::router::Decision::Send(route, packet) => { + // Probe replies use the ordinary peer carrier, never the TUN. + self.carrier.carry(route, packet); + return Ok(()); + } + crate::vip::router::Decision::Local(packet) => { + for part in crate::vip::packet::fragments(packet, self.mtu) { + if self.device.send(part).await.is_ok() { + self.tally.received.fetch_add(1, Ordering::Relaxed); + } + } + return Ok(()); + } + crate::vip::router::Decision::Drop => return Ok(()), + crate::vip::router::Decision::Pass => packet, + }; // Remote broadcasts terminate here. Only local TUN ingress can fan out. if self.device.send(packet).await.is_ok() { if broadcast { @@ -794,6 +874,14 @@ impl Interface { impl Drop for Interface { fn drop(&mut self) { + if let Some(task) = self + .vip_task + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take() + { + task.abort(); + } if let Ok(mut guard) = self.task.lock() && let Some(task) = guard.take() { diff --git a/crates/tsunagi-core/src/overlay/router.rs b/crates/tsunagi-core/src/overlay/router.rs index 93d110a..9dc098b 100644 --- a/crates/tsunagi-core/src/overlay/router.rs +++ b/crates/tsunagi-core/src/overlay/router.rs @@ -89,6 +89,7 @@ pub enum RouteError { /// Address ownership across every network this agent is in. #[derive(Debug, Default)] pub struct RoutingTable { + pub(crate) vips: Arc, broadcast: ArcSwap, networks: RwLock>, } @@ -146,6 +147,7 @@ impl RoutingTable { /// Forgets a network. pub fn remove_network(&self, network: NetworkId) { + self.vips.remove_network(network); let mut networks = self.write(); networks.remove(&network); self.broadcast diff --git a/crates/tsunagi-core/src/proto/message.rs b/crates/tsunagi-core/src/proto/message.rs index bc590fa..f9b0a2a 100644 --- a/crates/tsunagi-core/src/proto/message.rs +++ b/crates/tsunagi-core/src/proto/message.rs @@ -20,7 +20,7 @@ use crate::error::ProtocolError; /// The version in the ALPN is the wire-compatibility version of the control /// protocol. It is independent of the network identity scheme version, so /// bumping it must not change any existing [`crate::NetworkId`]. -pub const ALPN: &[u8] = b"tsunagi/ctrl/7"; +pub const ALPN: &[u8] = b"tsunagi/ctrl/8"; /// ALPN of the tsunagi data plane. /// @@ -43,7 +43,7 @@ pub const MAX_DATA_PROTOCOL_LEN: usize = 32; pub const MAX_SIGNATURE_LEN: usize = 64; /// Control protocol version carried inside the handshake. -pub const PROTOCOL_VERSION: u16 = 7; +pub const PROTOCOL_VERSION: u16 = 8; pub use crate::dataplane::transport::{MAX_TRANSPORT_PAYLOAD, TransportPayload}; @@ -181,6 +181,11 @@ pub enum ControlMessage { /// Short free-text reason. reason: String, }, + /// One complete signed VIP field snapshot, merged atomically. + Vips { + /// Signed registers bounded by the per-VIP field limit. + entries: Vec, + }, /// Signed cleanup markers, each proving the retired claim version. Retirements { /// Bounded by the per-message signed-state limit. @@ -278,6 +283,15 @@ pub fn validate_capability( /// network, the other networks or the agent. pub fn validate(message: &ControlMessage, limits: &Limits) -> Result<(), ProtocolError> { match message { + ControlMessage::Vips { entries } => { + check_len("vip.fields", entries.len(), limits.max_vip_fields)?; + for e in entries { + check_len("vip.id", e.vip.len(), 32)?; + check_len("vip.key", e.key.len(), 160)?; + check_len("vip.value", e.value.as_ref().map_or(0, Vec::len), 256)?; + check_len("vip.signature", e.signature.len(), 64)?; + } + } ControlMessage::Retirements { markers } => { check_len("retirements", markers.len(), limits.max_state_records)?; for marker in markers { @@ -373,6 +387,7 @@ pub fn kind(message: &ControlMessage) -> &'static str { ControlMessage::Pong { .. } => "pong", ControlMessage::State { .. } => "state", ControlMessage::Retirements { .. } => "retirements", + ControlMessage::Vips { .. } => "vips", ControlMessage::Reachable { .. } => "reachable", ControlMessage::Peers { .. } => "peers", ControlMessage::Bye { .. } => "bye", diff --git a/crates/tsunagi-core/src/state/mod.rs b/crates/tsunagi-core/src/state/mod.rs index 20404d8..258dc08 100644 --- a/crates/tsunagi-core/src/state/mod.rs +++ b/crates/tsunagi-core/src/state/mod.rs @@ -38,6 +38,8 @@ //! `docs/sync-model.md` for the implemented cleanup semantics. pub mod allocator; +/// Signed shared virtual service state and its merge rules. +pub mod vip; use std::collections::HashMap; use std::net::Ipv4Addr; diff --git a/crates/tsunagi-core/src/state/vip.rs b/crates/tsunagi-core/src/state/vip.rs new file mode 100644 index 0000000..79985a1 --- /dev/null +++ b/crates/tsunagi-core/src/state/vip.rs @@ -0,0 +1,439 @@ +//! Signed multi-writer virtual service registers and deterministic merge rules. +use crate::error::Result; +use crate::vip::{Backend, Listener, Protocol, VipConfig, VipStatus, invalid}; +use crate::{Limits, NetworkId}; +use iroh::{EndpointId, SecretKey, Signature}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::net::Ipv4Addr; + +/// A signed, independently mergeable field. A removed field keeps its tombstone. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VipEntry { + /// Network scope. + pub network: [u8; 32], + /// Stable service id. + pub vip: String, + /// Canonical field key. + pub key: String, + /// Encoded field value; absent means deletion. + pub value: Option>, + /// Lamport clock, independent of wall time. + pub clock: u64, + /// Persistent signing identity. + pub author: [u8; 32], + /// Author's durable network counter. + pub version: u64, + /// Ed25519 signature. + pub signature: Vec, +} +impl VipEntry { + fn bytes(&self) -> Vec { + let mut e = self.clone(); + e.signature.clear(); + let mut out = b"tsunagi-vip-field/v1\0".to_vec(); + if let Ok(bytes) = postcard::to_stdvec(&e) { + out.extend(bytes); + } + out + } + /// Verifies bounds, network and authorship before the entry is merged. + pub fn verify(&self, network: NetworkId) -> Result<()> { + if self.network != *network.as_bytes() + || self.vip.len() != 32 + || hex::decode(&self.vip).is_err() + || self.key.len() > 160 + || self.value.as_ref().is_some_and(|v| v.len() > 256) + || self.version == 0 + || self.version > i64::MAX as u64 + || self.clock == 0 + || self.clock > i64::MAX as u64 + { + return Err(invalid("invalid VIP field bounds or network")); + } + let author = + EndpointId::from_bytes(&self.author).map_err(|_| invalid("invalid VIP author"))?; + let raw: [u8; 64] = self + .signature + .as_slice() + .try_into() + .map_err(|_| invalid("invalid VIP signature length"))?; + author + .verify(&self.bytes(), &Signature::from_bytes(&raw)) + .map_err(|_| invalid("invalid VIP signature"))?; + if self.key != "deleted" + && self.key != "name" + && !self.key.starts_with("l/") + && !self.key.starts_with("b/") + { + return Err(invalid("unknown VIP field")); + } + match self.key.as_str() { + "deleted" => { + if self.value.as_deref() != Some(&[1][..]) { + return Err(invalid("invalid VIP deletion tombstone")); + } + } + "name" => { + let value = self + .value + .as_ref() + .ok_or_else(|| invalid("VIP name cannot be absent"))?; + let name: String = + postcard::from_bytes(value).map_err(|_| invalid("invalid VIP name field"))?; + VipConfig { + id: self.vip.clone(), + name, + listeners: vec![], + } + .validate(&Limits::default())?; + } + key if key.starts_with("l/") => { + let parts: Vec<_> = key.split('/').collect(); + if parts.len() != 3 + || parts[1].parse::().is_err() + || parts[2].parse::().ok().is_none_or(|p| p == 0) + { + return Err(invalid("invalid listener field key")); + } + if let Some(value) = &self.value { + let l: Listener = postcard::from_bytes(value) + .map_err(|_| invalid("invalid listener field"))?; + if !l.backends.is_empty() || key != format!("l/{}/{}", l.protocol, l.port) { + return Err(invalid("listener key does not match its value")); + } + VipConfig { + id: self.vip.clone(), + name: "validation".into(), + listeners: vec![l], + } + .validate(&Limits::default())?; + } + } + key if key.starts_with("b/") => { + let parts: Vec<_> = key.split('/').collect(); + if parts.len() != 6 + || parts[1].parse::().is_err() + || parts[2].parse::().ok().is_none_or(|p| p == 0) + || parts[3].len() != 32 + || hex::decode(parts[3]).is_err() + || parts[4].parse::().is_err() + || parts[5].parse::().ok().is_none_or(|p| p == 0) + { + return Err(invalid("invalid backend field key")); + } + if let Some(value) = &self.value { + let b: Backend = postcard::from_bytes(value) + .map_err(|_| invalid("invalid backend field"))?; + if b.peer != parts[4] || b.port.to_string() != parts[5] { + return Err(invalid("backend key does not match its value")); + } + } + } + _ => return Err(invalid("unknown VIP field")), + } + Ok(()) + } + fn rank(&self) -> (u64, [u8; 32], Vec) { + (self.clock, self.author, self.bytes()) + } +} +/// Signed registers retained across absences, including deletion tombstones. +#[derive(Debug, Clone, Default)] +pub struct VipSet { + entries: BTreeMap<(String, String), VipEntry>, +} +impl VipSet { + /// Highest retained version genuinely signed by this author. + pub fn authored_version(&self, author: EndpointId) -> u64 { + self.entries + .values() + .filter(|e| e.author == *author.as_bytes()) + .map(|e| e.version) + .max() + .unwrap_or(0) + } + /// Complete independently verifiable state of one service, including tombstones. + pub fn entries(&self, id: &str) -> Vec { + self.entries + .values() + .filter(|e| e.vip == id) + .cloned() + .collect() + } + /// All retained service ids, including deleted services. + pub fn ids(&self) -> Vec { + self.entries + .keys() + .map(|(id, _)| id.clone()) + .collect::>() + .into_iter() + .collect() + } + /// Opaque optimistic concurrency token for a whole service. + pub fn revision(&self, id: &str) -> String { + let mut hash = Sha256::new(); + for e in self.entries(id) { + hash.update(e.bytes()); + hash.update(&e.signature); + } + hex::encode(hash.finalize()) + } + /// Atomically verifies and merges one complete service snapshot. + pub fn merge( + &mut self, + network: NetworkId, + entries: Vec, + limits: &Limits, + ) -> Result { + if entries.len() > limits.max_vip_fields { + return Err(invalid("too many VIP fields")); + } + let mut next = self.clone(); + for e in entries { + e.verify(network)?; + let key = (e.vip.clone(), e.key.clone()); + if next + .entries + .get(&key) + .is_none_or(|old| e.rank() > old.rank()) + { + next.entries.insert(key, e); + } + } + let deleted: BTreeSet<_> = next + .entries + .values() + .filter(|e| e.key == "deleted" && e.value.is_some()) + .map(|e| e.vip.clone()) + .collect(); + // A permanent service tombstone alone prevents every former field's replay. + next.entries + .retain(|(id, key), _| !deleted.contains(id) || key == "deleted"); + let ids = next.ids(); + if ids.len().saturating_sub(deleted.len()) > limits.max_vips + || deleted.len() > limits.max_vip_tombstones + || ids + .iter() + .any(|id| next.entries(id).len() > limits.max_vip_fields) + { + return Err(invalid( + "VIP state capacity exceeded; deletion protection is retained", + )); + } + let changed = self.entries != next.entries; + *self = next; + Ok(changed) + } + /// Materialized configuration. A service tombstone permanently retires its id. + pub fn configs(&self) -> Vec { + self.ids() + .into_iter() + .filter_map(|id| self.config(&id)) + .collect() + } + fn config(&self, id: &str) -> Option { + let fields = self.entries(id); + if fields + .iter() + .any(|e| e.key == "deleted" && e.value.is_some()) + { + return None; + } + let name = fields + .iter() + .find(|e| e.key == "name")? + .value + .as_ref() + .and_then(|v| postcard::from_bytes::(v).ok())?; + let mut listeners: Vec = fields + .iter() + .filter(|e| e.key.starts_with("l/")) + .filter_map(|e| e.value.as_ref().and_then(|v| postcard::from_bytes(v).ok())) + .collect(); + for l in &mut listeners { + l.backends = fields + .iter() + .filter(|e| { + e.key + .starts_with(&format!("b/{}/{}/{}/", l.protocol, l.port, l.id)) + }) + .filter_map(|e| e.value.as_ref().and_then(|v| postcard::from_bytes(v).ok())) + .collect(); + l.backends.sort(); + } + listeners.sort_by_key(|l| (l.protocol, l.port)); + Some(VipConfig { + id: id.into(), + name, + listeners, + }) + } + /// Builds a signed patch from the user's complete, version-checked draft. + pub fn edit( + &self, + network: NetworkId, + secret: &SecretKey, + version: u64, + config: &VipConfig, + expected: Option<&str>, + limits: &Limits, + ) -> Result> { + config.validate(limits)?; + if let Some(revision) = expected { + if revision != self.revision(&config.id) { + return Err(invalid("VIP changed while editing; reload before saving")); + } + } else if !self.entries(&config.id).is_empty() { + return Err(invalid("VIP id already exists")); + } + if self + .entries + .get(&(config.id.clone(), "deleted".into())) + .is_some_and(|e| e.value.is_some()) + { + return Err(invalid("VIP was deleted; create a new VIP")); + } + let before = self + .config(&config.id) + .map(|c| flatten(&c)) + .transpose()? + .unwrap_or_default(); + let after = flatten(config)?; + let keys: BTreeSet<_> = before.keys().chain(after.keys()).cloned().collect(); + let changes = keys + .into_iter() + .filter_map(|key| { + let value = after.get(&key).cloned(); + (before.get(&key) != value.as_ref()).then_some((key, value)) + }) + .collect(); + self.sign(network, secret, version, &config.id, changes) + } + /// Builds a permanent service tombstone after checking the selected revision. + pub fn delete( + &self, + network: NetworkId, + secret: &SecretKey, + version: u64, + id: &str, + expected: &str, + ) -> Result> { + if self.config(id).is_none() || self.revision(id) != expected { + return Err(invalid("VIP missing or changed while editing")); + } + self.sign( + network, + secret, + version, + id, + vec![("deleted".into(), Some(vec![1]))], + ) + } + fn sign( + &self, + network: NetworkId, + secret: &SecretKey, + version: u64, + id: &str, + changes: Vec<(String, Option>)>, + ) -> Result> { + let clock = self + .entries + .values() + .map(|e| e.clock) + .max() + .unwrap_or(0) + .checked_add(1) + .filter(|n| *n <= i64::MAX as u64) + .ok_or_else(|| invalid("VIP clock exhausted"))?; + Ok(changes + .into_iter() + .map(|(key, value)| { + let mut e = VipEntry { + network: *network.as_bytes(), + vip: id.into(), + key, + value, + clock, + author: *secret.public().as_bytes(), + version, + signature: vec![], + }; + e.signature = secret.sign(&e.bytes()).to_bytes().to_vec(); + e + }) + .collect()) + } +} +pub(crate) fn flatten(cfg: &VipConfig) -> Result>> { + let mut fields = BTreeMap::new(); + fields.insert( + "name".into(), + postcard::to_stdvec(&cfg.name).map_err(|e| invalid(e.to_string()))?, + ); + for l in &cfg.listeners { + let mut settings = l.clone(); + settings.backends.clear(); + fields.insert( + format!("l/{}/{}", l.protocol, l.port), + postcard::to_stdvec(&settings).map_err(|e| invalid(e.to_string()))?, + ); + for b in &l.backends { + fields.insert( + format!("b/{}/{}/{}/{}/{}", l.protocol, l.port, l.id, b.peer, b.port), + postcard::to_stdvec(b).map_err(|e| invalid(e.to_string()))?, + ); + } + } + Ok(fields) +} +impl VipSet { + /// Allocates internal addresses deterministically against durable node claims. + pub(crate) fn views( + &self, + range: Option, + mut taken: std::collections::HashSet, + hostnames: &BTreeSet, + limits: &Limits, + ) -> Vec { + let mut names = hostnames.clone(); + let mut result = Vec::new(); + for config in self.configs() { + let mut error = config.validate(limits).err().map(|e| e.to_string()); + if !names.insert(config.name.clone()) { + error = Some("name conflicts with another VIP or node".into()); + } + let address = range.and_then(|r| { + let usable = crate::state::allocator::usable_addresses(r); + if usable == 0 { + return None; + } + let digest = Sha256::digest(config.id.as_bytes()); + let seed = u64::from_be_bytes([ + digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], + digest[7], + ]); + let bits = 32 - u32::from(r.prefix_len); + let mask = if bits == 32 { 0 } else { u32::MAX << bits }; + (0..usable).find_map(|step| { + let offset = (seed.wrapping_add(step) % usable + 1) as u32; + let ip = Ipv4Addr::from((u32::from(r.base) & mask) | offset); + taken.insert(ip).then_some(ip) + }) + }); + if address.is_none() { + error = Some("no free overlay address".into()); + } + result.push(VipStatus { + revision: self.revision(&config.id), + config, + address, + error, + listeners: vec![], + }); + } + result + } +} diff --git a/crates/tsunagi-core/src/storage/mod.rs b/crates/tsunagi-core/src/storage/mod.rs index 1274c58..ce888dd 100644 --- a/crates/tsunagi-core/src/storage/mod.rs +++ b/crates/tsunagi-core/src/storage/mod.rs @@ -370,6 +370,25 @@ impl Storage { .await } + /// Loads mandatory signed VIP fields with a strict capacity bound. + pub async fn vip_fields( + &self, + network: NetworkId, + max: usize, + ) -> Result> { + self.with_state(move |state| state.vip_fields(network, max)) + .await + } + /// Stores a complete merged service snapshot and optional own counter. + pub async fn put_vip_fields( + &self, + network: NetworkId, + entries: Vec, + own: Option<([u8; 32], u64)>, + ) -> Result<()> { + self.with_state(move |state| state.put_vip_fields(network, &entries, own)) + .await + } /// Loads verified durable cleanup proofs off the executor. pub async fn retirements( &self, diff --git a/crates/tsunagi-core/src/storage/state.rs b/crates/tsunagi-core/src/storage/state.rs index 3765fa0..53cbdd1 100644 --- a/crates/tsunagi-core/src/storage/state.rs +++ b/crates/tsunagi-core/src/storage/state.rs @@ -21,7 +21,7 @@ use crate::identity::{DeviceIdentity, NetworkId, NetworkName, NetworkSecret}; use crate::state::{MAX_RETIREMENT_LEN, RecordBody, SignedRecord, SignedRetirement}; /// Schema version written by this build. -pub const SCHEMA_VERSION: i64 = 9; +pub const SCHEMA_VERSION: i64 = 10; /// Key of the stored hostname setting. const SETTING_HOSTNAME: &str = "hostname"; @@ -263,6 +263,12 @@ impl StateStore { ) .map_err(|err| self.corrupt(format!("cannot migrate schema to 9: {err}")))?; } + if found < 10 { + self.conn.execute_batch("BEGIN; CREATE TABLE IF NOT EXISTS vip_fields ( + network_id BLOB NOT NULL, vip TEXT NOT NULL, field TEXT NOT NULL, data BLOB NOT NULL, + PRIMARY KEY(network_id, vip, field)); PRAGMA user_version = 10; COMMIT;") + .map_err(|err| self.corrupt(format!("cannot migrate VIP state: {err}")))?; + } Ok(()) } @@ -276,6 +282,7 @@ impl StateStore { "signed_records", "peer_hostnames", "retirements", + "vip_fields", ] { let present: Option = self .conn @@ -894,8 +901,77 @@ impl StateStore { .map_err(|err| Error::Storage(format!("cannot commit retirement: {err}"))) } + /// Reads bounded mandatory VIP state; corrupt records fail activation. + pub fn vip_fields(&self, network: NetworkId, max: usize) -> Result> { + let mut stmt = self.conn.prepare("SELECT CASE WHEN length(data) <= 1024 THEN data ELSE NULL END FROM vip_fields WHERE network_id=?1 ORDER BY vip, field") + .map_err(|e| self.corrupt(e.to_string()))?; + let mut rows = stmt + .query(params![network.as_bytes().as_slice()]) + .map_err(|e| self.corrupt(e.to_string()))?; + let mut entries = Vec::new(); + while let Some(row) = rows.next().map_err(|e| self.corrupt(e.to_string()))? { + if entries.len() >= max { + return Err(self.corrupt("stored VIP state exceeds configured limits")); + } + let bytes: Option> = row.get(0).map_err(|e| self.corrupt(e.to_string()))?; + let bytes = bytes.ok_or_else(|| self.corrupt("oversized VIP field"))?; + let entry: crate::vip::VipEntry = + postcard::from_bytes(&bytes).map_err(|e| self.corrupt(e.to_string()))?; + entry + .verify(network) + .map_err(|e| self.corrupt(e.to_string()))?; + entries.push(entry); + } + Ok(entries) + } + + /// Commits a validated VIP snapshot and optional author counter atomically. + pub fn put_vip_fields( + &self, + network: NetworkId, + entries: &[crate::vip::VipEntry], + own: Option<([u8; 32], u64)>, + ) -> Result<()> { + let tx = self + .conn + .unchecked_transaction() + .map_err(|e| Error::Storage(e.to_string()))?; + if let Some(first) = entries.first() { + if entries.iter().any(|e| e.vip != first.vip) { + return Err(Error::Storage( + "VIP snapshot mixes service identities".into(), + )); + } + tx.execute( + "DELETE FROM vip_fields WHERE network_id=?1 AND vip=?2", + params![network.as_bytes().as_slice(), first.vip], + ) + .map_err(|e| Error::Storage(e.to_string()))?; + } + for e in entries { + e.verify(network)?; + let bytes = postcard::to_stdvec(e).map_err(|e| Error::Storage(e.to_string()))?; + if bytes.len() > 1024 { + return Err(Error::Storage("oversized VIP field".into())); + } + tx.execute("INSERT INTO vip_fields(network_id,vip,field,data) VALUES(?1,?2,?3,?4) ON CONFLICT(network_id,vip,field) DO UPDATE SET data=excluded.data", + params![network.as_bytes().as_slice(),e.vip,e.key,bytes]).map_err(|e|Error::Storage(e.to_string()))?; + } + if let Some((author, version)) = own { + tx.execute("INSERT INTO own_record_version(network_id,author,version) VALUES(?1,?2,?3) ON CONFLICT(network_id,author) DO UPDATE SET version=max(version,excluded.version)", + params![network.as_bytes().as_slice(),author.as_slice(),version as i64]).map_err(|e|Error::Storage(e.to_string()))?; + } + tx.commit().map_err(|e| Error::Storage(e.to_string())) + } + /// Drops a forgotten network's local cleanup proofs. pub fn forget_retirements(&self, network: NetworkId) -> Result<()> { + self.conn + .execute( + "DELETE FROM vip_fields WHERE network_id = ?1", + params![network.as_bytes().as_slice()], + ) + .map_err(|e| Error::Storage(e.to_string()))?; self.conn .execute( "DELETE FROM retirements WHERE network_id = ?1", @@ -1333,3 +1409,59 @@ mod broadcast_storage_tests { ); } } + +#[cfg(test)] +mod vip_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[test] + fn vip_migration_and_counter_failure_preserve_identity_and_rollback_fields() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("state.sqlite"); + let store = StateStore::open(&path).unwrap(); + let identity = store.load_or_create_device_identity().unwrap(); + store + .conn + .execute_batch("DROP TABLE vip_fields; PRAGMA user_version=9;") + .unwrap(); + drop(store); + let store = StateStore::open(&path).unwrap(); + assert_eq!( + store.device_identity().unwrap().unwrap().endpoint_id(), + identity.endpoint_id() + ); + let net = NetworkId::from_bytes([7; 32]); + let key = identity.signing_key(); + let config = crate::vip::VipConfig::new("api"); + let fields = crate::vip::VipSet::default() + .edit(net, &key, 7, &config, None, &crate::Limits::default()) + .unwrap(); + store.conn.execute_batch("CREATE TRIGGER fail_vip_counter BEFORE INSERT ON own_record_version BEGIN SELECT RAISE(ABORT,'counter unavailable'); END;").unwrap(); + assert!( + store + .put_vip_fields(net, &fields, Some((*key.public().as_bytes(), 7))) + .is_err() + ); + assert!(store.vip_fields(net, 192).unwrap().is_empty()); + store + .conn + .execute_batch("DROP TRIGGER fail_vip_counter;") + .unwrap(); + store + .put_vip_fields(net, &fields, Some((*key.public().as_bytes(), 7))) + .unwrap(); + assert_eq!(store.vip_fields(net, 192).unwrap(), fields); + assert_eq!(store.own_record_version(net, key.public()).unwrap(), 7); + drop(store); + let reopened = StateStore::open(path).unwrap(); + assert_eq!(reopened.vip_fields(net, 192).unwrap(), fields); + reopened + .conn + .execute("UPDATE vip_fields SET data=?1", params![vec![0xffu8; 1025]]) + .unwrap(); + assert!(matches!( + reopened.vip_fields(net, 192), + Err(Error::StateCorrupted { .. }) + )); + } +} diff --git a/crates/tsunagi-core/src/vip/mod.rs b/crates/tsunagi-core/src/vip/mod.rs new file mode 100644 index 0000000..90c2710 --- /dev/null +++ b/crates/tsunagi-core/src/vip/mod.rs @@ -0,0 +1,239 @@ +//! Named, shared virtual services. Signed field registers converge without an +//! owner; routing, health and connection state remain local to each client. +use crate::Limits; +use crate::error::{Error, Result}; +use iroh::EndpointId; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::net::Ipv4Addr; +pub(crate) mod packet; +pub(crate) mod router; +pub use crate::state::vip::{VipEntry, VipSet}; +pub(crate) use router::VipRouter; +#[cfg(test)] +mod tests; + +/// IP transport carried by a listener. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Protocol { + /// TCP. + Tcp, + /// UDP. + Udp, +} +impl std::fmt::Display for Protocol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Tcp => "tcp", + Self::Udp => "udp", + }) + } +} +impl std::str::FromStr for Protocol { + type Err = String; + fn from_str(s: &str) -> std::result::Result { + match s { + "tcp" => Ok(Self::Tcp), + "udp" => Ok(Self::Udp), + _ => Err("protocol must be tcp or udp".into()), + } + } +} +/// A node identity and its application port. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Backend { + /// Full device identity, resolved when editing. + pub peer: String, + /// Destination port. + pub port: u16, +} +/// One independently routed port of a VIP. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Listener { + /// Stable listener generation; prevents removed backend fields resurfacing on re-creation. + pub id: String, + /// TCP or UDP. + pub protocol: Protocol, + /// Client-facing port. + pub port: u16, + /// Enable active TCP connect checks; UDP uses path checks and ICMP errors. + pub port_check: bool, + /// Period between checks in milliseconds. + pub check_interval_ms: u32, + /// Deadline of a port check in milliseconds. + pub check_timeout_ms: u32, + /// Consecutive failed checks before exclusion. + pub failure_threshold: u8, + /// Consecutive successes before recovery. + pub recovery_threshold: u8, + /// Idle expiry for this listener's flows, in seconds. + pub idle_timeout_secs: u32, + /// Allowed destination nodes and ports. + pub backends: Vec, +} +impl Listener { + /// Creates a listener with automatic recovery and bounded checks. + pub fn new(protocol: Protocol, port: u16) -> Self { + Self { + id: hex::encode(rand::random::<[u8; 16]>()), + protocol, + port, + port_check: true, + check_interval_ms: 3000, + check_timeout_ms: 2000, + failure_threshold: 2, + recovery_threshold: 1, + idle_timeout_secs: if protocol == Protocol::Tcp { 7200 } else { 120 }, + backends: vec![], + } + } +} +/// Entire editable service; its address is an internal derived reservation. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VipConfig { + /// Stable random 128-bit id in hexadecimal. + pub id: String, + /// One DNS label, unique within the network. + pub name: String, + /// Independent listeners. + pub listeners: Vec, +} +impl VipConfig { + /// Creates an empty named service with a fresh stable identity. + pub fn new(name: impl Into) -> Self { + Self { + id: hex::encode(rand::random::<[u8; 16]>()), + name: name.into(), + listeners: vec![], + } + } + /// Validates a complete draft before any state is changed. + pub fn validate(&self, limits: &Limits) -> Result<()> { + if self.id.len() != 32 || hex::decode(&self.id).is_err() { + return Err(invalid("invalid VIP id")); + } + if self.name.is_empty() + || self.name.len() > 63 + || self.name.starts_with('-') + || self.name.ends_with('-') + || !self + .name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + { + return Err(invalid( + "VIP name must be a lowercase DNS label (letters, digits and hyphens)", + )); + } + if self.listeners.len() > limits.max_vip_listeners { + return Err(invalid("too many VIP listeners")); + } + let mut seen = BTreeSet::new(); + for l in &self.listeners { + if l.id.len() != 32 || hex::decode(&l.id).is_err() { + return Err(invalid("invalid listener identity")); + } + if l.port == 0 || !seen.insert((l.protocol, l.port)) { + return Err(invalid( + "listener ports must be nonzero and unique per protocol", + )); + } + if !(250..=3_600_000).contains(&l.check_interval_ms) + || !(100..=60_000).contains(&l.check_timeout_ms) + || l.failure_threshold == 0 + || l.recovery_threshold == 0 + || !(1..=604800).contains(&l.idle_timeout_secs) + { + return Err(invalid( + "invalid check interval, timeout, thresholds or flow idle timeout", + )); + } + if l.backends.len() > limits.max_vip_backends { + return Err(invalid("too many listener backends")); + } + let mut backends = BTreeSet::new(); + for b in &l.backends { + if b.port == 0 || b.peer.parse::().is_err() || !backends.insert(b) { + return Err(invalid( + "backend must have a unique node identity and a nonzero port", + )); + } + } + } + if crate::state::vip::flatten(self)?.len() > limits.max_vip_fields { + return Err(invalid("VIP configuration exceeds signed field capacity")); + } + Ok(()) + } +} +pub(crate) fn invalid(reason: impl Into) -> Error { + Error::Rejected { + reason: reason.into(), + } +} +/// Cumulative IP traffic, excluding probes. Reset on agent restart. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Counters { + /// Outbound packets. + pub tx_packets: u64, + /// Outbound bytes. + pub tx_bytes: u64, + /// Inbound packets. + pub rx_packets: u64, + /// Inbound bytes. + pub rx_bytes: u64, + /// Created flow bindings. + pub flows: u64, + /// UDP failovers. + pub failovers: u64, + /// Packets rejected or lacking an eligible backend. + pub dropped: u64, +} +/// One backend, as seen by this client. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackendStatus { + /// Configured endpoint. + pub backend: Backend, + /// Eligibility and reason. + pub state: String, + /// Smoothed full data-plane RTT, microseconds; unknown stays absent. + pub rtt_us: Option, + /// Last successful path measurement age. + pub measurement_age_ms: Option, + /// Current bindings. + pub active_flows: u32, + /// Local traffic counters. + pub counters: Counters, +} +/// Runtime view of one listener. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ListenerStatus { + /// Protocol. + pub protocol: Protocol, + /// Frontend port. + pub port: u16, + /// Best backend for a new flow, if measured and eligible. + pub selected: Option, + /// Configured and draining endpoints. + pub backends: Vec, + /// Local counters. + pub counters: Counters, +} +/// Configuration and local diagnostics returned through the same API to CLI and GUI. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VipStatus { + /// Persisted configuration. + pub config: VipConfig, + /// Opaque revision used when saving a draft. + pub revision: String, + /// Internal service address. + pub address: Option, + /// Conflict or allocation issue, if any. + pub error: Option, + /// Local listener status. + pub listeners: Vec, +} diff --git a/crates/tsunagi-core/src/vip/packet.rs b/crates/tsunagi-core/src/vip/packet.rs new file mode 100644 index 0000000..9e8661a --- /dev/null +++ b/crates/tsunagi-core/src/vip/packet.rs @@ -0,0 +1,566 @@ +//! Bounded IPv4 parsing, NAT checksums and fragment reassembly at local ingress. +use super::Protocol; +use bytes::Bytes; +use std::collections::HashMap; +use std::net::Ipv4Addr; +use std::time::{Duration, Instant}; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct Tuple { + pub src: Ipv4Addr, + pub dst: Ipv4Addr, + pub sport: u16, + pub dport: u16, + pub protocol: Protocol, +} +impl Tuple { + pub fn reverse(self) -> Self { + Self { + src: self.dst, + dst: self.src, + sport: self.dport, + dport: self.sport, + protocol: self.protocol, + } + } +} +pub(crate) fn header(p: &[u8]) -> Option { + if p.len() < 20 || p[0] >> 4 != 4 { + return None; + } + let n = usize::from(p[0] & 15) * 4; + let total = usize::from(u16::from_be_bytes([p[2], p[3]])); + (n >= 20 && n <= p.len() && total == p.len() && checksum(&p[..n]) == 0).then_some(n) +} +pub(crate) fn addresses(p: &[u8]) -> Option<(Ipv4Addr, Ipv4Addr)> { + header(p)?; + Some(( + Ipv4Addr::new(p[12], p[13], p[14], p[15]), + Ipv4Addr::new(p[16], p[17], p[18], p[19]), + )) +} +pub(crate) fn tuple(p: &[u8]) -> Option { + let n = header(p)?; + if u16::from_be_bytes([p[6], p[7]]) & 0x3fff != 0 { + return None; + } + let protocol = match p[9] { + 6 => Protocol::Tcp, + 17 => Protocol::Udp, + _ => return None, + }; + let min = if protocol == Protocol::Tcp { 20 } else { 8 }; + if p.len() < n + min { + return None; + } + if protocol == Protocol::Tcp { + let h = usize::from(p[n + 12] >> 4) * 4; + if h < 20 || n + h > p.len() { + return None; + } + } else if usize::from(u16::from_be_bytes([p[n + 4], p[n + 5]])) != p.len() - n { + return None; + } + let (src, dst) = addresses(p)?; + Some(Tuple { + src, + dst, + sport: u16::from_be_bytes([p[n], p[n + 1]]), + dport: u16::from_be_bytes([p[n + 2], p[n + 3]]), + protocol, + }) +} +pub(crate) fn rewrite(p: &[u8], t: Tuple) -> Option { + let n = header(p)?; + if p.len() < n + if t.protocol == Protocol::Tcp { 20 } else { 8 } { + return None; + } + let mut out = p.to_vec(); + out[12..16].copy_from_slice(&t.src.octets()); + out[16..20].copy_from_slice(&t.dst.octets()); + out[n..n + 2].copy_from_slice(&t.sport.to_be_bytes()); + out[n + 2..n + 4].copy_from_slice(&t.dport.to_be_bytes()); + finish(&mut out, n, t.protocol); + Some(Bytes::from(out)) +} +fn finish(p: &mut [u8], n: usize, protocol: Protocol) { + p[10..12].fill(0); + let c = checksum(&p[..n]); + p[10..12].copy_from_slice(&c.to_be_bytes()); + let offset = n + if protocol == Protocol::Tcp { 16 } else { 6 }; + p[offset..offset + 2].fill(0); + let mut sum = 0u32; + for pair in p[12..20].as_chunks::<2>().0 { + sum += u32::from(u16::from_be_bytes([pair[0], pair[1]])); + } + sum += u32::from(p[9]) + (p.len() - n) as u32; + let c = checksum_seed(&p[n..], sum); + let c = if c == 0 && protocol == Protocol::Udp { + 0xffff + } else { + c + }; + p[offset..offset + 2].copy_from_slice(&c.to_be_bytes()); +} +pub(crate) fn checksum(p: &[u8]) -> u16 { + checksum_seed(p, 0) +} +fn checksum_seed(p: &[u8], mut sum: u32) -> u16 { + for pair in p.chunks(2) { + sum += u32::from(u16::from_be_bytes([pair[0], *pair.get(1).unwrap_or(&0)])); + } + while sum >> 16 != 0 { + sum = (sum & 0xffff) + (sum >> 16); + } + !(sum as u16) +} +pub(crate) fn tcp_syn(p: &[u8]) -> bool { + header(p).is_some_and(|n| p[9] == 6 && p.get(n + 13).is_some_and(|f| f & 0x12 == 2)) +} +pub(crate) fn syn_sequence(p: &[u8]) -> Option { + if !tcp_syn(p) { + return None; + } + let n = header(p)?; + let bytes = p.get(n + 4..n + 8)?; + Some(u32::from_be_bytes(bytes.try_into().ok()?)) +} +pub(crate) fn empty_syn(p: &[u8]) -> bool { + header(p).is_some_and(|n| { + tcp_syn(p) + && p.get(n + 12) + .is_some_and(|h| n + usize::from(h >> 4) * 4 == p.len()) + }) +} +pub(crate) fn syn_ack(p: &[u8]) -> bool { + header(p).is_some_and(|n| p[9] == 6 && p.get(n + 13).is_some_and(|f| f & 0x12 == 0x12)) +} +pub(crate) fn tcp_reset(p: &[u8]) -> bool { + header(p).is_some_and(|n| p[9] == 6 && p.get(n + 13).is_some_and(|f| f & 4 != 0)) +} +/// Rewrites an ICMP error and its quoted original header after NAT lookup. +pub(crate) fn icmp_quote(p: &[u8]) -> Option { + let n = header(p)?; + if p[9] != 1 || p.len() < n + 8 + 28 || ![3, 11, 12].contains(&p[n]) { + return None; + } + let q = &p[n + 8..]; + let h = usize::from(q[0] & 15) * 4; + if q[0] >> 4 != 4 || h < 20 || q.len() < h + 8 { + return None; + } + let protocol = match q[9] { + 6 => Protocol::Tcp, + 17 => Protocol::Udp, + _ => return None, + }; + Some(Tuple { + src: Ipv4Addr::new(q[12], q[13], q[14], q[15]), + dst: Ipv4Addr::new(q[16], q[17], q[18], q[19]), + sport: u16::from_be_bytes([q[h], q[h + 1]]), + dport: u16::from_be_bytes([q[h + 2], q[h + 3]]), + protocol, + }) +} +pub(crate) fn rewrite_icmp(p: &[u8], original: Tuple) -> Option { + let n = header(p)?; + icmp_quote(p)?; + let mut out = p.to_vec(); + out[12..16].copy_from_slice(&original.dst.octets()); + out[16..20].copy_from_slice(&original.src.octets()); + let q = &mut out[n + 8..]; + let h = usize::from(q[0] & 15) * 4; + // Incrementally update the quoted transport checksum when it is present. + let off = h + if original.protocol == Protocol::Tcp { + 16 + } else { + 6 + }; + if q.len() >= off + 2 { + let old = u16::from_be_bytes([q[off], q[off + 1]]); + if old != 0 || original.protocol == Protocol::Tcp { + let mut sum = u32::from(!old); + let mut replacement = Vec::from(original.src.octets()); + replacement.extend(original.dst.octets()); + replacement.extend(original.sport.to_be_bytes()); + replacement.extend(original.dport.to_be_bytes()); + let previous = [&q[12..20], &q[h..h + 4]].concat(); + for (a, b) in previous + .as_chunks::<2>() + .0 + .iter() + .zip(replacement.as_chunks::<2>().0) + { + sum += u32::from(!u16::from_be_bytes([a[0], a[1]])) + + u32::from(u16::from_be_bytes([b[0], b[1]])); + } + while sum >> 16 != 0 { + sum = (sum & 0xffff) + (sum >> 16); + } + let c = !(sum as u16); + q[off..off + 2].copy_from_slice( + &(if c == 0 && original.protocol == Protocol::Udp { + 0xffff + } else { + c + }) + .to_be_bytes(), + ); + } + } + q[12..16].copy_from_slice(&original.src.octets()); + q[16..20].copy_from_slice(&original.dst.octets()); + q[h..h + 2].copy_from_slice(&original.sport.to_be_bytes()); + q[h + 2..h + 4].copy_from_slice(&original.dport.to_be_bytes()); + q[10..12].fill(0); + let c = checksum(&q[..h]); + q[10..12].copy_from_slice(&c.to_be_bytes()); + out[n + 2..n + 4].fill(0); + let c = checksum(&out[n..]); + out[n + 2..n + 4].copy_from_slice(&c.to_be_bytes()); + out[10..12].fill(0); + let c = checksum(&out[..n]); + out[10..12].copy_from_slice(&c.to_be_bytes()); + Some(Bytes::from(out)) +} +pub(crate) fn reject(p: &[u8]) -> Option { + let t = tuple(p)?; + let n = header(p)?; + if t.protocol == Protocol::Tcp { + if !tcp_syn(p) { + return None; + } + let mut out = vec![0; 40]; + out[0] = 0x45; + out[2..4].copy_from_slice(&40u16.to_be_bytes()); + out[8] = 64; + out[9] = 6; + out[12..16].copy_from_slice(&t.dst.octets()); + out[16..20].copy_from_slice(&t.src.octets()); + out[20..22].copy_from_slice(&t.dport.to_be_bytes()); + out[22..24].copy_from_slice(&t.sport.to_be_bytes()); + let seq = u32::from_be_bytes(p[n + 4..n + 8].try_into().ok()?); + out[28..32].copy_from_slice(&seq.wrapping_add(1).to_be_bytes()); + out[32] = 0x50; + out[33] = 0x14; + finish(&mut out, 20, Protocol::Tcp); + Some(Bytes::from(out)) + } else { + let q = p.len().min(n + 8); + let mut out = vec![0; 28 + q]; + out[0] = 0x45; + let len = out.len() as u16; + out[2..4].copy_from_slice(&len.to_be_bytes()); + out[8] = 64; + out[9] = 1; + out[12..16].copy_from_slice(&t.dst.octets()); + out[16..20].copy_from_slice(&t.src.octets()); + out[20] = 3; + out[21] = 3; + out[28..].copy_from_slice(&p[..q]); + let c = checksum(&out[20..]); + out[22..24].copy_from_slice(&c.to_be_bytes()); + let c = checksum(&out[..20]); + out[10..12].copy_from_slice(&c.to_be_bytes()); + Some(Bytes::from(out)) + } +} +#[derive(Debug)] +struct Datagram { + at: Instant, + header: Option>, + parts: std::collections::BTreeMap>, + end: Option, + bytes: usize, +} +type FragmentKey = (Option<[u8; 32]>, Ipv4Addr, Ipv4Addr, u16, u8); +#[derive(Debug, Default)] +pub(crate) struct Reassembly { + pending: HashMap, +} +impl Reassembly { + pub fn expire(&mut self, now: Instant) { + self.pending + .retain(|_, d| now.duration_since(d.at) < Duration::from_secs(5)); + } + pub fn accept(&mut self, p: Bytes, max: usize, scope: Option<[u8; 32]>) -> Option { + let n = header(&p)?; + let flags = u16::from_be_bytes([p[6], p[7]]); + if flags & 0x3fff == 0 { + return Some(p); + } + let now = Instant::now(); + self.pending + .retain(|_, d| now.duration_since(d.at) < Duration::from_secs(5)); + let (src, dst) = addresses(&p)?; + let key = (scope, src, dst, u16::from_be_bytes([p[4], p[5]]), p[9]); + let start = usize::from(flags & 0x1fff) * 8; + let payload = &p[n..]; + let end = start.checked_add(payload.len())?; + if payload.is_empty() + || end > 65515 + || flags & 0x2000 != 0 && !payload.len().is_multiple_of(8) + { + self.pending.remove(&key); + return None; + } + if !self.pending.contains_key(&key) && self.pending.len() >= max { + return None; + } + let d = self.pending.entry(key).or_insert_with(|| Datagram { + at: now, + header: None, + parts: Default::default(), + end: None, + bytes: 0, + }); + if d.parts + .get(&start) + .is_some_and(|bytes| bytes.as_slice() == payload) + { + return None; + } + if d.parts.iter().any(|(s, b)| *s < end && start < s + b.len()) { + self.pending.remove(&key); + return None; + } + if flags & 0x2000 == 0 { + if d.end.is_some_and(|e| e != end) { + self.pending.remove(&key); + return None; + } + d.end = Some(end); + } + if d.end.is_some_and(|e| end > e) { + self.pending.remove(&key); + return None; + } + if start == 0 { + d.header = Some(p[..n].to_vec()); + } + d.bytes += payload.len(); + d.parts.insert(start, payload.to_vec()); + let complete = d.end.is_some_and(|e| e == d.bytes) && d.header.is_some(); + if !complete { + return None; + } + let d = self.pending.remove(&key)?; + let mut out = d.header?; + for part in d.parts.values() { + out.extend(part); + } + if out.len() > 65535 { + return None; + } + let len = out.len() as u16; + out[2..4].copy_from_slice(&len.to_be_bytes()); + out[6..8].fill(0); + out[10..12].fill(0); + let h = usize::from(out[0] & 15) * 4; + let c = checksum(&out[..h]); + out[10..12].copy_from_slice(&c.to_be_bytes()); + Some(Bytes::from(out)) + } +} + +/// Re-fragments an assembled NAT datagram for the unchanged overlay MTU. +pub(crate) fn fragments(packet: Bytes, mtu: u32) -> Vec { + if packet.len() <= mtu as usize { + return vec![packet]; + } + let Some(n) = header(&packet) else { + return vec![]; + }; + if u16::from_be_bytes([packet[6], packet[7]]) & 0x4000 != 0 { + return vec![]; + } + let size = (mtu as usize).saturating_sub(n) / 8 * 8; + if size == 0 { + return vec![]; + } + let mut copied = Vec::new(); + let mut offset = 20; + while offset < n { + let kind = packet[offset]; + if kind == 0 { + break; + } + if kind == 1 { + offset += 1; + continue; + } + let Some(&len) = packet.get(offset + 1) else { + return vec![]; + }; + let len = usize::from(len); + if len < 2 || offset + len > n { + return vec![]; + } + if kind & 0x80 != 0 { + copied.extend_from_slice(&packet[offset..offset + len]); + } + offset += len; + } + while !copied.len().is_multiple_of(4) { + copied.push(0); + } + let mut result = Vec::new(); + let payload = &packet[n..]; + for (start, part) in payload.chunks(size).enumerate() { + let offset = start * size; + let mut out = if offset == 0 { + packet[..n].to_vec() + } else { + let mut h = packet[..20].to_vec(); + h.extend_from_slice(&copied); + h[0] = 0x40 | (h.len() / 4) as u8; + h + }; + let header_len = out.len(); + out.extend(part); + let len = out.len() as u16; + out[2..4].copy_from_slice(&len.to_be_bytes()); + let flags = (offset / 8) as u16 + | if offset + part.len() < payload.len() { + 0x2000 + } else { + 0 + }; + out[6..8].copy_from_slice(&flags.to_be_bytes()); + out[10..12].fill(0); + let c = checksum(&out[..header_len]); + out[10..12].copy_from_slice(&c.to_be_bytes()); + result.push(Bytes::from(out)); + } + result +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + fn datagram() -> Bytes { + let mut p = vec![0; 3028]; + p[0] = 0x45; + p[2..4].copy_from_slice(&3028u16.to_be_bytes()); + p[8] = 64; + p[9] = 17; + p[12..16].copy_from_slice(&[10, 0, 0, 1]); + p[16..20].copy_from_slice(&[10, 0, 0, 2]); + p[20..22].copy_from_slice(&1000u16.to_be_bytes()); + p[22..24].copy_from_slice(&2000u16.to_be_bytes()); + p[24..26].copy_from_slice(&3008u16.to_be_bytes()); + finish(&mut p, 20, Protocol::Udp); + Bytes::from(p) + } + #[test] + fn fragmented_udp_reassembles_out_of_order_and_nat_checksums_are_valid() { + let p = datagram(); + let parts = fragments(p.clone(), 1280); + assert_eq!(parts.len(), 3); + let mut reassembly = Reassembly::default(); + let mut result = None; + for part in parts.into_iter().rev() { + if let Some(p) = reassembly.accept(part, 8, None) { + result = Some(p); + } + } + assert_eq!(result, Some(p.clone())); + let mut t = tuple(&p).unwrap(); + t.dst = Ipv4Addr::new(10, 0, 0, 240); + t.dport = 53; + let out = rewrite(&p, t).unwrap(); + assert_eq!(header(&out), Some(20)); + let mut sum = 0u32; + for pair in out[12..20].as_chunks::<2>().0 { + sum += u32::from(u16::from_be_bytes([pair[0], pair[1]])); + } + sum += 17 + (out.len() - 20) as u32; + assert_eq!(checksum_seed(&out[20..], sum), 0); + } + #[test] + fn overlaps_and_capacity_are_rejected_without_allocating_unbounded_state() { + let parts = fragments(datagram(), 1280); + let mut r = Reassembly::default(); + assert!(r.accept(parts[0].clone(), 0, None).is_none()); + assert!(r.pending.is_empty()); + assert!(r.accept(parts[0].clone(), 1, None).is_none()); + assert!(r.accept(parts[0].clone(), 1, None).is_none()); + assert_eq!(r.pending.len(), 1); + let mut conflict = parts[0].to_vec(); + conflict[20] ^= 1; + assert!(r.accept(Bytes::from(conflict), 1, None).is_none()); + assert!(r.pending.is_empty()); + for size in 0..80 { + let p = vec![0xff; size]; + assert!(tuple(&p).is_none()); + assert!(icmp_quote(&p).is_none()); + } + } + #[test] + fn fragment_assembly_is_scoped_to_authenticated_sender_and_expires_when_idle() { + let packet = datagram(); + let parts = fragments(packet.clone(), 1280); + let mut r = Reassembly::default(); + assert!(r.accept(parts[0].clone(), 8, Some([1; 32])).is_none()); + assert!(r.accept(parts[1].clone(), 8, Some([2; 32])).is_none()); + assert!(r.accept(parts[2].clone(), 8, Some([1; 32])).is_none()); + assert_eq!(r.accept(parts[1].clone(), 8, Some([1; 32])), Some(packet)); + assert_eq!(r.pending.len(), 1); + r.expire(Instant::now() + Duration::from_secs(6)); + assert!(r.pending.is_empty()); + } +} + +#[cfg(test)] +mod icmp_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[test] + fn pmtu_errors_restore_quoted_addresses_ports_and_checksums() { + let original = Tuple { + src: Ipv4Addr::new(10, 0, 0, 1), + dst: Ipv4Addr::new(10, 0, 0, 240), + sport: 40000, + dport: 53, + protocol: Protocol::Udp, + }; + let mut p = vec![0; 36]; + p[0] = 0x45; + p[2..4].copy_from_slice(&36u16.to_be_bytes()); + p[8] = 64; + p[9] = 17; + p[24..26].copy_from_slice(&16u16.to_be_bytes()); + let c = checksum(&p[..20]); + p[10..12].copy_from_slice(&c.to_be_bytes()); + let p = rewrite(&p, original).unwrap(); + let wire = Tuple { + dst: Ipv4Addr::new(10, 0, 0, 2), + sport: 50000, + dport: 5353, + ..original + }; + let translated = rewrite(&p, wire).unwrap(); + let mut error = vec![0; 56]; + error[0] = 0x45; + error[2..4].copy_from_slice(&56u16.to_be_bytes()); + error[8] = 64; + error[9] = 1; + error[12..16].copy_from_slice(&wire.dst.octets()); + error[16..20].copy_from_slice(&wire.src.octets()); + error[20] = 3; + error[21] = 4; + error[26..28].copy_from_slice(&1200u16.to_be_bytes()); + error[28..].copy_from_slice(&translated[..28]); + let c = checksum(&error[20..]); + error[22..24].copy_from_slice(&c.to_be_bytes()); + let c = checksum(&error[..20]); + error[10..12].copy_from_slice(&c.to_be_bytes()); + let restored = rewrite_icmp(&error, original).unwrap(); + assert_eq!(icmp_quote(&restored), Some(original)); + assert_eq!(checksum(&restored[20..]), 0); + assert_eq!(checksum(&restored[28..48]), 0); + assert_eq!(&restored[54..56], &p[26..28]); + assert_eq!(&restored[26..28], &1200u16.to_be_bytes()); + assert_eq!(&restored[12..16], &original.dst.octets()); + } +} diff --git a/crates/tsunagi-core/src/vip/router.rs b/crates/tsunagi-core/src/vip/router.rs new file mode 100644 index 0000000..470aaf0 --- /dev/null +++ b/crates/tsunagi-core/src/vip/router.rs @@ -0,0 +1,1186 @@ +//! Client-side flow NAT, data-plane probes and local service observations. +use super::packet::{self, Tuple}; +use super::{Backend, BackendStatus, Counters, Listener, ListenerStatus, Protocol, VipStatus}; +use crate::overlay::Route; +use crate::{Limits, NetworkId}; +use bytes::Bytes; +use iroh::EndpointId; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; +const PROBE: &[u8; 16] = b"tsunagi-vip-rtt1"; +#[derive(Debug)] +pub(crate) enum Decision { + Pass, + Send(Route, Bytes), + Local(Bytes), + Drop, +} +#[derive(Debug)] +enum Guard { + Tcp { _socket: tokio::net::TcpSocket }, + Udp { _socket: std::net::UdpSocket }, +} +fn reserve(protocol: Protocol) -> Option<(u16, Guard)> { + match protocol { + Protocol::Tcp => { + let socket = tokio::net::TcpSocket::new_v4().ok()?; + socket + .bind(SocketAddr::from((Ipv4Addr::UNSPECIFIED, 0))) + .ok()?; + Some(( + socket.local_addr().ok()?.port(), + Guard::Tcp { _socket: socket }, + )) + } + Protocol::Udp => { + let socket = std::net::UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)).ok()?; + Some(( + socket.local_addr().ok()?.port(), + Guard::Udp { _socket: socket }, + )) + } + } +} +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct Key { + network: NetworkId, + vip: String, + protocol: Protocol, + port: u16, +} +#[derive(Debug)] +struct Flow { + key: Key, + backend: Backend, + peer: EndpointId, + wire: Tuple, + at: Instant, + idle: Duration, + local: bool, + initial_syn: Option, + established: bool, + _guard: Guard, +} +#[derive(Debug, Default)] +struct Path { + address: Option, + available: bool, + rtt: Option, + heard: Option, + last_probe: Option, + pending: Option<(u64, Instant)>, + misses: u8, +} +#[derive(Debug, Default)] +struct Check { + at: Option, + pending: bool, + ever_ok: bool, + good: u8, + bad: u8, + negative_until: Option, +} +#[derive(Debug)] +struct Network { + local: EndpointId, + address: Option, + vips: Vec, +} +type CheckKey = (NetworkId, EndpointId, Ipv4Addr, Protocol, u16); +/// A bounded TCP connect check prepared off the packet path. +pub(crate) struct CheckJob { + key: CheckKey, + timeout: Duration, +} +impl CheckJob { + pub async fn run(self) -> (Self, bool) { + let ok = matches!( + tokio::time::timeout( + self.timeout, + tokio::net::TcpStream::connect((self.key.2, self.key.4)) + ) + .await, + Ok(Ok(_)) + ); + (self, ok) + } +} +#[derive(Debug, Default)] +struct State { + networks: HashMap, + paths: HashMap<(NetworkId, EndpointId), Path>, + checks: HashMap, + flows: HashMap, + reverse: HashMap<(NetworkId, EndpointId, Tuple), Tuple>, + retired: HashMap<(NetworkId, EndpointId, Tuple), (Instant, Guard)>, + counters: HashMap<(Key, Option), Counters>, + fragments: HashMap, + limits: Limits, + sequence: u64, + check_cursor: usize, +} +#[derive(Debug, Default)] +pub(crate) struct VipRouter { + state: Mutex, +} +impl VipRouter { + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + pub fn configure( + &self, + network: NetworkId, + local: EndpointId, + address: Option, + vips: Vec, + peers: Vec<(EndpointId, Ipv4Addr, bool)>, + limits: &Limits, + ) { + let mut s = self.lock(); + s.limits = limits.clone(); + let mut present = HashSet::new(); + for (peer, address, available) in peers { + present.insert(peer); + let p = s.paths.entry((network, peer)).or_default(); + if p.address != Some(address) { + p.rtt = None; + p.heard = None; + p.pending = None; + } + p.address = Some(address); + p.available = available; + } + for ((n, peer), p) in &mut s.paths { + if *n == network && !present.contains(peer) { + p.available = false; + } + } + if let Some(address) = address { + let p = s.paths.entry((network, local)).or_default(); + p.address = Some(address); + p.available = true; + p.rtt = Some(0); + p.heard = Some(Instant::now()); + } + s.networks.insert( + network, + Network { + local, + address, + vips, + }, + ); + } + pub fn remove_network(&self, network: NetworkId) { + let mut s = self.lock(); + s.networks.remove(&network); + s.fragments.remove(&network); + s.paths.retain(|(n, _), _| *n != network); + s.checks.retain(|(n, _, _, _, _), _| *n != network); + s.flows.retain(|_, f| f.key.network != network); + s.reverse.retain(|(n, _, _), _| *n != network); + s.retired.retain(|(n, _, _), _| *n != network); + s.counters.retain(|(k, _), _| k.network != network); + } + pub fn statuses(&self, network: NetworkId) -> Vec { + let s = self.lock(); + let Some(n) = s.networks.get(&network) else { + return vec![]; + }; + let mut views = n.vips.clone(); + for v in &mut views { + v.listeners = v + .config + .listeners + .iter() + .map(|l| { + let key = Key { + network, + vip: v.config.id.clone(), + protocol: l.protocol, + port: l.port, + }; + let mut backends: Vec<_> = l + .backends + .iter() + .map(|b| { + let (state, rtt, age) = eligibility(&s, network, n.local, l, b); + BackendStatus { + backend: b.clone(), + state, + rtt_us: rtt, + measurement_age_ms: age, + active_flows: s + .flows + .values() + .filter(|f| f.key == key && f.backend == *b) + .count() as u32, + counters: s + .counters + .get(&(key.clone(), Some(b.clone()))) + .cloned() + .unwrap_or_default(), + } + }) + .collect(); + for f in s.flows.values().filter(|f| f.key == key) { + if !backends.iter().any(|b| b.backend == f.backend) { + backends.push(BackendStatus { + backend: f.backend.clone(), + state: "draining".into(), + rtt_us: None, + measurement_age_ms: None, + active_flows: s + .flows + .values() + .filter(|x| x.key == key && x.backend == f.backend) + .count() as u32, + counters: s + .counters + .get(&(key.clone(), Some(f.backend.clone()))) + .cloned() + .unwrap_or_default(), + }); + } + } + let selected = if v.error.is_none() { + select(&s, network, n.local, l).map(|(b, _)| b) + } else { + None + }; + ListenerStatus { + protocol: l.protocol, + port: l.port, + selected, + backends, + counters: s.counters.get(&(key, None)).cloned().unwrap_or_default(), + } + }) + .collect(); + // A removed listener can still own draining flows; keep them observable. + let mut draining: BTreeMap<(Protocol, u16), Vec> = BTreeMap::new(); + for f in s + .flows + .values() + .filter(|f| f.key.network == network && f.key.vip == v.config.id) + { + if !v + .listeners + .iter() + .any(|l| l.protocol == f.key.protocol && l.port == f.key.port) + { + draining + .entry((f.key.protocol, f.key.port)) + .or_default() + .push(f.backend.clone()); + } + } + for ((protocol, port), mut backends) in draining { + backends.sort(); + backends.dedup(); + let key = Key { + network, + vip: v.config.id.clone(), + protocol, + port, + }; + let backends = backends + .into_iter() + .map(|backend| BackendStatus { + state: "draining".into(), + rtt_us: None, + measurement_age_ms: None, + active_flows: s + .flows + .values() + .filter(|f| f.key == key && f.backend == backend) + .count() as u32, + counters: s + .counters + .get(&(key.clone(), Some(backend.clone()))) + .cloned() + .unwrap_or_default(), + backend, + }) + .collect(); + v.listeners.push(ListenerStatus { + protocol, + port, + selected: None, + backends, + counters: s.counters.get(&(key, None)).cloned().unwrap_or_default(), + }); + } + } + views + } + pub fn check_capacity(&self) -> usize { + self.lock().limits.max_vip_checks + } + pub fn tick(&self, capacity: usize) -> (Vec<(Route, Bytes)>, Vec) { + let mut s = self.lock(); + let now = Instant::now(); + for fragments in s.fragments.values_mut() { + fragments.expire(now); + } + let expired: Vec<_> = s + .flows + .iter() + .filter(|(_, f)| now.duration_since(f.at) >= f.idle) + .map(|(t, _)| *t) + .collect(); + for t in expired { + remove_flow(&mut s, t); + } + s.retired.retain(|_, (until, _)| now < *until); + let live: HashSet<_> = s.flows.keys().copied().collect(); + s.reverse.retain(|_, t| live.contains(t)); + let mut probes = Vec::new(); + let mut jobs = Vec::new(); + let mut targets = BTreeMap::new(); + for (n, net) in &s.networks { + for v in &net.vips { + if v.error.is_some() { + continue; + } + for l in &v.config.listeners { + for b in &l.backends { + if let Ok(peer) = b.peer.parse::() { + targets + .entry((*n, peer)) + .or_insert_with(Vec::new) + .push((l.clone(), b.clone())); + } + } + } + } + } + let mut wanted_checks = HashSet::new(); + for ((network, peer), listeners) in targets { + let Some(net) = s.networks.get(&network) else { + continue; + }; + let (local, source) = (net.local, net.address); + let sequence = s.sequence.wrapping_add(1); + s.sequence = sequence; + let Some(path) = s.paths.get_mut(&(network, peer)) else { + continue; + }; + let Some(address) = path.address else { + continue; + }; + if peer == local { + path.rtt = Some(0); + path.heard = Some(now); + } else if path.available { + if path + .pending + .is_some_and(|(_, at)| now.duration_since(at) >= Duration::from_secs(3)) + { + path.pending = None; + path.misses = path.misses.saturating_add(1); + } + if path.pending.is_none() + && path + .last_probe + .is_none_or(|at| now.duration_since(at) >= Duration::from_secs(1)) + && let Some(source) = source + { + path.pending = Some((sequence, now)); + path.last_probe = Some(now); + probes.push((Route { network, peer }, probe(source, address, 1, sequence))); + } + } + for (l, b) in listeners { + let key = (network, peer, address, l.protocol, b.port); + wanted_checks.insert(key); + if l.protocol != Protocol::Tcp || !l.port_check { + continue; + } + let path = s.paths.get(&(network, peer)); + if !path.is_some_and(|p| p.available) { + continue; + } + let check = s.checks.entry(key).or_default(); + if !check.pending + && check.at.is_none_or(|at| { + now.duration_since(at) + >= Duration::from_millis(u64::from(l.check_interval_ms)) + }) + { + jobs.push(CheckJob { + key, + timeout: Duration::from_millis(u64::from(l.check_timeout_ms)), + }); + } + } + } + s.checks.retain(|key, _| wanted_checks.contains(key)); + let mut unique = HashSet::new(); + jobs.retain(|job| unique.insert(job.key)); + if !jobs.is_empty() { + let offset = s.check_cursor % jobs.len(); + jobs.rotate_left(offset); + s.check_cursor = s.check_cursor.wrapping_add(capacity.min(jobs.len())); + } + jobs.truncate(capacity); + for job in &jobs { + if let Some(check) = s.checks.get_mut(&job.key) { + check.pending = true; + check.at = Some(now); + } + } + (probes, jobs) + } + pub fn checked(&self, job: CheckJob, ok: bool) { + let mut s = self.lock(); + if let Some(c) = s.checks.get_mut(&job.key) { + c.pending = false; + if ok { + c.ever_ok = true; + c.good = c.good.saturating_add(1); + c.bad = 0; + } else { + c.bad = c.bad.saturating_add(1); + c.good = 0; + } + } + } + pub fn outgoing(&self, packet: Bytes) -> Decision { + let mut s = self.lock(); + let Some((src, dst)) = packet::addresses(&packet) else { + return Decision::Pass; + }; + let fragment = packet::header(&packet) + .is_some_and(|_| u16::from_be_bytes([packet[6], packet[7]]) & 0x3fff != 0); + let fragment_network = if fragment { + s.networks + .iter() + .find(|(_, n)| n.vips.iter().any(|v| v.address == Some(dst))) + .map(|(id, _)| *id) + .or_else(|| { + s.flows + .iter() + .find(|(t, f)| t.dst == dst || f.local && f.wire.src == dst) + .map(|(_, f)| f.key.network) + }) + } else { + None + }; + let packet = if let Some(network) = fragment_network { + let max = s.limits.max_vip_fragments; + let Some(p) = s + .fragments + .entry(network) + .or_default() + .accept(packet, max, None) + else { + return Decision::Drop; + }; + p + } else { + packet + }; + if let Some(quote) = packet::icmp_quote(&packet) { + let original = s + .networks + .iter() + .find_map(|(id, n)| s.reverse.get(&(*id, n.local, quote.reverse())).copied()); + if let Some(original) = original { + if let Some(f) = s.flows.get(&original) { + let check_key = ( + f.key.network, + f.peer, + quote.dst, + quote.protocol, + f.backend.port, + ); + if packet::header(&packet).is_some_and(|n| packet[n] == 3 && packet[n + 1] == 3) + { + s.checks.entry(check_key).or_default().negative_until = + Some(Instant::now() + Duration::from_secs(10)); + } + } + return packet::rewrite_icmp(&packet, original) + .map_or(Decision::Drop, Decision::Local); + } + } + // Local backends reply to the nonlocal VIP source through the TUN. + if let Some(t) = packet::tuple(&packet) { + let original = s + .networks + .iter() + .find_map(|(id, n)| s.reverse.get(&(*id, n.local, t)).copied()); + if let Some(original) = original { + return response(&mut s, original, packet, true); + } + } + if let Some(t) = packet::tuple(&packet) + && let Some(sequence) = packet::syn_sequence(&packet) + { + let replace = s.flows.get(&t).is_some_and(|f| { + let failed = s + .networks + .get(&f.key.network) + .and_then(|n| { + n.vips + .iter() + .find(|v| v.config.id == f.key.vip) + .and_then(|v| { + v.config + .listeners + .iter() + .find(|l| l.protocol == t.protocol && l.port == t.dport) + }) + .map(|l| { + eligibility(&s, f.key.network, n.local, l, &f.backend).0 != "ready" + }) + }) + .unwrap_or(true); + f.initial_syn != Some(sequence) + || !f.established && packet::empty_syn(&packet) && failed + }); + if replace { + remove_flow(&mut s, t); + } + } + // Existing bindings also retain old VIP addresses/listeners while draining. + if let Some(t) = packet::tuple(&packet) { + let draining = s.flows.get(&t).is_some_and(|f| { + if f.key.protocol == Protocol::Tcp { + return true; + } + let current = s.networks.get(&f.key.network).and_then(|n| { + n.vips + .iter() + .find(|v| v.address == Some(t.dst) && v.config.id == f.key.vip) + .and_then(|v| { + v.config + .listeners + .iter() + .find(|l| l.protocol == t.protocol && l.port == t.dport) + }) + .map(|l| (n.local, l)) + }); + current.is_none_or(|(local, l)| { + eligibility(&s, f.key.network, local, l, &f.backend).0 == "ready" + }) + }); + if draining { + return forward(&mut s, t, packet); + } + } + let Some((network, vip, local, address)) = s.networks.iter().find_map(|(id, n)| { + n.vips + .iter() + .find(|v| v.address == Some(dst)) + .map(|v| (*id, v.clone(), n.local, n.address)) + }) else { + return Decision::Pass; + }; + if address != Some(src) { + return Decision::Drop; + } + let max = s.limits.max_vip_fragments; + let Some(packet) = s + .fragments + .entry(network) + .or_default() + .accept(packet, max, None) + else { + return Decision::Drop; + }; + let Some(t) = packet::tuple(&packet) else { + return Decision::Drop; + }; + let Some(l) = vip + .config + .listeners + .iter() + .find(|l| l.protocol == t.protocol && l.port == t.dport) + .cloned() + else { + return packet::reject(&packet).map_or(Decision::Drop, Decision::Local); + }; + let key = Key { + network, + vip: vip.config.id.clone(), + protocol: t.protocol, + port: t.dport, + }; + if vip.error.is_some() { + count(&mut s, &key, None, |c| c.dropped += 1); + return packet::reject(&packet).map_or(Decision::Drop, Decision::Local); + } + if t.protocol == Protocol::Udp + && s.flows + .get(&t) + .is_some_and(|f| eligibility(&s, network, local, &l, &f.backend).0 != "ready") + { + remove_flow(&mut s, t); + count(&mut s, &key, None, |c| c.failovers += 1); + } + if s.flows.contains_key(&t) { + return forward(&mut s, t, packet); + } + if t.protocol == Protocol::Tcp && !packet::tcp_syn(&packet) { + return Decision::Drop; + } + let Some((backend, peer)) = select(&s, network, local, &l) else { + count(&mut s, &key, None, |c| c.dropped += 1); + return packet::reject(&packet).map_or(Decision::Drop, Decision::Local); + }; + if s.flows.len().saturating_add(s.retired.len()) >= s.limits.max_vip_flows { + count(&mut s, &key, None, |c| c.dropped += 1); + return packet::reject(&packet).map_or(Decision::Drop, Decision::Local); + } + let Some((port, guard)) = reserve(t.protocol) else { + return Decision::Drop; + }; + let Some(destination) = s.paths.get(&(network, peer)).and_then(|p| p.address) else { + return Decision::Drop; + }; + let is_local = peer == local; + let wire = Tuple { + src: if is_local { dst } else { src }, + dst: destination, + sport: port, + dport: backend.port, + protocol: t.protocol, + }; + let Some(out) = packet::rewrite(&packet, wire) else { + return Decision::Drop; + }; + s.reverse.insert((network, peer, wire.reverse()), t); + s.flows.insert( + t, + Flow { + key: key.clone(), + backend: backend.clone(), + peer, + wire, + at: Instant::now(), + idle: Duration::from_secs(u64::from(l.idle_timeout_secs)), + local: is_local, + initial_syn: packet::syn_sequence(&packet), + established: false, + _guard: guard, + }, + ); + count(&mut s, &key, Some(&backend), |c| { + c.flows += 1; + c.tx_packets += 1; + c.tx_bytes += out.len() as u64; + }); + if is_local { + Decision::Local(out) + } else { + Decision::Send(Route { network, peer }, out) + } + } + /// Called only after the overlay authenticated source and destination. + pub fn incoming(&self, network: NetworkId, peer: EndpointId, packet: Bytes) -> Decision { + let mut s = self.lock(); + if let Some((kind, sequence)) = parse_probe(&packet) { + let Some(net) = s.networks.get(&network) else { + return Decision::Drop; + }; + let Some(source) = net.address else { + return Decision::Drop; + }; + if kind == 1 { + let Some((_, destination)) = packet::addresses(&packet).map(|(a, b)| (b, a)) else { + return Decision::Drop; + }; + return Decision::Send( + Route { network, peer }, + probe(source, destination, 2, sequence), + ); + } + if kind == 2 + && let Some(p) = s.paths.get_mut(&(network, peer)) + && let Some((expected, at)) = p.pending + && expected == sequence + { + let rtt = at.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + p.rtt = Some( + p.rtt + .map_or(rtt, |old| old.saturating_mul(7).saturating_add(rtt) / 8), + ); + p.heard = Some(Instant::now()); + p.pending = None; + p.misses = 0; + } + return Decision::Drop; + } + if let Some(t) = packet::tuple(&packet) + && s.retired.contains_key(&(network, peer, t)) + { + return Decision::Drop; + } + // Ordinary non-VIP traffic retains its existing admission behavior. + if packet::header(&packet).is_none() || !s.flows.values().any(|f| f.key.network == network) + { + return Decision::Pass; + } + // Authentication happens before fragment assembly, preventing cross-peer mixing. + let was_fragmented = u16::from_be_bytes([packet[6], packet[7]]) & 0x3fff != 0; + let max = s.limits.max_vip_fragments; + let Some(packet) = + s.fragments + .entry(network) + .or_default() + .accept(packet, max, Some(*peer.as_bytes())) + else { + return Decision::Drop; + }; + if let Some(t) = packet::tuple(&packet) + && let Some(original) = s.reverse.get(&(network, peer, t)).copied() + { + return response(&mut s, original, packet, false); + } + if let Some(quote) = packet::icmp_quote(&packet) + && let Some(original) = s.reverse.get(&(network, peer, quote.reverse())).copied() + { + if let Some(f) = s.flows.get(&original) { + let backend = f.backend.clone(); + let key = f.key.clone(); + if packet::header(&packet).is_some_and(|n| packet[n] == 3 && packet[n + 1] == 3) { + let check = s + .checks + .entry((network, peer, quote.dst, quote.protocol, backend.port)) + .or_default(); + check.negative_until = Some(Instant::now() + Duration::from_secs(10)); + } + count(&mut s, &key, Some(&backend), |c| c.dropped += 1); + } + return packet::rewrite_icmp(&packet, original).map_or(Decision::Drop, Decision::Local); + } + if was_fragmented { + Decision::Local(packet) + } else { + Decision::Pass + } + } +} +fn forward(s: &mut State, t: Tuple, packet: Bytes) -> Decision { + let Some(f) = s.flows.get_mut(&t) else { + return Decision::Drop; + }; + f.at = Instant::now(); + if t.protocol == Protocol::Tcp && packet::tcp_reset(&packet) { + f.idle = Duration::from_secs(1); + } + let (wire, peer, local, backend, key) = + (f.wire, f.peer, f.local, f.backend.clone(), f.key.clone()); + let Some(out) = packet::rewrite(&packet, wire) else { + return Decision::Drop; + }; + count(s, &key, Some(&backend), |c| { + c.tx_packets += 1; + c.tx_bytes += out.len() as u64; + }); + if local { + Decision::Local(out) + } else { + Decision::Send( + Route { + network: key.network, + peer, + }, + out, + ) + } +} +fn remove_flow(s: &mut State, t: Tuple) { + if let Some(f) = s.flows.remove(&t) { + s.reverse.remove(&(f.key.network, f.peer, f.wire.reverse())); + s.retired.insert( + (f.key.network, f.peer, f.wire.reverse()), + (Instant::now() + Duration::from_secs(120), f._guard), + ); + } +} +fn count(s: &mut State, key: &Key, backend: Option<&Backend>, change: impl Fn(&mut Counters)) { + change(s.counters.entry((key.clone(), None)).or_default()); + if let Some(b) = backend { + change( + s.counters + .entry((key.clone(), Some(b.clone()))) + .or_default(), + ); + } +} +fn response(s: &mut State, original: Tuple, packet: Bytes, _local: bool) -> Decision { + let Some(f) = s.flows.get_mut(&original) else { + return Decision::Drop; + }; + f.at = Instant::now(); + if original.protocol == Protocol::Tcp && packet::syn_ack(&packet) { + f.established = true; + } + if original.protocol == Protocol::Tcp && packet::tcp_reset(&packet) { + f.idle = Duration::from_secs(1); + } + let (key, backend) = (f.key.clone(), f.backend.clone()); + let Some(out) = packet::rewrite(&packet, original.reverse()) else { + return Decision::Drop; + }; + count(s, &key, Some(&backend), |c| { + c.rx_packets += 1; + c.rx_bytes += out.len() as u64; + }); + Decision::Local(out) +} +fn eligibility( + s: &State, + network: NetworkId, + local: EndpointId, + l: &Listener, + b: &Backend, +) -> (String, Option, Option) { + let Ok(peer) = b.peer.parse::() else { + return ("invalid identity".into(), None, None); + }; + let Some(p) = s.paths.get(&(network, peer)) else { + return ("no data path".into(), None, None); + }; + let age = p + .heard + .map(|t| t.elapsed().as_millis().min(u128::from(u64::MAX)) as u64); + let rtt = p + .rtt + .filter(|_| peer == local || age.is_some_and(|n| n < 10000)); + if !p.available || p.misses >= 2 { + return ("no data path".into(), rtt, age); + } + if rtt.is_none() { + return ("measuring RTT".into(), None, age); + } + if let Some(address) = p.address { + let c = s.checks.get(&(network, peer, address, l.protocol, b.port)); + if c.is_some_and(|c| c.negative_until.is_some_and(|t| Instant::now() < t)) { + return ("port unreachable".into(), rtt, age); + } + if l.protocol == Protocol::Tcp && l.port_check { + let Some(c) = c else { + return ("checking port".into(), rtt, age); + }; + if c.bad >= l.failure_threshold { + return ("port check failed".into(), rtt, age); + } + if !c.ever_ok || c.good < l.recovery_threshold && c.bad == 0 { + return ("checking port".into(), rtt, age); + } + if c.good == 0 && c.bad < l.failure_threshold && c.at.is_none() { + return ("checking port".into(), rtt, age); + } + } + } + ("ready".into(), rtt, age) +} +fn select( + s: &State, + network: NetworkId, + local: EndpointId, + l: &Listener, +) -> Option<(Backend, EndpointId)> { + l.backends + .iter() + .filter_map(|b| { + let (state, rtt, _) = eligibility(s, network, local, l, b); + (state == "ready").then_some((rtt?, b.clone(), b.peer.parse::().ok()?)) + }) + .min_by_key(|(rtt, b, _)| (*rtt, b.clone())) + .map(|(_, b, p)| (b, p)) +} +fn probe(src: Ipv4Addr, dst: Ipv4Addr, kind: u8, sequence: u64) -> Bytes { + let mut p = vec![0; 45]; + p[0] = 0x45; + p[2..4].copy_from_slice(&45u16.to_be_bytes()); + p[8] = 64; + p[9] = 253; + p[12..16].copy_from_slice(&src.octets()); + p[16..20].copy_from_slice(&dst.octets()); + p[20..36].copy_from_slice(PROBE); + p[36] = kind; + p[37..45].copy_from_slice(&sequence.to_be_bytes()); + let c = packet::checksum(&p[..20]); + p[10..12].copy_from_slice(&c.to_be_bytes()); + Bytes::from(p) +} +fn parse_probe(p: &[u8]) -> Option<(u8, u64)> { + (packet::header(p) == Some(20) && p.len() == 45 && p[9] == 253 && &p[20..36] == PROBE).then( + || { + ( + p[36], + u64::from_be_bytes([p[37], p[38], p[39], p[40], p[41], p[42], p[43], p[44]]), + ) + }, + ) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + use crate::vip::VipConfig; + fn packet(t: Tuple, flags: u8) -> Bytes { + let len = if t.protocol == Protocol::Tcp { 40 } else { 36 }; + let mut p = vec![0; len]; + p[0] = 0x45; + p[2..4].copy_from_slice(&(len as u16).to_be_bytes()); + p[8] = 64; + p[9] = if t.protocol == Protocol::Tcp { 6 } else { 17 }; + p[12..16].copy_from_slice(&t.src.octets()); + p[16..20].copy_from_slice(&t.dst.octets()); + if t.protocol == Protocol::Tcp { + p[32] = 0x50; + p[33] = flags; + } else { + p[24..26].copy_from_slice(&((len - 20) as u16).to_be_bytes()); + } + let c = packet::checksum(&p[..20]); + p[10..12].copy_from_slice(&c.to_be_bytes()); + packet::rewrite(&p, t).unwrap() + } + fn fixture( + protocol: Protocol, + ) -> ( + VipRouter, + NetworkId, + EndpointId, + EndpointId, + EndpointId, + Tuple, + ) { + let network = NetworkId::from_bytes([7; 32]); + let local = iroh::SecretKey::generate().public(); + let a = iroh::SecretKey::generate().public(); + let b = iroh::SecretKey::generate().public(); + let mut cfg = VipConfig::new("service"); + let mut l = Listener::new(protocol, 443); + l.port_check = false; + l.backends = vec![ + Backend { + peer: a.to_string(), + port: 6443, + }, + Backend { + peer: b.to_string(), + port: 7443, + }, + ]; + cfg.listeners.push(l); + let router = VipRouter::default(); + let t = Tuple { + src: Ipv4Addr::new(10, 0, 0, 1), + dst: Ipv4Addr::new(10, 0, 0, 240), + sport: 40000, + dport: 443, + protocol, + }; + router.configure( + network, + local, + Some(t.src), + vec![VipStatus { + config: cfg, + revision: "r".into(), + address: Some(t.dst), + error: None, + listeners: vec![], + }], + vec![ + (a, Ipv4Addr::new(10, 0, 0, 2), true), + (b, Ipv4Addr::new(10, 0, 0, 3), true), + ], + &Limits::default(), + ); + { + let mut s = router.lock(); + for (peer, rtt) in [(a, 1000), (b, 2000)] { + let p = s.paths.get_mut(&(network, peer)).unwrap(); + p.rtt = Some(rtt); + p.heard = Some(Instant::now()); + } + } + (router, network, local, a, b, t) + } + #[tokio::test] + async fn tcp_uses_lowest_rtt_remaps_both_ports_and_drains_after_removal() { + let (r, n, _, a, _, t) = fixture(Protocol::Tcp); + let Decision::Send(route, out) = r.outgoing(packet(t, 2)) else { + panic!("route expected") + }; + assert_eq!(route.peer, a); + let wire = packet::tuple(&out).unwrap(); + assert_eq!(wire.dport, 6443); + assert_ne!(wire.sport, t.sport); + let reply = packet(wire.reverse(), 0x12); + let Decision::Local(reply) = r.incoming(n, a, reply) else { + panic!("reverse NAT expected") + }; + assert_eq!(packet::tuple(&reply), Some(t.reverse())); + assert_eq!(packet::checksum(&reply[..20]), 0); + { + r.lock().networks.get_mut(&n).unwrap().vips.clear(); + } + assert!(matches!(r.outgoing(packet(t, 0x10)), Decision::Send(_, _))); + } + #[tokio::test] + async fn udp_pins_until_failure_then_moves_without_replaying() { + let (r, n, _, a, b, t) = fixture(Protocol::Udp); + let Decision::Send(first, _) = r.outgoing(packet(t, 0)) else { + panic!("route expected") + }; + assert_eq!(first.peer, a); + { + let mut s = r.lock(); + s.paths.get_mut(&(n, b)).unwrap().rtt = Some(100); + } + let Decision::Send(still, _) = r.outgoing(packet(t, 0)) else { + panic!("route expected") + }; + assert_eq!(still.peer, a); + { + r.lock().paths.get_mut(&(n, a)).unwrap().available = false; + } + let Decision::Send(next, out) = r.outgoing(packet(t, 0)) else { + panic!("failover expected") + }; + assert_eq!(next.peer, b); + assert_eq!(packet::tuple(&out).unwrap().dport, 7443); + assert_eq!(r.statuses(n)[0].listeners[0].counters.failovers, 1); + } + #[tokio::test] + async fn local_backend_uses_nonlocal_source_and_restores_client_tuple() { + let (r, n, local, _, _, t) = fixture(Protocol::Udp); + { + let mut s = r.lock(); + let l = &mut s.networks.get_mut(&n).unwrap().vips[0].config.listeners[0]; + l.backends = vec![Backend { + peer: local.to_string(), + port: 5353, + }]; + } + let Decision::Local(out) = r.outgoing(packet(t, 0)) else { + panic!("local delivery expected") + }; + let wire = packet::tuple(&out).unwrap(); + assert_eq!(wire.src, t.dst); + assert_eq!(wire.dst, t.src); + assert_eq!(wire.dport, 5353); + let Decision::Local(reply) = r.outgoing(packet(wire.reverse(), 0)) else { + panic!("local reverse NAT expected") + }; + assert_eq!(packet::tuple(&reply), Some(t.reverse())); + } + #[tokio::test] + async fn active_tcp_port_checks_recover_after_a_real_listener_appears() { + let (r, n, local, _, _, _) = fixture(Protocol::Tcp); + let server = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let port = server.local_addr().unwrap().port(); + { + let mut s = r.lock(); + let net = s.networks.get_mut(&n).unwrap(); + let l = &mut net.vips[0].config.listeners[0]; + l.port_check = true; + l.failure_threshold = 1; + l.backends = vec![Backend { + peer: local.to_string(), + port, + }]; + s.paths.get_mut(&(n, local)).unwrap().address = Some(Ipv4Addr::LOCALHOST); + } + let (_, jobs) = r.tick(32); + for job in jobs { + let (job, ok) = job.run().await; + assert!(ok); + r.checked(job, ok); + } + assert!(r.statuses(n)[0].listeners[0].selected.is_some()); + drop(server); + { + for check in r.lock().checks.values_mut() { + check.at = None; + } + } + let (_, jobs) = r.tick(32); + for job in jobs { + let (job, ok) = job.run().await; + assert!(!ok); + r.checked(job, ok); + } + assert!(r.statuses(n)[0].listeners[0].selected.is_none()); + let _server = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .unwrap(); + { + for check in r.lock().checks.values_mut() { + check.at = None; + } + } + let (_, jobs) = r.tick(32); + for job in jobs { + let (job, ok) = job.run().await; + r.checked(job, ok); + } + assert!(r.statuses(n)[0].listeners[0].selected.is_some()); + } + #[test] + fn unknown_rtt_is_not_selected_and_reflected_probes_do_not_forge_measurements() { + let (r, n, _, a, _, _) = fixture(Protocol::Udp); + { + let mut s = r.lock(); + for p in s.paths.values_mut() { + p.rtt = None; + p.heard = None; + } + } + assert!(r.statuses(n)[0].listeners[0].selected.is_none()); + let (probes, _) = r.tick(32); + let (_, p) = probes + .into_iter() + .find(|(route, _)| route.peer == a) + .unwrap(); + r.incoming(n, a, p); + assert!(r.statuses(n)[0].listeners[0].selected.is_none()); + } + #[tokio::test] + async fn a_new_syn_reselects_backend_and_an_unanswered_empty_syn_can_fail_over() { + let (r, n, _, a, b, t) = fixture(Protocol::Tcp); + let Decision::Send(first, out) = r.outgoing(packet(t, 2)) else { + panic!("first SYN must be routed"); + }; + assert_eq!(first.peer, a); + let wire = packet::tuple(&out).unwrap(); + { + r.lock().paths.get_mut(&(n, a)).unwrap().available = false; + } + let Decision::Send(next, _) = r.outgoing(packet(t, 2)) else { + panic!("pending connect should recover"); + }; + assert_eq!(next.peer, b); + assert!(matches!( + r.incoming(n, a, packet(wire.reverse(), 0x12)), + Decision::Drop + )); + { + let mut s = r.lock(); + let path = s.paths.get_mut(&(n, a)).unwrap(); + path.available = true; + path.rtt = Some(1); + } + let mut fresh = packet(t, 2).to_vec(); + fresh[24..28].copy_from_slice(&1u32.to_be_bytes()); + let Decision::Send(new, _) = r.outgoing(Bytes::from(fresh)) else { + panic!("new connection should reselect"); + }; + assert_eq!(new.peer, a); + } +} diff --git a/crates/tsunagi-core/src/vip/tests.rs b/crates/tsunagi-core/src/vip/tests.rs new file mode 100644 index 0000000..5bffd9a --- /dev/null +++ b/crates/tsunagi-core/src/vip/tests.rs @@ -0,0 +1,144 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use super::*; +use crate::NetworkId; + +#[test] +fn concurrent_backend_edits_merge_and_deletion_cannot_resurrect() { + let net = NetworkId::from_bytes([7; 32]); + let a = iroh::SecretKey::generate(); + let b = iroh::SecretKey::generate(); + let limits = Limits::default(); + let mut state = VipSet::default(); + let id = hex::encode([9; 16]); + let initial = VipConfig { + id: id.clone(), + name: "api".into(), + listeners: vec![Listener::new(Protocol::Tcp, 443)], + }; + let changes = state.edit(net, &a, 1, &initial, None, &limits).unwrap(); + state.merge(net, changes, &limits).unwrap(); + let revision = state.revision(&id); + let mut first = initial.clone(); + first.listeners[0].backends.push(Backend { + peer: a.public().to_string(), + port: 6443, + }); + let mut second = initial.clone(); + second.listeners[0].backends.push(Backend { + peer: b.public().to_string(), + port: 7443, + }); + let x = state + .edit(net, &a, 2, &first, Some(&revision), &limits) + .unwrap(); + let y = state + .edit(net, &b, 1, &second, Some(&revision), &limits) + .unwrap(); + let mut reverse = state.clone(); + state.merge(net, x.clone(), &limits).unwrap(); + state.merge(net, y.clone(), &limits).unwrap(); + reverse.merge(net, y, &limits).unwrap(); + reverse.merge(net, x, &limits).unwrap(); + assert_eq!(state.configs(), reverse.configs()); + assert_eq!(state.configs()[0].listeners[0].backends.len(), 2); + assert!( + state + .edit(net, &a, 3, &initial, Some(&revision), &limits) + .is_err() + ); + let removed = state.delete(net, &a, 3, &id, &state.revision(&id)).unwrap(); + let old = state.entries(&id); + state.merge(net, removed, &limits).unwrap(); + state.merge(net, old, &limits).unwrap(); + assert!(state.configs().is_empty()); +} + +#[test] +fn signed_fields_reject_tampering_wrong_network_and_limits() { + let net = NetworkId::from_bytes([2; 32]); + let key = iroh::SecretKey::generate(); + let limits = Limits::default(); + let state = VipSet::default(); + let cfg = VipConfig { + id: hex::encode([3; 16]), + name: "dns".into(), + ..Default::default() + }; + let entries = state.edit(net, &key, 1, &cfg, None, &limits).unwrap(); + let mut bad = entries.clone(); + bad[0].clock += 1; + assert!(VipSet::default().merge(net, bad, &limits).is_err()); + assert!( + VipSet::default() + .merge(NetworkId::from_bytes([4; 32]), entries.clone(), &limits) + .is_err() + ); + let mut small = limits.clone(); + small.max_vips = 0; + assert!(VipSet::default().merge(net, entries, &small).is_err()); +} + +#[test] +fn deletion_history_does_not_exhaust_active_capacity_or_accept_stale_fields() { + let network = NetworkId::from_bytes([9; 32]); + let key = iroh::SecretKey::generate(); + let limits = Limits { + max_vips: 1, + max_vip_tombstones: 4, + ..Default::default() + }; + let mut state = VipSet::default(); + for index in 0..4 { + let config = VipConfig::new("reused-name"); + let fields = state + .edit(network, &key, index * 2 + 1, &config, None, &limits) + .unwrap(); + state.merge(network, fields.clone(), &limits).unwrap(); + let deletion = state + .delete( + network, + &key, + index * 2 + 2, + &config.id, + &state.revision(&config.id), + ) + .unwrap(); + state.merge(network, deletion, &limits).unwrap(); + assert_eq!(state.entries(&config.id).len(), 1); + assert!(!state.merge(network, fields, &limits).unwrap()); + } + let next = VipConfig::new("next"); + let fields = state.edit(network, &key, 9, &next, None, &limits).unwrap(); + state.merge(network, fields, &limits).unwrap(); + assert_eq!(state.configs().len(), 1); +} + +#[test] +fn reservations_avoid_nodes_and_name_conflicts_have_a_deterministic_holder() { + let network = NetworkId::from_bytes([4; 32]); + let key = iroh::SecretKey::generate(); + let limits = Limits::default(); + let mut state = VipSet::default(); + for version in 1..=2 { + let config = VipConfig::new("shared-name"); + let fields = state + .edit(network, &key, version, &config, None, &limits) + .unwrap(); + state.merge(network, fields, &limits).unwrap(); + } + let range = "10.0.0.0/29".parse().unwrap(); + let taken = ["10.0.0.1".parse().unwrap()].into_iter().collect(); + let views = state.views(Some(range), taken, &Default::default(), &limits); + assert!(views[0].error.is_none()); + assert!( + views[1] + .error + .as_deref() + .unwrap() + .contains("name conflicts") + ); + assert_ne!(views[0].address, views[1].address); + for v in views { + assert_ne!(v.address, Some("10.0.0.1".parse().unwrap())); + } +} diff --git a/crates/tsunagi-core/tests/vip.rs b/crates/tsunagi-core/tests/vip.rs new file mode 100644 index 0000000..70ef918 --- /dev/null +++ b/crates/tsunagi-core/tests/vip.rs @@ -0,0 +1,207 @@ +//! Shared services use real membership, signed state and mandatory SQLite. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::testing::{TestAgent, config_with, network, wait_for_peers, wait_until}; +use tsunagi::vip::{Backend, Listener, Protocol, VipConfig, VipStatus}; +use tsunagi::{Agent, NetworkId}; +async fn view(agent: &Agent, network: NetworkId, id: &str) -> VipStatus { + wait_until("VIP configuration reaches the replica", || async { + agent + .network_status(network) + .await + .ok()? + .vips + .into_iter() + .find(|v| v.config.id == id) + }) + .await +} +#[tokio::test] +async fn any_member_can_edit_services_and_an_offline_creator_is_not_required() { + let discovery = SharedMemoryDiscovery::new(); + let a = TestAgent::spawn(&discovery).await.unwrap(); + let b = TestAgent::spawn(&discovery).await.unwrap(); + let (name, secret) = network("shared-vips"); + let net = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&a.agent, net, 1).await; + let cfg = VipConfig::new("api"); + a.agent + .save_vip(net, cfg.clone(), None, false) + .await + .unwrap(); + let before = view(&b.agent, net, &cfg.id).await; + let mut edited = before.config.clone(); + let mut listener = Listener::new(Protocol::Tcp, 443); + listener.backends.push(Backend { + peer: b.agent.endpoint_id().to_string(), + port: 6443, + }); + edited.listeners.push(listener); + b.agent + .save_vip(net, edited.clone(), Some(before.revision.clone()), false) + .await + .unwrap(); + assert!( + b.agent + .save_vip(net, cfg.clone(), Some(before.revision), false) + .await + .is_err() + ); + wait_until("creator observes another member's edit", || async { + (view(&a.agent, net, &cfg.id).await.config == edited).then_some(()) + }) + .await; + let a_dir = a.stop().await; + let c = TestAgent::spawn(&discovery).await.unwrap(); + c.agent.join_network(&name, &secret).await.unwrap(); + assert_eq!(view(&c.agent, net, &cfg.id).await.config, edited); + let a = Agent::spawn(config_with(a_dir.path(), &discovery)) + .await + .unwrap(); + assert_eq!(view(&a, net, &cfg.id).await.config, edited); + let current = view(&c.agent, net, &cfg.id).await; + c.agent + .save_vip(net, current.config, Some(current.revision), true) + .await + .unwrap(); + wait_until("deletion reaches the original creator", || async { + a.network_status(net) + .await + .ok()? + .vips + .is_empty() + .then_some(()) + }) + .await; + a.shutdown().await; + b.stop().await; + c.stop().await; +} +#[tokio::test] +async fn invalid_drafts_are_atomic_and_services_do_not_cross_networks() { + let discovery = SharedMemoryDiscovery::new(); + let a = TestAgent::spawn(&discovery).await.unwrap(); + let (name, secret) = network("vip-one"); + let net = a.agent.join_network(&name, &secret).await.unwrap(); + let (other, secret) = network("vip-two"); + let second = a.agent.join_network(&other, &secret).await.unwrap(); + let mut cfg = VipConfig::new("api"); + cfg.listeners.push(Listener::new(Protocol::Udp, 53)); + a.agent + .save_vip(net, cfg.clone(), None, false) + .await + .unwrap(); + let before = view(&a.agent, net, &cfg.id).await; + let mut invalid = before.config.clone(); + invalid.name = "renamed".into(); + invalid.listeners.push(invalid.listeners[0].clone()); + assert!( + a.agent + .save_vip(net, invalid, Some(before.revision.clone()), false) + .await + .is_err() + ); + let after = view(&a.agent, net, &cfg.id).await; + assert_eq!(after.config, before.config); + assert_eq!(after.revision, before.revision); + assert!( + a.agent + .network_status(second) + .await + .unwrap() + .vips + .is_empty() + ); + a.stop().await; +} + +#[tokio::test] +async fn a_restored_author_advances_its_counter_past_its_own_updates_held_by_a_replica() { + use tsunagi::config::StoragePaths; + use tsunagi::storage::StateStore; + use tsunagi::vip::VipSet; + let discovery = SharedMemoryDiscovery::new(); + let a = TestAgent::spawn(&discovery).await.unwrap(); + let b = TestAgent::spawn(&discovery).await.unwrap(); + let (name, secret) = network("vip-author-recovery"); + let net = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&a.agent, net, 1).await; + let config = VipConfig::new("api"); + a.agent + .save_vip(net, config.clone(), None, false) + .await + .unwrap(); + view(&b.agent, net, &config.id).await; + let author = a.agent.endpoint_id(); + let adir = a.stop().await; + let bdir = b.stop().await; + let apath = StoragePaths::under(adir.path()).state_db(); + let bpath = StoragePaths::under(bdir.path()).state_db(); + // Model an old author backup with a newer, genuinely signed update retained + // by another offline replica. Connections and recovery below remain real. + tokio::task::spawn_blocking(move || { + let a = StateStore::open(apath).unwrap(); + let b = StateStore::open(bpath).unwrap(); + let mut set = VipSet::default(); + set.merge( + net, + a.vip_fields(net, 192).unwrap(), + &tsunagi::Limits::default(), + ) + .unwrap(); + let mut cfg = set.configs().remove(0); + cfg.name = "newer".into(); + let key = + iroh::SecretKey::from_bytes(&a.device_identity().unwrap().unwrap().secret_bytes()); + let fields = set + .edit( + net, + &key, + 100, + &cfg, + Some(&set.revision(&cfg.id)), + &tsunagi::Limits::default(), + ) + .unwrap(); + set.merge(net, fields, &tsunagi::Limits::default()).unwrap(); + b.put_vip_fields(net, &set.entries(&cfg.id), None).unwrap(); + }) + .await + .unwrap(); + let b = Agent::spawn(config_with(bdir.path(), &discovery)) + .await + .unwrap(); + let a = Agent::spawn(config_with(adir.path(), &discovery)) + .await + .unwrap(); + let learned = wait_until( + "the restored author learns its newer signed service", + || async { + let v = view(&a, net, &config.id).await; + (v.config.name == "newer").then_some(v) + }, + ) + .await; + let mut next = learned.config; + next.name = "latest".into(); + a.save_vip(net, next, Some(learned.revision), false) + .await + .unwrap(); + a.shutdown().await; + let path = StoragePaths::under(adir.path()).state_db(); + let version = tokio::task::spawn_blocking(move || { + StateStore::open(path) + .unwrap() + .own_record_version(net, author) + .unwrap() + }) + .await + .unwrap(); + assert!( + version > 100, + "the observed author counter must not be reused" + ); + b.shutdown().await; +} diff --git a/crates/tsunagi-tray/src/agent.rs b/crates/tsunagi-tray/src/agent.rs index 6c386ea..73588a8 100644 --- a/crates/tsunagi-tray/src/agent.rs +++ b/crates/tsunagi-tray/src/agent.rs @@ -21,6 +21,13 @@ const POLL_INTERVAL: Duration = Duration::from_secs(2); /// A request from the UI to the agent. pub(crate) enum Command { + SaveVip { + token: String, + network_id: String, + config: tsunagi::vip::VipConfig, + expected: Option, + delete: bool, + }, /// Retire the exact offline-node claim selected in the devices window. ForgetNode { network_id: String, @@ -55,8 +62,14 @@ pub(crate) enum Command { } /// The latest the worker knows, read by the UI each frame. +#[derive(Clone)] +pub(crate) struct VipSaveResult { + pub token: String, + pub result: Result, String>, +} #[derive(Clone, Default)] pub(crate) struct Snapshot { + pub vip_result: Option, /// The last status, or the reason it could not be read (agent down, no /// permission on the socket, …). pub status: Option>, @@ -84,6 +97,18 @@ pub(crate) struct AgentClient { } impl AgentClient { + #[cfg(test)] + pub(crate) fn test_client() -> (Self, mpsc::UnboundedReceiver) { + let (tx, rx) = mpsc::unbounded_channel(); + ( + Self { + tx, + shared: Arc::new(Mutex::new(Snapshot::default())), + pending_copy: Arc::new(Mutex::new(None)), + }, + rx, + ) + } /// Starts the worker on `runtime`, reporting to `ctx` (for repaints). pub(crate) fn spawn(runtime: &tokio::runtime::Handle, ctx: egui::Context) -> Self { let (tx, rx) = mpsc::unbounded_channel(); @@ -198,6 +223,22 @@ async fn run( ctx.request_repaint(); let outcome: Result = match command { + Command::SaveVip { + token, + network_id, + config, + expected, + delete, + } => { + let result = ipc::save_vip(socket, &network_id, config, expected, delete) + .await + .map_err(|e| e.to_string()); + lock(shared).vip_result = Some(VipSaveResult { + token, + result: result.clone(), + }); + result.map(|_| "VIP configuration saved".into()) + } Command::ForgetNode { network_id, peer, diff --git a/crates/tsunagi-tray/src/app.rs b/crates/tsunagi-tray/src/app.rs index 040d20b..d5e9dbb 100644 --- a/crates/tsunagi-tray/src/app.rs +++ b/crates/tsunagi-tray/src/app.rs @@ -64,6 +64,7 @@ impl eframe::App for App { ui::draw(ctx, &self.agent, &mut self.state, &snapshot, &self.traffic); self.show_devices_windows(ctx, &snapshot); + self.show_vip_windows(ctx, &snapshot); self.show_about_window(ctx, &snapshot); // A secret the worker fetched for the clipboard: copy it here, on the @@ -78,6 +79,51 @@ impl eframe::App for App { } impl App { + fn show_vip_windows(&mut self, ctx: &egui::Context, snapshot: &crate::agent::Snapshot) { + let Some(Ok(report)) = &snapshot.status else { + return; + }; + let ids: Vec<_> = self.state.vip_windows.keys().cloned().collect(); + for id in ids { + let Some(network) = report.networks.iter().find(|n| n.network_id == id) else { + self.state.vip_windows.remove(&id); + continue; + }; + let Some(state) = self.state.vip_windows.get_mut(&id) else { + continue; + }; + let builder = egui::ViewportBuilder::default() + .with_title(format!("tsunagi · VIP · {}", network.name)) + .with_app_id(crate::icon::APP_ID) + .with_icon(crate::icon::window()) + .with_inner_size([1000., 720.]) + .with_min_inner_size([650., 400.]); + let keep = ctx.show_viewport_immediate( + egui::ViewportId::from_hash_of(("vip", &id)), + builder, + |vctx, _| { + let close = crate::vip::show(vctx, &self.agent, state, network, snapshot); + if close { + return false; + } + if vctx.input(|i| i.viewport().close_requested()) { + if state.dirty() { + state.request_close(); + vctx.send_viewport_cmd(egui::ViewportCommand::CancelClose); + true + } else { + false + } + } else { + true + } + }, + ); + if !keep { + self.state.vip_windows.remove(&id); + } + } + } /// Renders the About window while it is open. fn show_about_window(&mut self, ctx: &egui::Context, snapshot: &crate::agent::Snapshot) { if !self.state.about_open { diff --git a/crates/tsunagi-tray/src/main.rs b/crates/tsunagi-tray/src/main.rs index f1c6507..923427a 100644 --- a/crates/tsunagi-tray/src/main.rs +++ b/crates/tsunagi-tray/src/main.rs @@ -19,6 +19,7 @@ mod icon; mod stats; mod tray; mod ui; +mod vip; mod watch; use eframe::egui; diff --git a/crates/tsunagi-tray/src/stats.rs b/crates/tsunagi-tray/src/stats.rs index 3c1a3bd..44c44ae 100644 --- a/crates/tsunagi-tray/src/stats.rs +++ b/crates/tsunagi-tray/src/stats.rs @@ -34,7 +34,7 @@ pub(crate) struct Series { } impl Series { - fn sample(&mut self, now: Instant, packets: u64, bytes: u64) { + pub(crate) fn sample(&mut self, now: Instant, packets: u64, bytes: u64) { if let Some((then, prev_packets, prev_bytes)) = self.last { let dt = now.saturating_duration_since(then).as_secs_f32(); if dt > 0.0 { diff --git a/crates/tsunagi-tray/src/ui.rs b/crates/tsunagi-tray/src/ui.rs index 9432dac..17d89de 100644 --- a/crates/tsunagi-tray/src/ui.rs +++ b/crates/tsunagi-tray/src/ui.rs @@ -41,6 +41,7 @@ pub(crate) struct UiState { pub(crate) host_edit: Option, /// Network ids whose devices window is open. pub(crate) open_devices: BTreeSet, + pub(crate) vip_windows: std::collections::BTreeMap, /// Whether the About window is open. pub(crate) about_open: bool, /// An exit-node choice waiting for the user to confirm it. @@ -56,6 +57,7 @@ impl Default for UiState { unit: Unit::Packets, host_edit: None, open_devices: BTreeSet::new(), + vip_windows: Default::default(), about_open: false, exit_confirm: None, forget_confirm: None, @@ -211,6 +213,7 @@ fn draw_network( egui::Frame::group(ui.style()).show(ui, |ui| { ui.horizontal(|ui| { format::copy_field(ui, network.name.as_str(), &network.name); + if ui.button("VIP").clicked(){state.vip_windows.entry(network.network_id.clone()).or_default();} ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { let leave = egui::Button::new(egui::RichText::new("Leave").color(egui::Color32::WHITE)) diff --git a/crates/tsunagi-tray/src/vip.rs b/crates/tsunagi-tray/src/vip.rs new file mode 100644 index 0000000..e0158fd --- /dev/null +++ b/crates/tsunagi-tray/src/vip.rs @@ -0,0 +1,461 @@ +//! Per-network VIP editor. Drafts never issue commands until Save is pressed. +use crate::agent::{AgentClient, Command, Snapshot}; +use crate::stats::{Series, Unit}; +use eframe::egui; +use std::collections::BTreeMap; +use tsunagi::ipc::NetworkReport; +use tsunagi::vip::{Backend, Listener, Protocol, VipConfig, VipStatus}; +#[derive(Clone)] +struct Draft { + config: VipConfig, + original: VipConfig, + revision: Option, + delete: bool, +} +impl Draft { + fn new(config: VipConfig, revision: Option) -> Self { + Self { + original: config.clone(), + config, + revision, + delete: false, + } + } + fn dirty(&self) -> bool { + self.revision.is_none() || self.delete || self.config != self.original + } +} +#[derive(Default)] +pub(crate) struct Window { + draft: Option, + pending: Option, + error: Option, + close_question: bool, + close_after_save: bool, + rates: BTreeMap, + generation: u64, + acknowledged_generation: Option, +} +impl Window { + pub fn request_close(&mut self) { + self.close_question = true; + self.close_after_save = true; + } + pub fn dirty(&self) -> bool { + self.draft.as_ref().is_some_and(Draft::dirty) || self.pending.is_some() + } +} +fn peer_name<'a>(n: &'a NetworkReport, id: &'a str) -> &'a str { + n.members + .iter() + .find(|m| m.endpoint_id == id) + .and_then(|m| m.hostname.as_deref()) + .unwrap_or(id) +} +pub(crate) fn show( + ctx: &egui::Context, + agent: &AgentClient, + state: &mut Window, + n: &NetworkReport, + snapshot: &Snapshot, +) -> bool { + if state.generation != snapshot.generation { + state.generation = snapshot.generation; + if let Some(at) = snapshot.at { + for v in &n.vips { + let counters = v.listeners.iter().fold((0, 0, 0, 0), |a, l| { + ( + a.0 + l.counters.tx_packets, + a.1 + l.counters.tx_bytes, + a.2 + l.counters.rx_packets, + a.3 + l.counters.rx_bytes, + ) + }); + let (tx, rx) = state.rates.entry(v.config.id.clone()).or_default(); + tx.sample(at, counters.0, counters.1); + rx.sample(at, counters.2, counters.3); + } + } + } + if let Some(result) = &snapshot.vip_result + && state.pending.as_deref() == Some(result.token.as_str()) + { + state.pending = None; + state.acknowledged_generation = Some(snapshot.generation); + match &result.result { + Ok(vips) => { + if let Some(draft) = &state.draft { + state.draft = vips + .iter() + .find(|v| v.config.id == draft.config.id) + .map(|v| Draft::new(v.config.clone(), Some(v.revision.clone()))); + } + state.error = None; + } + Err(error) => { + state.error = Some(error.clone()); + state.close_after_save = false; + } + } + } + let mut close = state.close_after_save && !state.dirty(); + let mut discard_new = false; + egui::CentralPanel::default().show(ctx, |ui| { + ui.heading(format!("VIP · {}", n.name)); + ui.weak("Configuration is shared with every member. Changes take effect only after Save."); + if state.close_question { + ui.horizontal(|ui| { + ui.label("Unsaved changes."); + if ui + .add_enabled( + state.pending.is_none(), + egui::Button::new("Discard and close"), + ) + .clicked() + { + state.draft = None; + state.close_question = false; + close = true; + } + if ui.button("Keep editing").clicked() { + state.close_question = false; + state.close_after_save = false; + } + }); + } + if let Some(error) = &state.error { + ui.colored_label(egui::Color32::LIGHT_RED, error); + } + ui.separator(); + ui.horizontal_wrapped(|ui| { + for vip in &n.vips { + let selected = state + .draft + .as_ref() + .is_some_and(|d| d.config.id == vip.config.id); + if ui.selectable_label(selected, &vip.config.name).clicked() { + if state.dirty() { + state.error = Some( + "Save or discard the current draft before selecting another VIP." + .into(), + ); + } else { + state.draft = + Some(Draft::new(vip.config.clone(), Some(vip.revision.clone()))); + state.error = None; + } + } + } + if ui.button("+ New VIP").clicked() { + if state.dirty() { + state.error = Some("Save or discard the current draft first.".into()); + } else { + state.draft = Some(Draft::new(VipConfig::new(""), None)); + state.error = None; + } + } + }); + let Some(draft) = &mut state.draft else { + ui.label("Select a VIP or create one."); + return; + }; + let current = n.vips.iter().find(|v| v.config.id == draft.config.id); + if state.pending.is_none() + && state.acknowledged_generation != Some(snapshot.generation) + && !draft.dirty() + && let Some(current) = current + && draft.revision.as_ref() != Some(¤t.revision) + { + *draft = Draft::new(current.config.clone(), Some(current.revision.clone())); + } + let changed = + current.is_some_and(|v| draft.revision.as_ref().is_some_and(|r| r != &v.revision)); + if changed { + ui.colored_label( + egui::Color32::YELLOW, + "This VIP changed on the network. Reload before saving; your draft is retained.", + ); + } + if draft.revision.is_some() && current.is_none() { + ui.colored_label( + egui::Color32::YELLOW, + "This VIP was removed on the network. The draft cannot overwrite its deletion.", + ); + } + let busy = state.pending.is_some(); + ui.horizontal(|ui| { + let valid = draft.config.validate(&tsunagi::Limits::default()); + if !n.active { + ui.weak("Start the network to save changes."); + } + if ui + .add_enabled( + n.active + && !busy + && draft.dirty() + && !changed + && (valid.is_ok() || draft.delete), + egui::Button::new("Save"), + ) + .clicked() + { + let token = VipConfig::new("").id; + state.pending = Some(token.clone()); + agent.send(Command::SaveVip { + token, + network_id: n.network_id.clone(), + config: draft.config.clone(), + expected: draft.revision.clone(), + delete: draft.delete, + }); + } + if ui + .add_enabled(!busy, egui::Button::new("Discard / reload")) + .clicked() + { + if let Some(v) = current { + *draft = Draft::new(v.config.clone(), Some(v.revision.clone())); + } else { + discard_new = true; + } + state.error = None; + } + if busy { + ui.spinner(); + ui.label("Saving…"); + } else if draft.dirty() { + ui.weak("Unsaved changes"); + } + if let Err(error) = valid { + ui.colored_label(egui::Color32::LIGHT_RED, error.to_string()); + } + }); + ui.separator(); + egui::ScrollArea::vertical().show(ui, |ui| { + ui.add_enabled_ui(!busy, |ui| { + ui.horizontal(|ui| { + ui.label("Name"); + ui.add( + egui::TextEdit::singleline(&mut draft.config.name) + .id(egui::Id::new(("vip-name", &draft.config.id))), + ); + ui.weak(format!(".{}", n.name)); + }); + if draft.revision.is_some() { + ui.checkbox(&mut draft.delete, "Delete this VIP when saved"); + } + if !draft.delete { + edit_listeners(ui, &mut draft.config, n); + } + }); + if let Some(current) = current { + ui.separator(); + status(ui, current, n, state.rates.get(¤t.config.id)); + } + }); + }); + if discard_new { + state.draft = None; + state.error = None; + } + close +} +fn edit_listeners(ui: &mut egui::Ui, config: &mut VipConfig, n: &NetworkReport) { + let mut remove = None; + for (index, l) in config.listeners.iter_mut().enumerate() { + ui.push_id(index,|ui|{egui::Frame::group(ui.style()).show(ui,|ui|{ + ui.horizontal(|ui|{egui::ComboBox::from_id_salt("protocol").selected_text(l.protocol.to_string()).show_ui(ui,|ui|{ui.selectable_value(&mut l.protocol,Protocol::Tcp,"TCP");ui.selectable_value(&mut l.protocol,Protocol::Udp,"UDP");});ui.label("Port");ui.add(egui::DragValue::new(&mut l.port).range(1..=65535));if ui.button("Remove listener").clicked(){remove=Some(index);}}); + if l.protocol==Protocol::Tcp{ui.checkbox(&mut l.port_check,"Active TCP port checks");}else{ui.weak("UDP health: encrypted path probes and explicit port errors; application readiness is unknown.");} + ui.collapsing("Checks and flow expiry",|ui|{egui::Grid::new("settings").show(ui,|ui|{ + ui.label("Check interval, ms");ui.add(egui::DragValue::new(&mut l.check_interval_ms).range(250..=3_600_000));ui.end_row(); + ui.label("Check timeout, ms");ui.add(egui::DragValue::new(&mut l.check_timeout_ms).range(100..=60_000));ui.end_row(); + ui.label("Failures before exclusion");ui.add(egui::DragValue::new(&mut l.failure_threshold).range(1..=255));ui.end_row(); + ui.label("Successes before recovery");ui.add(egui::DragValue::new(&mut l.recovery_threshold).range(1..=255));ui.end_row(); + ui.label("Flow idle timeout, seconds");ui.add(egui::DragValue::new(&mut l.idle_timeout_secs).range(1..=604800));ui.end_row(); + });}); + let mut remove_backend=None;for(i,b)in l.backends.iter_mut().enumerate(){ui.push_id(i,|ui|{ui.horizontal(|ui|{ + egui::ComboBox::from_id_salt("node").width(240.).selected_text(peer_name(n,&b.peer)).show_ui(ui,|ui|{for m in &n.members{let label=format!("{} ({})",m.hostname.as_deref().unwrap_or("unnamed"),m.endpoint_id.chars().take(8).collect::());ui.selectable_value(&mut b.peer,m.endpoint_id.clone(),label);}}); + ui.label("Target port");ui.add(egui::DragValue::new(&mut b.port).range(1..=65535));if ui.button("Remove").clicked(){remove_backend=Some(i);} + });});}if let Some(i)=remove_backend{l.backends.remove(i);} + if ui.button("+ Backend").clicked(){l.backends.push(Backend{peer:n.members.first().map(|m|m.endpoint_id.clone()).unwrap_or_default(),port:l.port});} +});}); + } + if let Some(index) = remove { + config.listeners.remove(index); + } + if ui.button("+ Listener").clicked() { + config.listeners.push(Listener::new(Protocol::Tcp, 443)); + } +} +fn status(ui: &mut egui::Ui, v: &VipStatus, n: &NetworkReport, rates: Option<&(Series, Series)>) { + ui.heading("Status · this device"); + if let Some(error) = &v.error { + ui.colored_label(egui::Color32::LIGHT_RED, error); + } + if let Some((tx, rx)) = rates { + ui.label(format!( + "TX {:.1} KiB/s · RX {:.1} KiB/s", + tx.rate(Unit::Bytes) / 1024., + rx.rate(Unit::Bytes) / 1024. + )); + } + for l in &v.listeners { + ui.strong(format!("{}/{}", l.protocol, l.port)); + let selected = l + .selected + .as_ref() + .map(|b| format!("{}:{}", peer_name(n, &b.peer), b.port)) + .unwrap_or_else(|| "none".into()); + ui.label(format!("Next new flow: {selected}")); + ui.label(format!( + "TX {} bytes / {} packets · RX {} bytes / {} packets · drops {} · UDP failovers {}", + l.counters.tx_bytes, + l.counters.tx_packets, + l.counters.rx_bytes, + l.counters.rx_packets, + l.counters.dropped, + l.counters.failovers + )); + for b in &l.backends { + let rtt = b + .rtt_us + .map(|v| format!("{:.2} ms", v as f64 / 1000.)) + .unwrap_or_else(|| "unknown".into()); + let age = b + .measurement_age_ms + .map(|v| format!("{:.1}s", v as f64 / 1000.)) + .unwrap_or_else(|| "unknown".into()); + ui.label(format!( + "{}:{} · {} · RTT {} (age {}) · {} active · TX/RX {}/{} bytes", + peer_name(n, &b.backend.peer), + b.backend.port, + b.state, + rtt, + age, + b.active_flows, + b.counters.tx_bytes, + b.counters.rx_bytes + )); + } + } + ui.collapsing("Internal address and identity", |ui| { + ui.monospace(&v.config.id); + if let Some(address) = v.address { + ui.monospace(address.to_string()); + } + }); +} +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + #[test] + fn editing_the_actual_form_sends_nothing_until_save_is_clicked() { + let (agent, mut commands) = AgentClient::test_client(); + let ctx = egui::Context::default(); + let mut config = VipConfig::new("api"); + config.listeners.push(Listener::new(Protocol::Tcp, 443)); + let mut state = Window { + draft: Some(Draft::new(config.clone(), Some("revision".into()))), + ..Default::default() + }; + let network = NetworkReport { + active: true, + name: "lab".into(), + vips: vec![VipStatus { + config: config.clone(), + revision: "revision".into(), + address: None, + error: None, + listeners: vec![], + }], + ..Default::default() + }; + let snapshot = Snapshot::default(); + let frame = |events: Vec, state: &mut Window| { + ctx.run( + egui::RawInput { + screen_rect: Some(egui::Rect::from_min_size( + egui::Pos2::ZERO, + egui::vec2(1000., 800.), + )), + events, + ..Default::default() + }, + |ctx| { + show(ctx, &agent, state, &network, &snapshot); + }, + ) + }; + frame(vec![], &mut state); + ctx.memory_mut(|m| m.request_focus(egui::Id::new(("vip-name", &config.id)))); + frame(vec![egui::Event::Text("new".into())], &mut state); + assert!(state.dirty()); + assert_ne!(state.draft.as_ref().unwrap().config.name, config.name); + assert!(commands.try_recv().is_err()); + assert_eq!(network.vips[0].config, config); + fn save_position(shape: &egui::Shape) -> Option { + match shape { + egui::Shape::Text(text) if text.galley.text() == "Save" => { + Some(text.pos + text.galley.rect.center().to_vec2()) + } + egui::Shape::Vec(shapes) => shapes.iter().find_map(save_position), + _ => None, + } + } + // Save was disabled before this edit; allow the next layout to register + // its now-enabled hit target before dispatching the pointer press. + let ready = frame(vec![], &mut state); + let position = ready + .shapes + .iter() + .find_map(|s| save_position(&s.shape)) + .unwrap(); + frame( + vec![ + egui::Event::PointerMoved(position), + egui::Event::PointerButton { + pos: position, + button: egui::PointerButton::Primary, + pressed: true, + modifiers: egui::Modifiers::NONE, + }, + ], + &mut state, + ); + frame( + vec![egui::Event::PointerButton { + pos: position, + button: egui::PointerButton::Primary, + pressed: false, + modifiers: egui::Modifiers::NONE, + }], + &mut state, + ); + let Command::SaveVip { + config: saved, + expected, + delete, + .. + } = commands.try_recv().unwrap() + else { + panic!("Save must submit one complete VIP draft"); + }; + assert_ne!(saved.name, config.name); + assert_eq!(expected.as_deref(), Some("revision")); + assert!(!delete); + assert!(commands.try_recv().is_err()); + } + #[test] + fn all_listener_settings_render_without_a_live_agent() { + let ctx = egui::Context::default(); + let mut cfg = VipConfig::new("api"); + cfg.listeners.push(Listener::new(Protocol::Tcp, 443)); + cfg.listeners.push(Listener::new(Protocol::Udp, 53)); + let n = NetworkReport::default(); + let _ = ctx.run(egui::RawInput::default(), |ctx| { + egui::CentralPanel::default().show(ctx, |ui| edit_listeners(ui, &mut cfg, &n)); + }); + assert_eq!(cfg.listeners.len(), 2); + } +} diff --git a/crates/tsunagi-wg-quic/tests/wireguard.rs b/crates/tsunagi-wg-quic/tests/wireguard.rs index d936cf7..cd338c2 100644 --- a/crates/tsunagi-wg-quic/tests/wireguard.rs +++ b/crates/tsunagi-wg-quic/tests/wireguard.rs @@ -1770,3 +1770,188 @@ async fn lan_broadcast_fanout_is_scoped_opt_in_and_does_not_reflood() { agent.shutdown().await; } } + +fn vip_segment( + source: Ipv4Addr, + destination: Ipv4Addr, + sport: u16, + dport: u16, + udp: bool, + flags: u8, +) -> Bytes { + fn sum(bytes: &[u8]) -> u16 { + let mut s: u32 = bytes + .chunks(2) + .map(|b| u32::from(u16::from_be_bytes([b[0], *b.get(1).unwrap_or(&0)]))) + .sum(); + while s >> 16 != 0 { + s = (s & 0xffff) + (s >> 16); + } + !(s as u16) + } + let mut p = vec![0; if udp { 36 } else { 40 }]; + p[0] = 0x45; + let len = p.len() as u16; + p[2..4].copy_from_slice(&len.to_be_bytes()); + p[8] = 64; + p[9] = if udp { 17 } else { 6 }; + p[12..16].copy_from_slice(&source.octets()); + p[16..20].copy_from_slice(&destination.octets()); + p[20..22].copy_from_slice(&sport.to_be_bytes()); + p[22..24].copy_from_slice(&dport.to_be_bytes()); + if udp { + p[24..26].copy_from_slice(&16u16.to_be_bytes()); + } else { + p[32] = 0x50; + p[33] = flags; + p[24..28].copy_from_slice(&123u32.to_be_bytes()); + } + let mut pseudo = p[12..20].to_vec(); + pseudo.extend([0, p[9]]); + pseudo.extend(((p.len() - 20) as u16).to_be_bytes()); + pseudo.extend(&p[20..]); + let c = sum(&pseudo); + let offset = if udp { 26 } else { 36 }; + p[offset..offset + 2].copy_from_slice(&c.to_be_bytes()); + let c = sum(&p[..20]); + p[10..12].copy_from_slice(&c.to_be_bytes()); + Bytes::from(p) +} +#[tokio::test] +async fn vip_tcp_and_udp_remap_ports_over_real_encryption_and_preserve_source_admission() { + use tsunagi::vip::{Backend, Listener, Protocol, VipConfig}; + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("vip-encrypted"); + let a = WgAgent::spawn(&discovery, "vipa").await; + let b = WgAgent::spawn(&discovery, "vipb").await; + let net = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + a.wait_for_tunnels(net, 1).await; + b.wait_for_tunnels(net, 1).await; + let aa = a.overlay(net).await; + let ab = b.overlay(net).await; + let ta = a.tun(net).await; + let tb = b.tun(net).await; + let mut cfg = VipConfig::new("service"); + for (protocol, port, backend) in [(Protocol::Tcp, 443, 6443), (Protocol::Udp, 53, 5353)] { + let mut l = Listener::new(protocol, port); + l.port_check = false; + l.backends.push(Backend { + peer: b.endpoint_id().to_string(), + port: backend, + }); + cfg.listeners.push(l); + } + a.agent + .save_vip(net, cfg.clone(), None, false) + .await + .unwrap(); + let vip = wait_until("data-plane probes measure the backend RTT", || async { + let status = a.agent.network_status(net).await.ok()?; + let v = status.vips.iter().find(|v| v.config.id == cfg.id)?; + (v.listeners.iter().all(|l| l.selected.is_some())) + .then_some(v.address?) + .filter(|_| v.listeners.len() == 2) + }) + .await; + for (udp, port, target) in [(false, 443, 6443), (true, 53, 5353)] { + ta.push_from_os(vip_segment(aa, vip, 40000, port, udp, 2)); + let received = tokio::time::timeout(Duration::from_secs(5), tb.pop_to_os()) + .await + .unwrap() + .unwrap(); + assert_eq!(&received[12..16], &aa.octets()); + assert_eq!(&received[16..20], &ab.octets()); + assert_eq!(u16::from_be_bytes([received[22], received[23]]), target); + let translated = u16::from_be_bytes([received[20], received[21]]); + tb.push_from_os(vip_segment(ab, aa, target, translated, udp, 0x12)); + let reply = tokio::time::timeout(Duration::from_secs(5), ta.pop_to_os()) + .await + .unwrap() + .unwrap(); + assert_eq!(&reply[12..16], &vip.octets()); + assert_eq!(u16::from_be_bytes([reply[20], reply[21]]), port); + assert_eq!(u16::from_be_bytes([reply[22], reply[23]]), 40000); + // A backend cannot send a VIP source over its authenticated tunnel. + let forged = vip_segment(vip, aa, port, 40000, udp, 0x12); + tb.push_from_os(forged); + } + wait_until("forged VIP sources are rejected before NAT", || async { + (a.agent.overlay()?.counters.wrong_source >= 2).then_some(()) + }) + .await; + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn vip_udp_reaches_another_backend_after_the_selected_agent_stops() { + use tsunagi::vip::{Backend, Listener, Protocol, VipConfig}; + let discovery = SharedMemoryDiscovery::new(); + let (name, secret) = network("vip-failover"); + let a = WgAgent::spawn(&discovery, "vipfa").await; + let b = WgAgent::spawn(&discovery, "vipfb").await; + let c = WgAgent::spawn(&discovery, "vipfc").await; + let net = a.agent.join_network(&name, &secret).await.unwrap(); + for node in [&b, &c] { + node.agent.join_network(&name, &secret).await.unwrap(); + } + for node in [&a, &b, &c] { + node.wait_for_tunnels(net, 2).await; + } + let mut config = VipConfig::new("datagrams"); + let mut listener = Listener::new(Protocol::Udp, 8888); + listener.backends = vec![ + Backend { + peer: b.endpoint_id().to_string(), + port: 9000, + }, + Backend { + peer: c.endpoint_id().to_string(), + port: 9001, + }, + ]; + config.listeners.push(listener); + a.agent + .save_vip(net, config.clone(), None, false) + .await + .unwrap(); + let first = wait_until("both UDP backends have measured paths", || async { + let s = a.agent.network_status(net).await.ok()?; + let v = s.vips.iter().find(|v| v.config.id == config.id)?; + let l = v.listeners.first()?; + (l.backends.iter().all(|b| b.rtt_us.is_some())).then_some((v.address?, l.selected.clone()?)) + }) + .await; + let (dead, alive) = if first.1.peer == b.endpoint_id().to_string() { + (b, c) + } else { + (c, b) + }; + let address = a.overlay(net).await; + let tun = a.tun(net).await; + let dest = dead.tun(net).await; + tun.push_from_os(vip_segment(address, first.0, 41000, 8888, true, 0)); + tokio::time::timeout(Duration::from_secs(5), dest.pop_to_os()) + .await + .unwrap() + .unwrap(); + dead.shutdown().await; + wait_until("the remaining backend becomes the next choice", || async { + let s = a.agent.network_status(net).await.ok()?; + let v = s.vips.iter().find(|v| v.config.id == config.id)?; + (v.listeners.first()?.selected.as_ref()?.peer == alive.endpoint_id().to_string()) + .then_some(()) + }) + .await; + tun.push_from_os(vip_segment(address, first.0, 41000, 8888, true, 0)); + let packet = tokio::time::timeout(Duration::from_secs(5), alive.tun(net).await.pop_to_os()) + .await + .unwrap() + .unwrap(); + assert_eq!(&packet[16..20], &alive.overlay(net).await.octets()); + let view = a.agent.network_status(net).await.unwrap(); + assert_eq!(view.vips[0].listeners[0].counters.failovers, 1); + a.shutdown().await; + alive.shutdown().await; +} diff --git a/crates/tsunagi/Cargo.toml b/crates/tsunagi/Cargo.toml index a4bea6c..42e329d 100644 --- a/crates/tsunagi/Cargo.toml +++ b/crates/tsunagi/Cargo.toml @@ -27,6 +27,8 @@ clap = { version = "4.5", features = ["derive", "env"] } # colour is never written where it would show up as rubbish. anstream = "1.0" anstyle = "1.0" +serde_json = "1.0" +tempfile.workspace = true # Local interface addresses, for the diagnostics in `status`. netwatch = "0.19.3" diff --git a/crates/tsunagi/src/main.rs b/crates/tsunagi/src/main.rs index 93ace3e..230ca7a 100644 --- a/crates/tsunagi/src/main.rs +++ b/crates/tsunagi/src/main.rs @@ -14,6 +14,7 @@ use std::time::Duration; #[cfg(windows)] mod winservice; +mod vip_cli; use clap::{Args, Parser, Subcommand, ValueEnum}; use tsunagi::agent::Event; use tsunagi::config::{AgentConfig, StoragePaths, TransportPolicy}; @@ -52,6 +53,8 @@ struct Cli { #[derive(Debug, Subcommand)] enum Command { + /// Manage named shared TCP/UDP virtual services. + Vip(vip_cli::VipArgs), /// Shows this device's identity, and changes it. /// /// Every item follows the same shape: name it to see it, name it with a @@ -927,6 +930,7 @@ async fn run(command: Command) -> Result<(), Box> { Command::Join(args) => join_command(args).await, Command::Network(args) => network_command(args).await, Command::Dns(args) => dns_command(args).await, + Command::Vip(args) => vip_cli::run(args).await, Command::ExitNode(args) => exit_node_command(args).await, Command::Wipe(args) => wipe(args).await, // Taken in `main`, before there is a runtime: the service manager @@ -1317,7 +1321,12 @@ fn spawn_dns(agent: Agent, port: u16) -> DnsService { let members = network.members.iter().filter_map(|member| { Some((member.hostname.clone()?, member.overlay_address_v4?)) }); - zones.push(Zone::new(zone, members)); + let services = network + .vips + .iter() + .filter(|v| v.error.is_none()) + .filter_map(|v| Some((v.config.name.clone(), v.address?))); + zones.push(Zone::new(zone, members.chain(services))); } let zones = Zones::new(zones); let names = zones.names() as u32; @@ -1577,6 +1586,23 @@ struct AgentControl { } impl tsunagi::ipc::ReportSource for AgentControl { + fn save_vip( + &self, + network: String, + config: tsunagi::vip::VipConfig, + expected: Option, + delete: bool, + ) -> tsunagi::BoxFuture<'_, Result, String>> { + Box::pin(async move { + let network = network + .parse() + .map_err(|e| format!("invalid network id: {e}"))?; + self.agent + .save_vip(network, config, expected, delete) + .await + .map_err(|e| e.to_string()) + }) + } fn forget_node( &self, network: String, @@ -5307,6 +5333,7 @@ async fn build_report( }; NetworkReport { + vips: network.vips.clone(), protocols: network.protocols.clone(), auto_start: network.auto_start, exit: exit_report(network), @@ -5740,6 +5767,7 @@ mod status_tests { /// The situation that prompted this: one peer left and came back. fn network_after_a_peer_returned() -> NetworkReport { NetworkReport { + vips: Vec::new(), protocols: vec!["wg-quic".into()], auto_start: true, exit: Default::default(), diff --git a/crates/tsunagi/src/vip_cli.rs b/crates/tsunagi/src/vip_cli.rs new file mode 100644 index 0000000..4a6cbf1 --- /dev/null +++ b/crates/tsunagi/src/vip_cli.rs @@ -0,0 +1,506 @@ +//! Named virtual service administration; every mutation submits one complete draft. +use super::{PathArgs, control_socket, resolve_network_index}; +use clap::{Args, Subcommand}; +use std::path::PathBuf; +use tsunagi::ipc::{self, NetworkReport}; +use tsunagi::vip::{Backend, Listener, Protocol, VipConfig, VipStatus}; +type Result = std::result::Result>; +#[derive(Debug, Args)] +#[command( + after_help = "Examples:\n tsng vip create lab k8s\n tsng vip port add lab k8s tcp/443 --backend cp1:6443 --backend cp2:7443\n tsng vip backend add lab k8s tcp/443 cp3:6443\n tsng vip status lab k8s --watch\n tsng vip edit lab k8s\n\nNames, full ids and unique id prefixes are accepted. Changes are shared with the network.\nTCP port checks default to on; UDP uses data-path probes and explicit ICMP errors." +)] +pub(crate) struct VipArgs { + #[command(flatten)] + paths: PathArgs, + /// Agent control socket; defaults to the running service. + #[arg(long, global = true, env = "TSUNAGI_CONTROL_SOCKET")] + control_socket: Option, + /// Machine-readable configuration and local observations. + #[arg(long, global = true)] + json: bool, + #[command(subcommand)] + action: Action, +} +#[derive(Debug, Args)] +struct Target { + /// Network exact name, id or unique id prefix. + network: String, + /// VIP exact name, id or unique id prefix. + vip: String, +} +#[derive(Debug, Args)] +struct PortTarget { + #[command(flatten)] + target: Target, + /// Listener as tcp/443 or udp/53. + listener: String, +} +#[derive(Debug, Subcommand)] +enum Action { + /// List services in one network, or every configured network. + List { network: Option }, + /// Create a named VIP with an automatically reserved internal address. + Create { network: String, name: String }, + /// Show the complete saved configuration and local status. + Show(Target), + /// Show local health, routing choices and cumulative traffic counters. + Status { + #[command(flatten)] + target: Target, + /// Refresh once a second until interrupted. + #[arg(long)] + watch: bool, + }, + /// Edit the entire configuration in VISUAL/EDITOR and save it atomically. + Edit(Target), + /// Remove a VIP; existing flows drain until completion or idle expiry. + Remove(Target), + /// Add, configure or remove listeners independently of the VIP. + Port { + #[command(subcommand)] + action: PortAction, + }, + /// Change a listener's destination nodes and application ports. + Backend { + #[command(subcommand)] + action: BackendAction, + }, +} +#[derive(Debug, Subcommand)] +enum PortAction { + /// Add a listener, optionally with its complete initial backend set. + Add { + #[command(flatten)] + target: PortTarget, + /// Node name/id followed by :port. Repeat for multiple destinations. + #[arg(long)] + backend: Vec, + }, + /// Change check settings and flow idle expiry. + Set { + #[command(flatten)] + target: PortTarget, + /// Active TCP connect checks: on or off. + #[arg(long,value_parser=["on","off"])] + port_check: Option, + /// Milliseconds between active checks (250..3600000). + #[arg(long)] + check_interval_ms: Option, + /// TCP check deadline in milliseconds (100..60000). + #[arg(long)] + check_timeout_ms: Option, + /// Consecutive failures before exclusion. + #[arg(long)] + failures: Option, + /// Consecutive successes before recovery. + #[arg(long)] + successes: Option, + /// Flow idle expiry in seconds (1..604800). + #[arg(long)] + idle_timeout: Option, + }, + /// Remove a listener from new-flow selection; existing flows drain. + Remove(PortTarget), +} +#[derive(Debug, Subcommand)] +enum BackendAction { + /// Add a destination node and port to a listener. + Add { + #[command(flatten)] + target: PortTarget, + backend: String, + }, + /// Remove a destination from new-flow selection. + Remove { + #[command(flatten)] + target: PortTarget, + backend: String, + }, +} +impl Action { + fn network(&self) -> Option<&str> { + match self { + Self::List { network } => network.as_deref(), + Self::Create { network, .. } => Some(network), + Self::Show(t) | Self::Edit(t) | Self::Remove(t) | Self::Status { target: t, .. } => { + Some(&t.network) + } + Self::Port { action } => Some( + &match action { + PortAction::Add { target, .. } + | PortAction::Set { target, .. } + | PortAction::Remove(target) => target, + } + .target + .network, + ), + Self::Backend { action } => Some( + &match action { + BackendAction::Add { target, .. } | BackendAction::Remove { target, .. } => { + target + } + } + .target + .network, + ), + } + } +} +fn parse_listener(s: &str) -> Result<(Protocol, u16)> { + let (p, n) = s + .split_once('/') + .ok_or("listener must be tcp/PORT or udp/PORT")?; + let port = n.parse::()?; + if port == 0 { + return Err("port must not be zero".into()); + } + Ok((p.parse()?, port)) +} +fn backend(n: &NetworkReport, s: &str) -> Result { + let (node, port) = s.rsplit_once(':').ok_or("backend must be NODE:PORT")?; + let peer = match ipc::resolve_node(n, node) { + Ok(member) => member.endpoint_id.clone(), + Err(error) => { + let matches = n + .members + .iter() + .any(|m| m.hostname.as_deref() == Some(node) || m.endpoint_id.starts_with(node)); + if matches { + return Err(error.into()); + } + node.parse::() + .map_err(|_| error)? + .to_string() + } + }; + let port = port.parse()?; + if port == 0 { + return Err("backend port must not be zero".into()); + } + Ok(Backend { peer, port }) +} +fn selected(n: &NetworkReport, t: &Target) -> Result { + Ok(ipc::resolve_vip(n, &t.vip)?.clone()) +} +fn listener<'a>(c: &'a mut VipConfig, target: &PortTarget) -> Result<&'a mut Listener> { + let (p, n) = parse_listener(&target.listener)?; + c.listeners + .iter_mut() + .find(|l| l.protocol == p && l.port == n) + .ok_or_else(|| format!("no listener {} on {}", target.listener, c.name).into()) +} +fn name_of<'a>(n: &'a NetworkReport, peer: &'a str) -> &'a str { + n.members + .iter() + .find(|m| m.endpoint_id == peer) + .and_then(|m| m.hostname.as_deref()) + .unwrap_or(peer) +} +fn show(n: &NetworkReport, v: &VipStatus, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string(v)?); + return Ok(()); + } + println!( + "{}.{} {}", + v.config.name, + n.name, + v.error.as_deref().unwrap_or("configured") + ); + for l in &v.listeners { + let choice = l + .selected + .as_ref() + .map(|b| format!("{}:{}", name_of(n, &b.peer), b.port)) + .unwrap_or_else(|| "none".into()); + println!( + " {}/{} next: {} flows: {} tx/rx: {}/{} bytes drops: {} UDP failovers: {}", + l.protocol, + l.port, + choice, + l.backends.iter().map(|b| b.active_flows).sum::(), + l.counters.tx_bytes, + l.counters.rx_bytes, + l.counters.dropped, + l.counters.failovers + ); + for b in &l.backends { + let rtt = b + .rtt_us + .map(|n| format!("{:.2} ms", n as f64 / 1000.)) + .unwrap_or_else(|| "unknown".into()); + println!( + " {}:{} {} RTT {} active {}", + name_of(n, &b.backend.peer), + b.backend.port, + b.state, + rtt, + b.active_flows + ); + } + } + Ok(()) +} +pub(crate) async fn run(args: VipArgs) -> Result<()> { + let custom = args.paths.state_dir.is_some(); + let paths = args.paths.resolve()?; + let socket = control_socket(&paths, args.control_socket.as_ref(), custom); + let report = ipc::request_status(&socket).await?; + if let Action::List { network: None } = &args.action { + for n in &report.networks { + for v in &n.vips { + show(n, v, args.json)?; + } + } + return Ok(()); + } + let selector = args.action.network().ok_or("network is required")?; + let names: Vec<_> = report + .networks + .iter() + .map(|n| (n.network_id.clone(), n.name.clone())) + .collect(); + let network = &report.networks[resolve_network_index(&names, selector)?]; + if let Action::Status { + target, + watch: true, + } = &args.action + { + let mut previous: Option<(std::time::Instant, u64, u64)> = None; + loop { + let report = ipc::request_status(&socket).await?; + let n = report + .networks + .iter() + .find(|n| n.network_id == network.network_id) + .ok_or("network was removed")?; + let vip = selected(n, target)?; + let (tx, rx) = vip.listeners.iter().fold((0u64, 0u64), |a, l| { + ( + a.0.saturating_add(l.counters.tx_bytes), + a.1.saturating_add(l.counters.rx_bytes), + ) + }); + let now = std::time::Instant::now(); + if let Some((at, old_tx, old_rx)) = previous + && !args.json + { + let secs = now.duration_since(at).as_secs_f64(); + if secs > 0. { + println!( + "TX {:.1} KiB/s · RX {:.1} KiB/s", + tx.saturating_sub(old_tx) as f64 / secs / 1024., + rx.saturating_sub(old_rx) as f64 / secs / 1024. + ); + } + } + previous = Some((now, tx, rx)); + show(n, &vip, args.json)?; + tokio::select! {_=tokio::signal::ctrl_c()=>break,_=tokio::time::sleep(std::time::Duration::from_secs(1))=>{}} + } + return Ok(()); + } + let mut edited_file = None; + let (mut config, expected, delete) = match &args.action { + Action::List { .. } => { + for v in &network.vips { + show(network, v, args.json)?; + } + return Ok(()); + } + Action::Show(t) | Action::Status { target: t, .. } => { + let v = selected(network, t)?; + show(network, &v, args.json)?; + if matches!(args.action, Action::Show(_)) && !args.json { + println!("{}", serde_json::to_string_pretty(&v.config)?); + } + return Ok(()); + } + Action::Create { name, .. } => (VipConfig::new(name), None, false), + Action::Remove(t) => { + let v = selected(network, t)?; + (v.config, Some(v.revision), true) + } + Action::Edit(t) => { + let v = selected(network, t)?; + let mut file = tempfile::Builder::new() + .prefix("tsunagi-vip-") + .suffix(".json") + .tempfile()?; + use std::io::Write; + file.write_all(serde_json::to_string_pretty(&v.config)?.as_bytes())?; + file.flush()?; + let editor = std::env::var("VISUAL") + .or_else(|_| std::env::var("EDITOR")) + .unwrap_or_else(|_| "vi".into()); + let mut words = editor.split_whitespace(); + let program = words.next().ok_or("empty EDITOR")?; + let status = std::process::Command::new(program) + .args(words) + .arg(file.path()) + .status()?; + if !status.success() { + return Err("editor failed; VIP unchanged".into()); + } + let bytes = std::fs::read(file.path())?; + let cfg: VipConfig = match serde_json::from_slice(&bytes) { + Ok(c) => c, + Err(e) => { + let (_, path) = file.keep()?; + return Err( + format!("invalid draft: {e}; retained at {}", path.display()).into(), + ); + } + }; + if cfg.id != v.config.id { + return Err("VIP id cannot be edited".into()); + } + edited_file = Some(file); + (cfg, Some(v.revision), false) + } + Action::Port { action } => { + let target = match action { + PortAction::Add { target, .. } + | PortAction::Set { target, .. } + | PortAction::Remove(target) => target, + }; + let v = selected(network, &target.target)?; + let mut cfg = v.config; + match action { + PortAction::Add { + target, + backend: destinations, + } => { + let (p, port) = parse_listener(&target.listener)?; + let mut l = Listener::new(p, port); + l.backends = destinations + .iter() + .map(|b| backend(network, b)) + .collect::>()?; + cfg.listeners.push(l); + } + PortAction::Remove(target) => { + let (p, port) = parse_listener(&target.listener)?; + listener(&mut cfg, target)?; + cfg.listeners.retain(|l| l.protocol != p || l.port != port); + } + PortAction::Set { + target, + port_check, + check_interval_ms, + check_timeout_ms, + failures, + successes, + idle_timeout, + } => { + let l = listener(&mut cfg, target)?; + if let Some(v) = port_check { + l.port_check = v == "on"; + } + if let Some(v) = check_interval_ms { + l.check_interval_ms = *v; + } + if let Some(v) = check_timeout_ms { + l.check_timeout_ms = *v; + } + if let Some(v) = failures { + l.failure_threshold = *v; + } + if let Some(v) = successes { + l.recovery_threshold = *v; + } + if let Some(v) = idle_timeout { + l.idle_timeout_secs = *v; + } + } + } + (cfg, Some(v.revision), false) + } + Action::Backend { action } => { + let (target, destination, remove) = match action { + BackendAction::Add { target, backend } => (target, backend, false), + BackendAction::Remove { target, backend } => (target, backend, true), + }; + let v = selected(network, &target.target)?; + let mut cfg = v.config; + let b = backend(network, destination)?; + let l = listener(&mut cfg, target)?; + if remove { + if !l.backends.contains(&b) { + return Err("backend is not configured".into()); + } + l.backends.retain(|x| x != &b); + } else { + l.backends.push(b); + } + (cfg, Some(v.revision), false) + } + }; + // Canonical order makes displayed drafts and shared state deterministic. + config.listeners.sort_by_key(|l| (l.protocol, l.port)); + for l in &mut config.listeners { + l.backends.sort(); + } + let id = config.id.clone(); + let vips = match ipc::save_vip(&socket, &network.network_id, config, expected, delete).await { + Ok(vips) => vips, + Err(error) => { + if let Some(file) = edited_file { + let (_, path) = file.keep()?; + return Err(format!("{error}; draft retained at {}", path.display()).into()); + } + return Err(error.into()); + } + }; + if delete { + if args.json { + println!("{{\"removed\":\"{id}\"}}"); + } else { + println!("VIP removed; existing flows drain."); + } + } else if let Some(v) = vips.iter().find(|v| v.config.id == id) { + show(network, v, args.json)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + #[test] + fn vip_commands_parse_complete_listener_and_settings() { + let cli = super::super::Cli::try_parse_from([ + "tsng", + "vip", + "port", + "add", + "lab", + "k8s", + "tcp/443", + "--backend", + "cp1:6443", + "--backend", + "cp2:7443", + ]); + assert!(cli.is_ok(), "{cli:?}"); + assert!( + super::super::Cli::try_parse_from([ + "tsng", + "vip", + "port", + "set", + "lab", + "k8s", + "tcp/443", + "--port-check", + "off", + "--failures", + "3" + ]) + .is_ok() + ); + assert!(parse_listener("tcp/0").is_err()); + assert!(parse_listener("sctp/53").is_err()); + } +} diff --git a/crates/tsunagi/tests/dns_service.rs b/crates/tsunagi/tests/dns_service.rs index f3fcbd6..81cf372 100644 --- a/crates/tsunagi/tests/dns_service.rs +++ b/crates/tsunagi/tests/dns_service.rs @@ -345,3 +345,95 @@ fn opposing_dns_flags_are_rejected() { assert!(!result.status.success()); assert!(String::from_utf8_lossy(&result.stderr).contains("cannot be used with")); } + +#[test] +fn vip_cli_creates_multiple_ports_updates_dns_and_rejects_partial_invalid_changes() { + let agent = start("vip-test", 15367); + let created = agent.run(&["vip", "create", "vip-test", "api", "--json"]); + assert!( + created.status.success(), + "{}", + String::from_utf8_lossy(&created.stderr) + ); + let vip: tsunagi::vip::VipStatus = serde_json::from_slice(&created.stdout).unwrap(); + assert!(vip.config.listeners.is_empty()); + let reply = wait_for_answer("127.0.0.1:15367".parse().unwrap(), "api.vip-test"); + assert!(has_answer(&reply)); + let socket = tsunagi::ipc::control_socket_path(&agent.dir.path().join("state")); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let status = runtime + .block_on(tsunagi::ipc::request_status(socket)) + .unwrap(); + let backend = format!("{}:6443", status.endpoint_id); + let added = agent.run(&[ + "vip", + "port", + "add", + "vip-test", + "api", + "tcp/443", + "--backend", + &backend, + ]); + assert!( + added.status.success(), + "{}", + String::from_utf8_lossy(&added.stderr) + ); + assert!( + agent + .run(&["vip", "port", "add", "vip-test", "api", "udp/53"]) + .status + .success() + ); + assert!( + !agent + .run(&["vip", "port", "add", "vip-test", "api", "tcp/443"]) + .status + .success() + ); + assert!( + agent + .run(&[ + "vip", + "port", + "set", + "vip-test", + "api", + "tcp/443", + "--port-check", + "off" + ]) + .status + .success() + ); + let shown = agent.run(&["vip", "show", "vip-test", "api", "--json"]); + assert!(shown.status.success()); + let shown: tsunagi::vip::VipStatus = serde_json::from_slice(&shown.stdout).unwrap(); + assert_eq!(shown.config.listeners.len(), 2); + assert!( + !shown + .config + .listeners + .iter() + .find(|l| l.protocol == tsunagi::vip::Protocol::Tcp) + .unwrap() + .port_check + ); + assert_eq!(shown.config.listeners[0].backends[0].port, 6443); + let removed = agent.run(&["vip", "remove", "vip-test", "api"]); + assert!(removed.status.success()); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(reply) = query("127.0.0.1:15367".parse().unwrap(), "api.vip-test", TYPE::A) + && !has_answer(&reply) + { + break; + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(25)); + } +} diff --git a/docs/protocol.md b/docs/protocol.md index cabbf5b..6f283aa 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -9,24 +9,26 @@ Two versions exist and are independent: - **Identity scheme**, `tsunagi-network-id-v1`. Frozen. Changing it creates a different network space for the same name and secret. -- **Control protocol**, ALPN `tsunagi/ctrl/7`, `PROTOCOL_VERSION = 7`. +- **Control protocol**, ALPN `tsunagi/ctrl/8`, `PROTOCOL_VERSION = 8`. Upgrading the crate or bumping the control protocol must never change an existing `NetworkId`. -Control version 7 adds signed offline-node retirement messages to version 6's -private AWG transport negotiation and delivery checks. Data ALPN remains -`tsunagi/data/5`. All members must upgrade together; network ids, handshake -encoding and existing record signatures are unchanged. Local CLI/tray IPC is -version 21, so restart the agent with matching clients. SQLite schema 9 adds -retirement storage while preserving existing identities, settings and records. -See [sync-model.md](sync-model.md#offline-node-cleanup) and [amnezia.md](amnezia.md). +Control version 8 adds signed, multi-writer VIP field snapshots to version 7's +node retirements. Data ALPN remains `tsunagi/data/5`. All members must upgrade +together. Network ids, derivation labels, handshake encoding and existing record +signatures are unchanged. Local CLI/tray IPC is version 22; restart the agent +with matching clients. SQLite schema 10 adds `vip_fields`, preserving identity, +network configuration, signed node records and retirement proofs. +See [virtual services](vip.md), [state sync](sync-model.md) and [AWG](amnezia.md). ## Canonical encoding -Everything that is hashed or MAC'd uses length-prefixed fields, written here as -`LP(x) = u32_be(len(x)) || x`. String concatenation is never used, so no two -different field splits can produce the same bytes. +Network identity derivation and the membership transcript use length-prefixed +fields, written here as `LP(x) = u32_be(len(x)) || x`. String concatenation is +never used, so different field splits cannot produce the same bytes. VIP field +signatures use their separate domain and canonical postcard encoding described +in [virtual services](vip.md#shared-state-and-addresses). ## Network space identity diff --git a/docs/routing.md b/docs/routing.md index 917069b..0f71c33 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -124,5 +124,5 @@ counts sends without storing frames. Results are CPU forwarding cost, **not** end-to-end network latency or VPN throughput; encryption, fragmentation, sockets, congestion and scheduling contribute separately. -Wire compatibility: control ALPN `tsunagi/ctrl/7`, data ALPN `tsunagi/data/5`. +Wire compatibility: control ALPN `tsunagi/ctrl/8`, data ALPN `tsunagi/data/5`. Upgrade every participant together; saved identities and network state persist. diff --git a/docs/testing.md b/docs/testing.md index 9f7f923..2212d35 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -228,3 +228,18 @@ scroll area. They check stretching and shrinking without an expanding first column, separate handshake/control RTT values, unknown measurements and short end-to-end hints that do not substitute the carrier protocol. This does not establish native desktop rendering on every platform. + + +VIP tests cover signed multi-writer backend merging, stale draft rejection, +permanent deletion and replay, separate active/deleted capacity, and mandatory +SQLite migration/counter rollback. Real agent instances verify equal editing +rights, network isolation, restart and recovery without the creator. Packet +checks exercise RTT selection, port translation, UDP failover, a local backend, +fragment reassembly/checksums and ICMP quoted-header/PMTU translation. TCP check +recovery uses real loopback listening sockets. The WireGuard suite moves VIP +TCP/UDP packets through real encrypted tunnels, rejects forged VIP sources and +checks UDP failover after the selected agent stops. CLI/DNS tests run the real +binary, configure independent listeners, disable checks, reject an invalid +change atomically and observe DNS creation/deletion. GUI tests cover draft +isolation and form rendering. These do not establish live host TUN behavior or +native desktop behavior on Linux, macOS or Windows. See [VIP semantics](vip.md). diff --git a/docs/vip.md b/docs/vip.md new file mode 100644 index 0000000..f377aec --- /dev/null +++ b/docs/vip.md @@ -0,0 +1,171 @@ +# Named virtual services + +A VIP is a shared name such as `api.home`, with independent TCP/UDP listeners. +Each listener maps its port to a set of node identities and application ports. +The application uses the name; the automatically allocated IPv4 address is an +internal reservation. The current overlay carries IPv4 only. + +## CLI and tray + +```sh +tsng vip list home +tsng vip create home api +tsng vip port add home api tcp/443 --backend server1:6443 --backend server2:7443 +tsng vip port add home api udp/53 --backend server1:5353 +tsng vip backend add home api tcp/443 server3:6443 +tsng vip backend remove home api tcp/443 server2:7443 +tsng vip port set home api tcp/443 --port-check off +tsng vip port set home api tcp/443 --check-interval-ms 3000 --check-timeout-ms 2000 --failures 2 --successes 1 --idle-timeout 7200 +tsng vip show home api +tsng vip status home api --watch +tsng vip edit home api +tsng vip port remove home api udp/53 +tsng vip remove home api +``` + +`list` without a network lists all configured networks. Network and VIP +selectors accept exact names, full ids and unique id prefixes; backend node +selectors also accept stored node names. Ambiguous selections change nothing. +Names resolve to identities when saved, so a renamed node remains a backend. +`--json` provides configuration, revision, counters and observations; +`list`/`watch` emit one JSON object per service/sample. `edit` +opens JSON in `VISUAL`/`EDITOR` and submits one complete version-checked draft. +An invalid or rejected edited file is retained with its path reported. + +Each mutating command is atomic. A listener can be added with its full initial +backend set, and `edit` can replace the entire configuration at once. Empty +VIPs/listeners are valid and resolve, but cannot accept traffic. Configuration +updates currently require the network to be active locally. Saved configuration remains visible while a +network is stopped. + +The tray's network tile has a **VIP** button. Its separate window edits names, +listeners, backend nodes/ports, check settings and flow timeouts. All edits, +including deletion, remain drafts until **Save**. Status updates do not +replace a dirty draft. A save conflict retains the draft; **Discard / reload** +loads the latest configuration. Closing a dirty window offers explicit discard +or continued editing. Addresses and ids are available in a collapsed detail. + +## Delivery and recovery + +The client agent selects the eligible backend with the lowest smoothed full +round-trip latency through the actual encrypted data plane, including relays. +Unknown or stale RTT is not treated as zero. Local backends have zero path RTT. +A new flow is pinned to its selected backend; a faster RTT does not move it. +This can concentrate new traffic on the closest node. + +At local ingress, NAT replaces destination address/port and reserves an OS +source port for an unambiguous reverse mapping. Return traffic passes normal +network/peer source and destination admission **before** reverse NAT restores +the virtual source address/port. Plugins and encrypted transit stay unchanged. +A peer never gains permission to claim a VIP source. Fragment assembly is +bounded, checksums are updated, packets are fragmented back to the overlay MTU, +and ICMP errors have their quoted NAT headers restored. Fragment caches are +scoped to network and authenticated sender, with idle expiry independent of +further traffic. + +For a backend on the client itself, the request is injected into the same TUN +with the nonlocal VIP as source, and the host reply is reverse-translated at +local ingress. The local backend therefore sees the VIP as source; remote +backends see the client's ordinary overlay address. Firewalls and application +bind addresses remain the administrator's responsibility. + +Removing a backend/listener/VIP prevents new bindings while existing bindings +continue until idle expiry. Established TCP connections are not migrated. A retransmitted empty SYN may +choose another backend before a SYN-ACK has arrived; SYNs carrying application +data are not retargeted. A new SYN sequence creates a new binding even if the +client reuses its address/port tuple. TCP RST shortens binding retention. An unavailable UDP backend is replaced for subsequent +datagrams; already sent datagrams are never replayed. Stateful UDP applications +may need to re-establish their application session. Protocols that embed IP +addresses or negotiate additional ports require their own configuration; no +application payloads are parsed or rewritten. + +Idle expiry defaults to 7200 seconds for TCP and 120 seconds for UDP. Shadow +ports are quarantined for 120 seconds after retirement to reject late replies +and avoid immediate reuse. Active and quarantined bindings share a bounded +capacity. State and sockets are released on network stop or agent shutdown. +A restart drops bindings; shared service configuration remains intact. + +## Health + +One probe exchange per second measures each used peer through the encrypted +IP tunnel. The EWMA uses 7/8 of the previous sample and 1/8 of the new one. +A probe expires after three seconds; two consecutive missing responses or a +missing data path exclude the peer. Measurements expire after ten seconds. +Probes continue during failure so recovery is automatic. + +TCP connect checks default to on. Defaults are a three-second interval, +two-second timeout, two failures to exclude and one success to recover. +Checks are shared for the same network/node/address/TCP-port, run off the +packet path with at most 32 concurrent connects, and are scheduled fairly. +A 250ms worker tick services timers. Connect checks send no application data; +they confirm port acceptance and may appear in server logs. They cannot prove +application readiness after a successful handshake. Disabling port checks +retains data-plane probes. + +UDP uses encrypted path checks and explicit ICMP port errors. No arbitrary +probe datagrams are injected into an unknown application protocol. Silence +cannot prove failure of an arbitrary UDP application. An explicit unreachable +port is excluded for ten seconds and then becomes eligible to retry. + +## Shared state and addresses + +Any authenticated member can create, edit or remove any VIP. Each field is a +self-contained signed register: network, service id, field key, optional encoded +value, Lamport clock, author, durable author counter and signature. Signing uses +the separate `tsunagi-vip-field/v1\0` domain followed by postcard encoding of +those fields with the signature vector empty. The encoding/order is a wire +contract; the identity scheme and handshake transcript are unchanged. + +Concurrent edits to different backend fields merge. A conflict on one field +uses `(Lamport clock, author id, canonical bytes)` ordering, independent of wall +time and arrival order. Listener generations prevent backend fields from an +old removed listener reappearing when the port is recreated. Service deletion +is a permanent signed tombstone for its id; recreation gets a fresh id. +Missing snapshot fields are never deletion. + +Saving validates the entire draft and checks the revision shown by the editor. +The merged snapshot and author's counter commit together in SQLite before local +routing changes or publication. Receivers verify and persist complete service +snapshots before exposing them. Periodic rotating snapshots repair queue loss +and returning replicas without requiring the creator online. Concurrent edits +made before participants have seen each other can still merge after a save. +A merged configuration exceeding listener/backend limits is reported as an +error and excluded from routing rather than selected according to arrival order. + +VIP allocation walks the agreed overlay range from a service-id-derived seed, +excluding known node claims and other allocated VIPs. Nodes avoid known VIP +reservations when choosing a new address. Names colliding with node names or +another service are reported; node names take precedence and service ties use +id order. A partition can temporarily have incomplete state; after merge an +address or conflicting name's effective holder can change. Use DNS names. +DNS TTL is 30 seconds; DNS does not migrate established connections or force +applications to discard cached addresses. Existing NAT bindings retain their +original VIP address while draining. + +`Limits` bounds active services (64 by default), permanent deletion markers +(1024), signed fields per active service (192), listeners (16), backends per +listener (32), active TCP checks (32), flow/port reservations (4096) and incomplete +IPv4 datagrams (64). A deleted service retains only its signed permanent +marker; obsolete fields can be dropped because replay cannot override deletion. +A full deletion history is an explicit error; protection is never silently +evicted. Deleted services do not consume the active-service limit. Whole service +snapshots must also fit the configured control frame limit. Corrupt mandatory VIP storage +is a hard error on activation, never reset as a cache. + +## Observations and verification + +Status is this client's observation. Per VIP/listener/backend it reports the +next selected backend, health reason, RTT and sample age, active bindings, +user IP packet/byte counters, drops and UDP failovers. Probe bytes do not enter +user traffic counters. The GUI and CLI watch derive rates from cumulative +counters; counters reset on restart. These are IP forwarding counters, not +application acknowledgements or guaranteed delivered-byte totals. + +Automated tests cover field merging, stale drafts, signed deletion, tampering, +network isolation, SQLite migration and transactional rollback, real port +connect failure/recovery, RTT selection, UDP failover, local hairpin NAT, +fragmentation/checksums, real encrypted TCP/UDP packet delivery and source +admission, real CLI/DNS wiring and draft form rendering. In-memory TUN tests +exercise packets, not an operating system TCP stack. Native desktop behavior, +physical interfaces and cross-platform kernel/firewall behavior need live-host +validation; automated tests do not establish those.