diff --git a/README.md b/README.md index 5168766..dd0aac0 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,24 @@ a common protocol at the same wire version. `tsng up --protocol` supplies the initial defaults for networks without a saved choice; it does not restrict which built-in protocols can be enabled later. +Remove an obsolete offline device from a network's shared history: + +```sh +tsng network forget-node home old-laptop +tsng network forget-node home 10.13.37.143 +``` + +Networks accept exact names, full ids or unique id prefixes; nodes accept those +id selectors, exact hostnames or overlay IPv4 addresses. Ambiguous selectors +change nothing. Start the network first if it is stopped. In **Show devices**, +click **Forget** beside an offline node and confirm its identity. + +Every member has the same cleanup right. The signed cleanup frees the observed +claim's name and IPv4 reservation, survives restarts and reaches offline replicas +when they reconnect. It does not ban the device: a newer signed claim can return. +Online nodes and the currently selected exit node cannot be forgotten. +[State and limits](docs/sync-model.md#offline-node-cleanup). + ## Platforms | Platform | Build | CLI `tsng` | Tray | Exit node: offer / use | Packages | diff --git a/crates/tsunagi-core/src/agent/mod.rs b/crates/tsunagi-core/src/agent/mod.rs index 1a7e060..6b83b05 100644 --- a/crates/tsunagi-core/src/agent/mod.rs +++ b/crates/tsunagi-core/src/agent/mod.rs @@ -72,6 +72,17 @@ pub struct ConfiguredNetwork { pub active: bool, } +/// A durable version-scoped cleanup, not an identity ban. +#[derive(Debug, Clone)] +pub struct ForgetNodeOutcome { + /// Identity whose observed claim was retired. + pub target: EndpointId, + /// The retired claim version; future signed claims remain admissible. + pub record_version: u64, + /// Authenticated peers to which cleanup was queued immediately. + pub peers_told: usize, +} + /// A running agent. /// /// Cloning gives another handle to the same agent. @@ -298,6 +309,27 @@ impl TransportContext for TransportCtx { } impl Agent { + /// Retires one offline member's observed claim without revoking membership. + /// Every member has this right. The version must still match the selection. + pub async fn forget_node( + &self, + network: NetworkId, + peer: EndpointId, + expected_version: u64, + ) -> Result { + let _lifecycle = self.inner.lifecycle.lock().await; + let (reply, receive) = oneshot::channel(); + self.command( + network, + NetCommand::ForgetNode { + peer, + expected_version, + reply, + }, + ) + .await?; + receive.await.map_err(|_| Error::Stopped)? + } fn effective_protocols(&self, saved: Option<&Vec>) -> Vec { saved .or(self.inner.config.default_protocols.as_ref()) @@ -878,6 +910,12 @@ impl Agent { .await? .into_iter() .find(|stored| stored.network_id == network_id); + // Mandatory cleanup proofs must never be silently dropped on startup. + let retirements = self + .inner + .storage + .retirements(network_id, self.inner.limits.max_retirements) + .await?; let range = self.desired_range(network_id).await?; if let Some(range) = range { let conflicts = self.inner.routes.overlapping_networks(network_id, range); @@ -930,6 +968,7 @@ impl Agent { .and_then(|stored| stored.exit_via) .and_then(|bytes| EndpointId::from_bytes(&bytes).ok()); let handle = network::spawn(RuntimeParams { + retirements, broadcast, exit_offer, exit_via, diff --git a/crates/tsunagi-core/src/agent/network.rs b/crates/tsunagi-core/src/agent/network.rs index de7b04c..e39cf08 100644 --- a/crates/tsunagi-core/src/agent/network.rs +++ b/crates/tsunagi-core/src/agent/network.rs @@ -26,7 +26,9 @@ use crate::net::{EndpointAdapter, PathAddr, snapshot_connection}; use crate::proto::handshake::{self, HandshakeOutcome, Role}; use crate::proto::message::{Announcement, ControlMessage, Envelope, encode, kind}; use crate::state::allocator::allocate; -use crate::state::{Ipv4Range, Merged, RecordBody, SignedRecord, StateSet}; +use crate::state::{ + Ipv4Range, MAX_RETIREMENT_LEN, Merged, RecordBody, SignedRecord, SignedRetirement, StateSet, +}; use crate::storage::Storage; use super::events::Event; @@ -48,6 +50,11 @@ pub(crate) struct InboundSession { /// Commands accepted by a network runtime. pub(crate) enum NetCommand { + ForgetNode { + peer: EndpointId, + expected_version: u64, + reply: oneshot::Sender>, + }, Inbound(Box), Send { peer: EndpointId, @@ -118,6 +125,7 @@ pub(crate) enum NetCommand { impl std::fmt::Debug for NetCommand { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + NetCommand::ForgetNode { peer, .. } => write!(f, "ForgetNode({peer})"), NetCommand::Inbound(_) => f.write_str("Inbound"), NetCommand::InboundLink(link) => write!(f, "InboundLink({})", link.protocol), NetCommand::Send { peer, message, .. } => { @@ -162,6 +170,7 @@ impl NetworkHandle { /// Everything a network runtime needs to run. pub(crate) struct RuntimeParams { + pub(crate) retirements: Vec, pub(crate) broadcast: bool, /// This agent offers itself as an exit node here. pub(crate) exit_offer: bool, @@ -366,6 +375,9 @@ struct Runtime { state: StateSet, /// Snapshots received while dispatching, handled on the next loop pass. pending_state: Vec<(EndpointId, Vec)>, + pending_retirements: Vec<(EndpointId, Vec)>, + /// Next cleanup batch per live session; prevents queue pressure starving a suffix. + retirement_cursor: HashMap, /// The highest version this agent has ever published for this network. own_version: u64, /// Whether the exit node in use was reachable when last looked at, so @@ -448,6 +460,8 @@ impl Runtime { reported_name_taken: None, state: StateSet::new(), pending_state: Vec::new(), + pending_retirements: Vec::new(), + retirement_cursor: HashMap::new(), own_version: 0, exit_via_was_online: false, announced_exit: false, @@ -547,6 +561,14 @@ impl Runtime { async fn handle_command(&mut self, command: NetCommand) { match command { + NetCommand::ForgetNode { + peer, + expected_version, + reply, + } => { + let result = self.forget_node(peer, expected_version).await; + let _ = reply.send(result); + } NetCommand::Inbound(inbound) => { self.install_session(*inbound).await; } @@ -801,6 +823,7 @@ impl Runtime { // the check is a comparison when nothing has changed. self.ensure_own_claim().await; self.report_hostname_taken(); + self.broadcast_retirements(); // The same timer repairs the relay picture: reachability is soft // state, repeated while it holds and forgotten when it stops. self.update_paths(); @@ -814,7 +837,7 @@ impl Runtime { // still dialable when the endpoint's own discovery can resolve // one. This is what makes a network a mesh rather than a star // around whoever was named on a command line. - for record in self.state.records() { + for record in self.state.active_records() { if let Ok(author) = record.author_id() && author != self.local_id { @@ -831,6 +854,9 @@ impl Runtime { let Ok(endpoint_id) = EndpointId::from_bytes(&hint.endpoint_id) else { continue; }; + if self.state.is_retired(&endpoint_id) { + continue; + } let Some(addr) = decode_hint(endpoint_id, &hint.addr) else { continue; }; @@ -1036,6 +1062,22 @@ impl Runtime { /// boundary, because a restored backup or a copied file could hold /// anything. async fn load_state(&mut self) { + for marker in std::mem::take(&mut self.params.retirements) { + // Storage verified these proofs and activation enforced capacity. + if let Err(err) = self.state.merge_retirement( + self.network_id, + marker, + self.params.limits.max_retirements, + ) { + self.emit(Event::PluginError { + network: self.network_id, + protocol: "state".into(), + reason: err.to_string(), + }); + self.shutdown.trigger(); + return; + } + } let stored = self .params .storage @@ -1051,7 +1093,8 @@ impl Runtime { .storage .own_record_version(self.network_id, self.local_id) .await - .unwrap_or(0); + .unwrap_or(0) + .max(self.state.authored_version(&self.local_id)); for (id, hostname) in self .params .storage @@ -1163,6 +1206,7 @@ impl Runtime { let kept = self .state .get(&self.local_id) + .filter(|_| !self.state.is_retired(&self.local_id)) .filter(|record| record.body.hostname() == Some(name.as_str())) .and_then(|record| record.body.hostname_since()); kept.unwrap_or_else(unix_millis) @@ -1195,7 +1239,7 @@ impl Runtime { if self .state .get(&self.local_id) - .is_some_and(|record| record.body == body) + .is_some_and(|record| record.body == body && !self.state.is_retired(&self.local_id)) { return; } @@ -1203,13 +1247,187 @@ impl Runtime { self.publish_record(body).await; } + async fn forget_node( + &mut self, + peer: EndpointId, + expected_version: u64, + ) -> Result { + if peer == self.local_id { + return Err(Error::ForgetNode( + "use network leave for this device".into(), + )); + } + if self.is_online(peer) { + return Err(Error::ForgetNode( + "the node is online; only offline history can be forgotten".into(), + )); + } + if self.params.exit_via == Some(peer) { + return Err(Error::ForgetNode( + "stop using this exit node before forgetting it".into(), + )); + } + let target = self + .state + .get(&peer) + .filter(|_| !self.state.is_retired(&peer)) + .filter(|record| matches!(record.body, RecordBody::Claim { .. })) + .cloned() + .ok_or_else(|| Error::ForgetNode("no current signed claim for this node".into()))?; + if target.version != expected_version { + return Err(Error::ForgetNode( + "the node's claim changed; refresh and select it again".into(), + )); + } + let version = self + .own_version + .max(self.state.authored_version(&self.local_id)) + .checked_add(1) + .filter(|v| *v <= i64::MAX as u64) + .ok_or_else(|| Error::ForgetNode("author counter exhausted".into()))?; + let marker = + SignedRetirement::sign(&self.params.device_secret, self.network_id, version, target); + let mut candidate = self.state.clone(); + candidate + .merge_retirement( + self.network_id, + marker.clone(), + self.params.limits.max_retirements, + ) + .map_err(|err| Error::ForgetNode(err.to_string()))?; + // Signed cleanup and our author counter commit together before visibility. + self.params + .storage + .publish_own_retirement(marker.clone()) + .await?; + self.own_version = version; + self.state = candidate; + self.known_hostnames.remove(&peer); + self.dial_states.remove(&peer); + self.candidate_addrs.remove(&peer); + self.publish_allocations().await; + self.report_hostname_taken(); + let message = ControlMessage::Retirements { + markers: vec![marker], + }; + let peers: Vec<_> = self.sessions.keys().copied().collect(); + let peers_told = peers + .into_iter() + .filter(|peer| self.send_to(*peer, message.clone()).is_ok()) + .count(); + Ok(super::ForgetNodeOutcome { + target: peer, + record_version: expected_version, + peers_told, + }) + } + + fn broadcast_retirements(&mut self) -> usize { + let markers = self.state.retirements(); + if markers.is_empty() { + return 0; + } + // Each proof has an encoded bound; reserve envelope/list overhead and + // chunk every marker, so a low-id prefix cannot starve later cleanup. + let batch = (self.params.limits.max_frame_len.saturating_sub(128) / MAX_RETIREMENT_LEN) + .max(1) + .min(self.params.limits.max_state_records.max(1)); + let mut delivered = HashSet::new(); + let peers: Vec<_> = self.sessions.keys().copied().collect(); + let chunks: Vec<_> = markers.chunks(batch).collect(); + for peer in peers { + let start = self.retirement_cursor.get(&peer).copied().unwrap_or(0) % chunks.len(); + for offset in 0..chunks.len() { + let index = (start + offset) % chunks.len(); + let message = ControlMessage::Retirements { + markers: chunks[index].to_vec(), + }; + if self.send_to(peer, message).is_err() { + self.retirement_cursor.insert(peer, index); + break; + } + delivered.insert(peer); + self.retirement_cursor + .insert(peer, (index + 1) % chunks.len()); + } + } + delivered.len() + } + + async fn receive_retirements(&mut self, peer: EndpointId, markers: Vec) { + let mut changed = false; + for marker in markers { + let mut candidate = self.state.clone(); + match candidate.merge_retirement( + self.network_id, + marker.clone(), + self.params.limits.max_retirements, + ) { + Ok(false) => continue, + Ok(true) => {} + Err(err) => { + self.metrics.protocol_violations += 1; + self.emit(Event::ProtocolViolation { + network: Some(self.network_id), + peer: Some(peer), + reason: format!("unusable cleanup proof: {err}"), + }); + continue; + } + } + let target = marker.target.author_id().ok(); + if let Err(err) = self.params.storage.put_retirement(marker).await { + self.emit(Event::PluginError { + network: self.network_id, + protocol: "state".into(), + reason: format!("cannot persist cleanup proof: {err}"), + }); + continue; + } + self.state = candidate; + if let Some(target) = target + && self.state.is_retired(&target) + && !self.sessions.contains_key(&target) + { + self.known_hostnames.remove(&target); + self.dial_states.remove(&target); + self.candidate_addrs.remove(&target); + } + changed = true; + } + if changed { + // A live target answers cleanup with a fresh claim; its identity + // is still authenticated, and old hostname priority is not reused. + self.ensure_own_claim().await; + self.publish_allocations().await; + self.broadcast_retirements(); + } + } + /// Signs, stores and announces one of this agent's own records. /// /// Stored before it is announced, in one transaction with the version /// counter, so a crash can never let us reuse a version we already put on /// the wire. async fn publish_record(&mut self, body: RecordBody) { - let version = self.own_version.saturating_add(1); + let observed = self.state.authored_version(&self.local_id).max( + self.state + .retirement_of(&self.local_id) + .map_or(0, |m| m.target.version), + ); + let Some(version) = self + .own_version + .max(observed) + .checked_add(1) + .filter(|v| *v <= i64::MAX as u64) + else { + self.emit(Event::PluginError { + network: self.network_id, + protocol: "state".into(), + reason: "author counter exhausted".into(), + }); + return; + }; let record = SignedRecord::sign(&self.params.device_secret, self.network_id, version, body); if let Err(err) = self.params.storage.publish_own_record(record.clone()).await { @@ -1234,6 +1452,9 @@ impl Runtime { /// Handles snapshots collected while dispatching messages. async fn drain_pending_state(&mut self) { + for (peer, markers) in std::mem::take(&mut self.pending_retirements) { + self.receive_retirements(peer, markers).await; + } for (peer, records) in std::mem::take(&mut self.pending_state) { self.receive_state(peer, records).await; } @@ -1242,6 +1463,7 @@ impl Runtime { /// Sends everything we know to every peer. fn broadcast_state(&mut self) { + self.broadcast_retirements(); let mut records = self.state.records(); records.truncate(self.params.limits.max_state_records); if records.is_empty() { @@ -2071,6 +2293,7 @@ impl Runtime { /// Drops every link to a peer. fn drop_links_for(&mut self, peer: EndpointId) { + self.retirement_cursor.remove(&peer); self.reset_peer_data(peer, "peer session ended"); self.hub.remove_peer(peer); self.reachable.remove(&peer); @@ -2434,6 +2657,9 @@ impl Runtime { ControlMessage::State { records } => { self.pending_state.push((peer, records.clone())); } + ControlMessage::Retirements { markers } => { + self.pending_retirements.push((peer, markers.clone())); + } // Members somebody else knows of. Candidates, nothing more: // each still has to pass the handshake, and each is tried the // same way as one from any other source. @@ -2631,12 +2857,13 @@ impl Runtime { // listing it as one turns leaving into a peer that looks broken. let mut members: Vec = self .state - .records() + .active_records() .into_iter() .filter(|record| !matches!(record.body, RecordBody::Release)) .filter_map(|record| { let endpoint_id = record.author_id().ok()?; Some(MemberStatus { + record_version: record.version, endpoint_id, // Only uncontested claims are reported as held: two // members may have claimed the same thing, and saying diff --git a/crates/tsunagi-core/src/agent/status.rs b/crates/tsunagi-core/src/agent/status.rs index f99263b..8c52d3e 100644 --- a/crates/tsunagi-core/src/agent/status.rs +++ b/crates/tsunagi-core/src/agent/status.rs @@ -108,6 +108,8 @@ pub struct HostnameConflict { /// connected. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemberStatus { + /// Current signed claim version, used to bind cleanup confirmation. + pub record_version: u64, /// The member's device identity. pub endpoint_id: EndpointId, /// The IPv4 overlay address it claimed and signed for. diff --git a/crates/tsunagi-core/src/config.rs b/crates/tsunagi-core/src/config.rs index 3b56f78..0ed1afe 100644 --- a/crates/tsunagi-core/src/config.rs +++ b/crates/tsunagi-core/src/config.rs @@ -174,6 +174,8 @@ pub struct Limits { pub max_reason_len: usize, /// Largest number of signed records accepted in one snapshot. pub max_state_records: usize, + /// Maximum durable retirement markers retained per network. + pub max_retirements: usize, /// Deadline for the whole handshake. pub handshake_timeout: Duration, /// Deadline for one outbound dial attempt. @@ -211,6 +213,7 @@ impl Default for Limits { max_echo_payload_len: 4 * 1024, max_reason_len: 256, max_state_records: crate::state::MAX_RECORDS_PER_MESSAGE, + max_retirements: 1024, handshake_timeout: Duration::from_secs(10), dial_timeout: Duration::from_secs(10), write_timeout: Duration::from_secs(30), diff --git a/crates/tsunagi-core/src/error.rs b/crates/tsunagi-core/src/error.rs index b52b34a..5a13a60 100644 --- a/crates/tsunagi-core/src/error.rs +++ b/crates/tsunagi-core/src/error.rs @@ -15,6 +15,9 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { + /// A node-history cleanup is unsafe, stale or cannot be performed. + #[error("cannot forget node: {0}")] + ForgetNode(String), /// An ordered local data-plane protocol selection is invalid. #[error("invalid protocol selection: {0}")] InvalidProtocols(String), diff --git a/crates/tsunagi-core/src/ipc/mod.rs b/crates/tsunagi-core/src/ipc/mod.rs index 879b653..a846c7f 100644 --- a/crates/tsunagi-core/src/ipc/mod.rs +++ b/crates/tsunagi-core/src/ipc/mod.rs @@ -171,6 +171,15 @@ pub fn default_control_socket_path() -> PathBuf { #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum Request { + /// Retire the selected offline member's observed claim. + ForgetNode { + /// Exact public network id. + network_id: String, + /// Exact device identity selected by the client. + peer: String, + /// Claim version displayed when the user selected the node. + expected_version: u64, + }, /// Replace one network's ordered local data-plane selection. SetProtocols { /// Public network identifier. @@ -262,6 +271,14 @@ pub enum Request { impl std::fmt::Debug for Request { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Request::ForgetNode { + network_id, + peer, + expected_version, + } => write!( + f, + "ForgetNode {{ {network_id}, peer: {peer}, version: {expected_version} }}" + ), Request::SetProtocols { network_id, protocols, @@ -300,6 +317,8 @@ impl std::fmt::Debug for Request { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum Response { + /// An observed offline claim was durably retired. + NodeForgotten(ForgottenNodeReport), /// Accepted ordered protocol selection. Protocols(Vec), /// A status report. @@ -328,6 +347,141 @@ pub enum Response { Error(String), } +/// The durable result of an offline-node cleanup. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ForgottenNodeReport { + /// Selected device identity. + pub peer: String, + /// The retired claim version, not a future-version ban. + pub record_version: u64, + /// Peers to which cleanup was immediately queued. + pub peers_told: u32, +} + +/// Resolves a node by exact hostname, full id, unique id prefix or overlay IPv4. +/// Ambiguity is reported with names and full ids and never guessed. +pub fn resolve_node<'a>( + network: &'a NetworkReport, + selector: &str, +) -> std::result::Result<&'a MemberReport, String> { + let selector = selector.trim(); + if selector.is_empty() { + return Err("select a node by name, id or IPv4 address".into()); + } + if let Some(member) = network.members.iter().find(|m| m.endpoint_id == selector) { + return Ok(member); + } + let prefix = selector.trim_end_matches('…'); + let matches: Vec<_> = network + .members + .iter() + .filter(|m| { + m.hostname.as_deref() == Some(selector) + || m.overlay_address_v4.as_deref() == Some(selector) + || (!prefix.is_empty() && m.endpoint_id.starts_with(prefix)) + }) + .collect(); + match matches.as_slice() { + [one] => Ok(one), + [] => Err(format!( + "no signed node matches `{selector}` in `{}`", + network.name + )), + many => Err(format!( + "`{selector}` is ambiguous; use a full id or unique id prefix:\n{}", + many.iter() + .map(|m| format!( + " {} {} {}", + m.hostname.as_deref().unwrap_or("unnamed"), + m.endpoint_id, + m.overlay_address_v4.as_deref().unwrap_or("no IPv4") + )) + .collect::>() + .join("\n") + )), + } +} + +/// Retires an exact selected claim; an online or changed target is rejected. +pub async fn forget_node( + socket: &Path, + network: &str, + peer: &str, + expected_version: u64, +) -> Result { + let request = Request::ForgetNode { + network_id: network.into(), + peer: peer.into(), + expected_version, + }; + match exchange(socket, &request, EXCHANGE_TIMEOUT).await? { + Response::NodeForgotten(report) => Ok(report), + Response::Error(reason) => Err(Error::ForgetNode(reason)), + other => Err(Error::Storage(format!("unexpected response: {other:?}"))), + } +} + +#[cfg(test)] +mod node_selector_tests { + use super::*; + + #[test] + fn exact_names_addresses_and_unique_prefixes_select_the_same_signed_node() { + let node = MemberReport { + endpoint_id: "abcdef01".into(), + hostname: Some("laptop".into()), + overlay_address_v4: Some("10.2.0.3".into()), + record_version: 42, + ..Default::default() + }; + let network = NetworkReport { + name: "home".into(), + members: vec![node.clone()], + ..Default::default() + }; + for selector in ["laptop", "10.2.0.3", "abcdef01", "abc…"] { + assert_eq!(resolve_node(&network, selector), Ok(&node)); + } + for selector in ["", "LAPTOP", "missing"] { + assert!(resolve_node(&network, selector).is_err()); + } + } + + #[test] + fn collisions_between_names_addresses_and_prefixes_require_an_explicit_identity() { + let first = MemberReport { + endpoint_id: "abcdef01".into(), + hostname: Some("duplicate".into()), + overlay_address_v4: Some("10.2.0.3".into()), + ..Default::default() + }; + let second = MemberReport { + endpoint_id: "abcdef02".into(), + hostname: Some("duplicate".into()), + overlay_address_v4: Some("10.2.0.3".into()), + ..Default::default() + }; + let third = MemberReport { + endpoint_id: "12345678".into(), + hostname: Some("abc".into()), + ..Default::default() + }; + let network = NetworkReport { + name: "home".into(), + members: vec![first.clone(), second, third], + ..Default::default() + }; + for selector in ["duplicate", "10.2.0.3", "abc"] { + let result = resolve_node(&network, selector); + assert!(result.is_err_and(|error| error.contains("ambiguous") + && error.contains("abcdef01") + && error.contains("abcdef02") + && error.contains("duplicate"))); + } + assert_eq!(resolve_node(&network, "abcdef01"), Ok(&first)); + } +} + /// What happened when a network was stopped or started. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ActiveReport { @@ -580,6 +734,8 @@ pub struct HostnameConflictReport { /// One member of the network, from signed state. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct MemberReport { + /// Signed claim version used to reject stale cleanup requests. + pub record_version: u64, /// The member's device identity. pub endpoint_id: String, /// The IPv4 overlay address it claimed and signed for. @@ -749,6 +905,15 @@ impl OverlayPeerReport { /// running and what they can report or change. That is what keeps this module /// free of any knowledge of them. pub trait ReportSource: Send + Sync + 'static { + /// Persists and announces a version-scoped node cleanup. + fn forget_node( + &self, + _network: String, + _peer: String, + _expected_version: u64, + ) -> BoxFuture<'_, std::result::Result> { + Box::pin(async { Err("this agent cannot forget node history".into()) }) + } /// Saves and applies a local ordered protocol set for one network. fn set_protocols( &self, @@ -886,7 +1051,7 @@ pub const EXCHANGE_TIMEOUT: Duration = Duration::from_secs(5); /// /// Bump it whenever [`Request`], [`Response`] or anything they contain /// changes shape. -pub const CONTROL_PROTOCOL: u32 = u32::from_be_bytes([b'T', b'S', b'N', 20]); +pub const CONTROL_PROTOCOL: u32 = u32::from_be_bytes([b'T', b'S', b'N', 21]); /// Reads one request off an accepted stream, answers it, writes the response. /// @@ -910,6 +1075,14 @@ where } }; let response = match request { + Request::ForgetNode { + network_id, + peer, + expected_version, + } => match source.forget_node(network_id, peer, expected_version).await { + Ok(report) => Response::NodeForgotten(report), + Err(reason) => Response::Error(reason), + }, Request::SetProtocols { network_id, protocols, diff --git a/crates/tsunagi-core/src/proto/message.rs b/crates/tsunagi-core/src/proto/message.rs index a6acc2b..bc590fa 100644 --- a/crates/tsunagi-core/src/proto/message.rs +++ b/crates/tsunagi-core/src/proto/message.rs @@ -20,7 +20,7 @@ use crate::error::ProtocolError; /// The version in the ALPN is the wire-compatibility version of the control /// protocol. It is independent of the network identity scheme version, so /// bumping it must not change any existing [`crate::NetworkId`]. -pub const ALPN: &[u8] = b"tsunagi/ctrl/6"; +pub const ALPN: &[u8] = b"tsunagi/ctrl/7"; /// ALPN of the tsunagi data plane. /// @@ -43,7 +43,7 @@ pub const MAX_DATA_PROTOCOL_LEN: usize = 32; pub const MAX_SIGNATURE_LEN: usize = 64; /// Control protocol version carried inside the handshake. -pub const PROTOCOL_VERSION: u16 = 6; +pub const PROTOCOL_VERSION: u16 = 7; pub use crate::dataplane::transport::{MAX_TRANSPORT_PAYLOAD, TransportPayload}; @@ -181,6 +181,11 @@ pub enum ControlMessage { /// Short free-text reason. reason: String, }, + /// Signed cleanup markers, each proving the retired claim version. + Retirements { + /// Bounded by the per-message signed-state limit. + markers: Vec, + }, } /// Somewhere a member has been seen. @@ -273,6 +278,28 @@ pub fn validate_capability( /// network, the other networks or the agent. pub fn validate(message: &ControlMessage, limits: &Limits) -> Result<(), ProtocolError> { match message { + ControlMessage::Retirements { markers } => { + check_len("retirements", markers.len(), limits.max_state_records)?; + for marker in markers { + check_len( + "retirement.signature", + marker.signature.len(), + MAX_SIGNATURE_LEN, + )?; + check_len( + "retirement.target.signature", + marker.target.signature.len(), + MAX_SIGNATURE_LEN, + )?; + if let Some(name) = marker.target.body.hostname() { + check_len( + "retirement.target.hostname", + name.len(), + limits.max_hostname_len, + )?; + } + } + } ControlMessage::Transport { protocol, payload } => { if protocol.is_empty() { return Err(ProtocolError::Malformed("empty transport protocol")); @@ -345,6 +372,7 @@ pub fn kind(message: &ControlMessage) -> &'static str { ControlMessage::Ping { .. } => "ping", ControlMessage::Pong { .. } => "pong", ControlMessage::State { .. } => "state", + ControlMessage::Retirements { .. } => "retirements", ControlMessage::Reachable { .. } => "reachable", ControlMessage::Peers { .. } => "peers", ControlMessage::Bye { .. } => "bye", diff --git a/crates/tsunagi-core/src/state/mod.rs b/crates/tsunagi-core/src/state/mod.rs index 6bcdcdb..20404d8 100644 --- a/crates/tsunagi-core/src/state/mod.rs +++ b/crates/tsunagi-core/src/state/mod.rs @@ -28,16 +28,23 @@ //! replica that has the record needs nothing else to interpret it, and //! recovery never depends on replaying a chain from the beginning. //! +//! Any member may also sign a separate retirement of another author's proved +//! claim version. It suppresses stale reservations without changing that +//! author's record, and a newer claim remains valid. +//! //! # What this slice does not do yet //! -//! Compaction, revocation of a whole author, and snapshots covering more than -//! one record type. See `docs/sync-model.md` for the shape those take. +//! Compaction and permanent revocation of a whole author. See +//! `docs/sync-model.md` for the implemented cleanup semantics. pub mod allocator; use std::collections::HashMap; use std::net::Ipv4Addr; +mod retirement; +pub use retirement::{MAX_RETIREMENT_LEN, SignedRetirement}; + use iroh::{EndpointId, SecretKey, Signature}; use serde::{Deserialize, Serialize}; @@ -488,6 +495,7 @@ pub enum Merged { #[derive(Debug, Clone, Default)] pub struct StateSet { records: HashMap, + retirements: HashMap, } impl StateSet { @@ -500,6 +508,7 @@ impl StateSet { pub fn from_verified(records: impl IntoIterator) -> Self { Self { records: records.into_iter().collect(), + retirements: HashMap::new(), } } @@ -528,6 +537,52 @@ impl StateSet { .collect() } + /// Visible records, excluding versions retired by authenticated members. + pub fn active_records(&self) -> Vec { + self.records() + .into_iter() + .filter(|r| r.author_id().is_ok_and(|id| !self.is_retired(&id))) + .collect() + } + + /// Whether the current known version is at or below a proved retirement. + pub fn is_retired(&self, author: &EndpointId) -> bool { + self.retirements + .get(author) + .is_some_and(|m| self.version_of(author) <= m.target.version) + } + + /// Strongest retained cleanup proof for an identity. + pub fn retirement_of(&self, target: &EndpointId) -> Option<&SignedRetirement> { + self.retirements.get(target) + } + + /// Durable markers in target-id order, including markers surpassed by a fresh claim. + pub fn retirements(&self) -> Vec { + let mut markers: Vec<_> = self.retirements.values().cloned().collect(); + markers.sort_by_key(|m| m.target.author); + markers + } + + /// Accepts any member's signed cleanup after checking both signatures. + pub fn merge_retirement( + &mut self, + network: NetworkId, + marker: SignedRetirement, + max: usize, + ) -> Result { + let target = marker.verify(network)?; + if let Some(old) = self.retirements.get(&target) { + if !marker.supersedes(old) { + return Ok(false); + } + } else if self.retirements.len() >= max { + return Err(StateError::Malformed("retirement capacity reached")); + } + self.retirements.insert(target, marker); + Ok(true) + } + /// Merges one record, verifying it first. /// /// Merging is into the existing set, never a wholesale replacement, and an @@ -591,6 +646,9 @@ impl StateSet { pub fn address_holders(&self) -> HashMap { let mut holders: HashMap = HashMap::new(); for (author, record) in &self.records { + if self.is_retired(author) { + continue; + } let Some(address) = record.body.claimed_address() else { continue; }; @@ -618,6 +676,9 @@ impl StateSet { pub fn hostname_holders(&self) -> HashMap { let mut holders: HashMap = HashMap::new(); for (author, record) in &self.records { + if self.is_retired(author) { + continue; + } let Some(hostname) = record.body.hostname() else { continue; }; @@ -646,6 +707,9 @@ impl StateSet { /// The name an author claimed and the member that holds it instead, when /// somebody who claimed it earlier does. pub fn hostname_taken(&self, author: &EndpointId) -> Option<(&str, EndpointId)> { + if self.is_retired(author) { + return None; + } let hostname = self.records.get(author)?.body.hostname()?; let holder = *self.hostname_holders().get(hostname)?; (holder != *author).then_some((hostname, holder)) @@ -671,6 +735,12 @@ impl StateSet { pub fn agreed_range(&self) -> Option { self.records .iter() + .chain( + self.retirements + .iter() + .filter(|(author, _)| !self.records.contains_key(*author)) + .map(|(author, marker)| (author, &marker.target)), + ) .filter_map(|(author, record)| { let range = record.body.range()?; let derived = derived_ipv4_range(NetworkId::from_bytes(record.network)); @@ -684,6 +754,18 @@ impl StateSet { pub fn version_of(&self, author: &EndpointId) -> u64 { self.records.get(author).map_or(0, |record| record.version) } + + /// Highest witnessed author counter across claims and authored cleanup. + /// This differs from the claim version used to decide retirement. + pub fn authored_version(&self, author: &EndpointId) -> u64 { + self.retirements + .values() + .filter(|m| m.author == *author.as_bytes()) + .map(|m| m.version) + .chain(std::iter::once(self.version_of(author))) + .max() + .unwrap_or(0) + } } #[cfg(test)] diff --git a/crates/tsunagi-core/src/state/retirement.rs b/crates/tsunagi-core/src/state/retirement.rs new file mode 100644 index 0000000..a608bf7 --- /dev/null +++ b/crates/tsunagi-core/src/state/retirement.rs @@ -0,0 +1,219 @@ +//! A member may retire an observed claim, never another identity's future. +use iroh::{EndpointId, SecretKey, Signature}; +use serde::{Deserialize, Serialize}; + +use super::{RecordBody, SignedRecord, StateError, push_lp}; +use crate::NetworkId; + +/// Maximum encoded retirement size, including its bounded claim proof. +pub const MAX_RETIREMENT_LEN: usize = 1024; + +/// A network member's signed retirement of a proved claim version. +/// +/// The original author's signed claim proves the cutoff really existed. +/// Newer claims remain valid. This is neither a forged Release nor a ban. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SignedRetirement { + /// Device identity of the member requesting cleanup. + pub author: [u8; 32], + /// Network in which cleanup applies. + pub network: [u8; 32], + /// Requester's durable author counter, committed with this marker. + pub version: u64, + /// The target's original signed claim; bounds the retired version. + pub target: SignedRecord, + /// Requester's Ed25519 signature over this marker and its proof. + pub signature: Vec, +} + +impl SignedRetirement { + fn canonical_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(768); + push_lp(&mut out, b"tsunagi-node-retirement/v1"); + push_lp(&mut out, &self.network); + push_lp(&mut out, &self.author); + push_lp(&mut out, &self.version.to_be_bytes()); + // Bind every field of the original signed claim, including its signature. + push_lp(&mut out, &self.target.network); + push_lp(&mut out, &self.target.author); + push_lp(&mut out, &self.target.version.to_be_bytes()); + push_lp(&mut out, &self.target.body.canonical()); + push_lp(&mut out, &self.target.signature); + out + } + + /// Signs a marker with the requesting member's persistent device key. + pub fn sign( + secret: &SecretKey, + network: NetworkId, + version: u64, + target: SignedRecord, + ) -> Self { + let mut marker = Self { + author: *secret.public().as_bytes(), + network: *network.as_bytes(), + version, + target, + signature: Vec::new(), + }; + marker.signature = secret.sign(&marker.canonical_bytes()).to_bytes().to_vec(); + marker + } + + /// Verifies both authors, scope and a finite, genuinely signed cutoff. + pub fn verify(&self, network: NetworkId) -> Result { + if self.network != *network.as_bytes() { + return Err(StateError::WrongNetwork); + } + if self.version == 0 || self.version > i64::MAX as u64 { + return Err(StateError::Malformed("retirement version is out of range")); + } + if !matches!(self.target.body, RecordBody::Claim { .. }) { + return Err(StateError::Malformed("only a claim can be retired")); + } + let target = self.target.verify(network)?; + let author = EndpointId::from_bytes(&self.author).map_err(|_| StateError::BadAuthor)?; + if author == target { + return Err(StateError::Malformed( + "use Release to retire your own claim", + )); + } + let raw: [u8; Signature::LENGTH] = + self.signature + .as_slice() + .try_into() + .map_err(|_| StateError::BadSignatureLength { + expected: Signature::LENGTH, + })?; + author + .verify(&self.canonical_bytes(), &Signature::from_bytes(&raw)) + .map_err(|_| StateError::BadSignature)?; + Ok(target) + } + + /// A deterministic strongest marker per target, independent of arrival order. + pub(crate) fn supersedes(&self, other: &Self) -> bool { + self.target.version > other.target.version + || (self.target.version == other.target.version + && (self.canonical_bytes(), &self.signature) + < (other.canonical_bytes(), &other.signature)) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + use crate::state::{Ipv4Range, StateSet}; + + fn claim(key: &SecretKey, id: NetworkId, version: u64) -> SignedRecord { + SignedRecord::sign( + key, + id, + version, + RecordBody::Claim { + address: Some("10.23.0.2".parse().unwrap()), + range: Some("10.23.0.0/24".parse::().unwrap()), + hostname: Some("retired".into()), + hostname_since: Some(123), + }, + ) + } + + #[test] + fn markers_release_claims_in_any_merge_order_and_older_snapshots_cannot_restore_them() { + let id = NetworkId::from_bytes([7; 32]); + let target = SecretKey::generate(); + let cleaner = SecretKey::generate(); + let record = claim(&target, id, 5); + let marker = SignedRetirement::sign(&cleaner, id, 8, record.clone()); + for first in [true, false] { + let mut state = StateSet::new(); + if first { + state.merge_retirement(id, marker.clone(), 16).unwrap(); + } + state.merge(id, record.clone()).unwrap(); + if !first { + state.merge_retirement(id, marker.clone(), 16).unwrap(); + } + assert!(state.address_holders().is_empty()); + assert!(state.hostname_holders().is_empty()); + state.merge(id, claim(&target, id, 4)).unwrap(); + assert!(state.is_retired(&target.public())); + assert!( + state.agreed_range().is_some(), + "cleanup does not renumber the network" + ); + state.merge(id, claim(&target, id, 6)).unwrap(); + assert!(!state.is_retired(&target.public())); + assert_eq!(state.hostname_of(&target.public()), Some("retired")); + } + } + + #[test] + fn proof_signature_scope_tampering_future_cutoffs_and_capacity_are_checked() { + let id = NetworkId::from_bytes([7; 32]); + let target = SecretKey::generate(); + let cleaner = SecretKey::generate(); + let marker = SignedRetirement::sign(&cleaner, id, 8, claim(&target, id, 5)); + assert!(marker.verify(NetworkId::from_bytes([8; 32])).is_err()); + let mut forged = marker.clone(); + forged.target.version = 99; + assert!(forged.verify(id).is_err()); + let mut forged = marker.clone(); + forged.signature[0] ^= 1; + assert!(forged.verify(id).is_err()); + let mut proof = marker.target.clone(); + proof.version = 99; + assert!( + SignedRetirement::sign(&cleaner, id, 8, proof) + .verify(id) + .is_err() + ); + let mut state = StateSet::new(); + assert!(state.merge_retirement(id, marker.clone(), 0).is_err()); + assert!(state.merge_retirement(id, marker.clone(), 1).unwrap()); + assert!(!state.merge_retirement(id, marker.clone(), 1).unwrap()); + assert!(postcard::to_stdvec(&marker).unwrap().len() <= MAX_RETIREMENT_LEN); + let another = SecretKey::generate(); + assert!( + state + .merge_retirement( + id, + SignedRetirement::sign(&cleaner, id, 9, claim(&another, id, 1)), + 1 + ) + .is_err() + ); + } + + #[test] + fn competing_cleanup_markers_converge_and_keep_the_highest_observed_cutoff() { + let id = NetworkId::from_bytes([7; 32]); + let target = SecretKey::generate(); + let a = SecretKey::generate(); + let b = SecretKey::generate(); + let markers = [ + SignedRetirement::sign(&a, id, 1, claim(&target, id, 4)), + SignedRetirement::sign(&a, id, 2, claim(&target, id, 5)), + SignedRetirement::sign(&b, id, 3, claim(&target, id, 5)), + ]; + let mut forward = StateSet::new(); + let mut reverse = StateSet::new(); + for marker in &markers { + forward.merge_retirement(id, marker.clone(), 16).unwrap(); + } + for marker in markers.iter().rev() { + reverse.merge_retirement(id, marker.clone(), 16).unwrap(); + } + assert_eq!(forward.retirements(), reverse.retirements()); + assert_eq!( + forward + .retirement_of(&target.public()) + .unwrap() + .target + .version, + 5 + ); + } +} diff --git a/crates/tsunagi-core/src/storage/mod.rs b/crates/tsunagi-core/src/storage/mod.rs index a7b4bf4..1274c58 100644 --- a/crates/tsunagi-core/src/storage/mod.rs +++ b/crates/tsunagi-core/src/storage/mod.rs @@ -310,6 +310,7 @@ impl Storage { self.with_state(move |state| { state.remove_network(network_id)?; state.forget_signed_records(network_id)?; + state.forget_retirements(network_id)?; state.forget_peer_hostnames(network_id) }) .await?; @@ -369,6 +370,31 @@ impl Storage { .await } + /// Loads verified durable cleanup proofs off the executor. + pub async fn retirements( + &self, + network: NetworkId, + max: usize, + ) -> Result> { + self.with_state(move |state| state.retirements(network, max)) + .await + } + + /// Persists a received cleanup proof before applying it. + pub async fn put_retirement(&self, marker: crate::state::SignedRetirement) -> Result<()> { + self.with_state(move |state| state.put_retirement(&marker)) + .await + } + + /// Commits our cleanup and author counter together before announcement. + pub async fn publish_own_retirement( + &self, + marker: crate::state::SignedRetirement, + ) -> Result<()> { + self.with_state(move |state| state.publish_own_retirement(&marker)) + .await + } + /// Stores a record received from another replica. pub async fn put_signed_record(&self, record: SignedRecord) -> Result<()> { self.with_state(move |state| state.put_signed_record(&record)) diff --git a/crates/tsunagi-core/src/storage/state.rs b/crates/tsunagi-core/src/storage/state.rs index fac7b5d..3765fa0 100644 --- a/crates/tsunagi-core/src/storage/state.rs +++ b/crates/tsunagi-core/src/storage/state.rs @@ -18,10 +18,10 @@ use rusqlite::{Connection, OptionalExtension, params}; use crate::error::{Error, Result}; use crate::identity::{DeviceIdentity, NetworkId, NetworkName, NetworkSecret}; -use crate::state::{RecordBody, SignedRecord}; +use crate::state::{MAX_RETIREMENT_LEN, RecordBody, SignedRecord, SignedRetirement}; /// Schema version written by this build. -pub const SCHEMA_VERSION: i64 = 8; +pub const SCHEMA_VERSION: i64 = 9; /// Key of the stored hostname setting. const SETTING_HOSTNAME: &str = "hostname"; @@ -250,6 +250,19 @@ impl StateStore { ) .map_err(|err| self.corrupt(format!("cannot migrate schema to 8: {err}")))?; } + if found < 9 { + self.conn + .execute_batch( + "BEGIN; + CREATE TABLE IF NOT EXISTS retirements ( + network_id BLOB NOT NULL, target BLOB NOT NULL, data BLOB NOT NULL, + PRIMARY KEY (network_id, target) + ); + PRAGMA user_version = 9; + COMMIT;", + ) + .map_err(|err| self.corrupt(format!("cannot migrate schema to 9: {err}")))?; + } Ok(()) } @@ -262,6 +275,7 @@ impl StateStore { "settings", "signed_records", "peer_hostnames", + "retirements", ] { let present: Option = self .conn @@ -785,6 +799,112 @@ impl StateStore { Ok(version.unwrap_or(0).max(0) as u64) } + /// Loads durable cleanup markers without allocating oversized stored blobs. + pub fn retirements(&self, network: NetworkId, max: usize) -> Result> { + let mut stmt = self + .conn + .prepare( + "SELECT CASE WHEN length(data) <= ?2 THEN data ELSE NULL END + FROM retirements WHERE network_id = ?1 ORDER BY target", + ) + .map_err(|err| self.corrupt(err))?; + let rows = stmt + .query_map( + params![network.as_bytes().as_slice(), MAX_RETIREMENT_LEN as i64], + |row| row.get::<_, Option>>(0), + ) + .map_err(|err| self.corrupt(err))?; + let mut out = Vec::new(); + for row in rows { + if out.len() >= max { + return Err(Error::Storage( + "stored retirements exceed the configured limit".into(), + )); + } + let bytes = row + .map_err(|err| self.corrupt(err))? + .ok_or_else(|| self.corrupt("oversized retirement"))?; + let marker: SignedRetirement = + postcard::from_bytes(&bytes).map_err(|err| self.corrupt(err))?; + marker.verify(network).map_err(|err| self.corrupt(err))?; + out.push(marker); + } + Ok(out) + } + + /// Persists a strongest verified marker received from an authenticated session. + pub fn put_retirement(&self, marker: &SignedRetirement) -> Result<()> { + let network = NetworkId::from_bytes(marker.network); + marker + .verify(network) + .map_err(|err| Error::Storage(err.to_string()))?; + let old: Option>> = self.conn.query_row( + "SELECT CASE WHEN length(data) <= ?3 THEN data ELSE NULL END FROM retirements WHERE network_id = ?1 AND target = ?2", + params![marker.network.as_slice(), marker.target.author.as_slice(), MAX_RETIREMENT_LEN as i64], |row| row.get(0)) + .optional().map_err(|err| self.corrupt(err))?; + if let Some(bytes) = old { + let bytes = bytes.ok_or_else(|| self.corrupt("oversized retirement"))?; + let old: SignedRetirement = + postcard::from_bytes(&bytes).map_err(|err| self.corrupt(err))?; + old.verify(network).map_err(|err| self.corrupt(err))?; + if !marker.supersedes(&old) { + return Ok(()); + } + } + let bytes = postcard::to_stdvec(marker).map_err(|err| Error::Storage(err.to_string()))?; + if bytes.len() > MAX_RETIREMENT_LEN { + return Err(Error::Storage( + "retirement exceeds its encoded bound".into(), + )); + } + self.conn + .execute( + "INSERT INTO retirements (network_id, target, data) VALUES (?1, ?2, ?3) + ON CONFLICT(network_id, target) DO UPDATE SET data = excluded.data", + params![ + marker.network.as_slice(), + marker.target.author.as_slice(), + bytes + ], + ) + .map_err(|err| Error::Storage(format!("cannot store retirement: {err}")))?; + Ok(()) + } + + /// Commits our signed cleanup marker and author counter in one transaction. + pub fn publish_own_retirement(&self, marker: &SignedRetirement) -> Result<()> { + let transaction = self + .conn + .unchecked_transaction() + .map_err(|err| Error::Storage(err.to_string()))?; + self.put_retirement(marker)?; + transaction + .execute( + "INSERT INTO own_record_version (network_id, author, version) VALUES (?1, ?2, ?3) + ON CONFLICT(network_id, author) DO UPDATE SET version = max(version, excluded.version)", + params![ + marker.network.as_slice(), + marker.author.as_slice(), + marker.version as i64 + ], + ) + .map_err(|err| Error::Storage(format!("cannot store retirement counter: {err}")))?; + transaction + .commit() + .map_err(|err| Error::Storage(format!("cannot commit retirement: {err}"))) + } + + /// Drops a forgotten network's local cleanup proofs. + pub fn forget_retirements(&self, network: NetworkId) -> Result<()> { + self.conn + .execute( + "DELETE FROM retirements WHERE network_id = ?1", + params![network.as_bytes().as_slice()], + ) + .map_err(|err| Error::Storage(err.to_string()))?; + Ok(()) + } + /// Forgets every record of a network. pub fn forget_signed_records(&self, network_id: NetworkId) -> Result<()> { self.conn @@ -1006,6 +1126,96 @@ mod broadcast_storage_tests { assert!(store.peer_hostnames(id).unwrap().is_empty()); } + #[test] + fn retirement_migration_and_atomic_counter_commit_preserve_existing_state() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.sqlite"); + let store = StateStore::open(&path).unwrap(); + let identity = store.load_or_create_device_identity().unwrap(); + let name = NetworkName::new("retirement-migration").unwrap(); + let secret = NetworkSecret::from_bytes([6; 32]).unwrap(); + let id = NetworkKeys::derive(&name, &secret).network_id(); + store.upsert_network(id, &name, &secret, true).unwrap(); + store.set_protocols(id, &["awg".into()]).unwrap(); + let own = SignedRecord::sign(&identity.signing_key(), id, 8, RecordBody::Release); + store.publish_own_record(&own).unwrap(); + store + .conn + .execute_batch("DROP TABLE retirements; PRAGMA user_version = 8;") + .unwrap(); + drop(store); + let store = StateStore::open(&path).unwrap(); + assert_eq!( + store.device_identity().unwrap().unwrap().endpoint_id(), + identity.endpoint_id() + ); + assert_eq!( + store.list_networks().unwrap()[0].protocols, + Some(vec!["awg".into()]) + ); + assert_eq!(store.signed_records(id).unwrap(), vec![own]); + let target_key = iroh::SecretKey::generate(); + let target = SignedRecord::sign( + &target_key, + id, + 3, + RecordBody::Claim { + address: Some("10.13.37.2".parse().unwrap()), + range: Some(crate::state::DEFAULT_IPV4_RANGE), + hostname: Some("old-device".into()), + hostname_since: Some(123), + }, + ); + let marker = SignedRetirement::sign(&identity.signing_key(), id, 9, target); + store + .conn + .execute_batch( + "CREATE TRIGGER fail_retirement_counter BEFORE INSERT ON own_record_version + WHEN NEW.version = 9 BEGIN SELECT RAISE(ABORT, 'counter write refused'); END;", + ) + .unwrap(); + assert!(store.publish_own_retirement(&marker).is_err()); + assert!( + store.retirements(id, 16).unwrap().is_empty(), + "marker must roll back with counter failure" + ); + assert_eq!( + store + .own_record_version(id, identity.endpoint_id()) + .unwrap(), + 8 + ); + store + .conn + .execute_batch("DROP TRIGGER fail_retirement_counter;") + .unwrap(); + store.publish_own_retirement(&marker).unwrap(); + drop(store); + let store = StateStore::open(&path).unwrap(); + assert_eq!(store.retirements(id, 16).unwrap(), vec![marker.clone()]); + assert_eq!( + store + .own_record_version(id, identity.endpoint_id()) + .unwrap(), + 9 + ); + assert!( + store.retirements(id, 0).is_err(), + "capacity changes must not silently drop markers" + ); + store + .conn + .execute( + "UPDATE retirements SET data = ?1", + params![vec![0u8; MAX_RETIREMENT_LEN + 1]], + ) + .unwrap(); + assert!(matches!( + store.retirements(id, 16), + Err(Error::StateCorrupted { .. }) + )); + } + #[test] fn v7_migration_keeps_signed_state_and_protocol_overrides_are_local_and_durable() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/tsunagi-core/tests/forget_node.rs b/crates/tsunagi-core/tests/forget_node.rs new file mode 100644 index 0000000..4215a3d --- /dev/null +++ b/crates/tsunagi-core/tests/forget_node.rs @@ -0,0 +1,358 @@ +//! Equal-rights retirement uses real membership, signatures and persistent state. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use tsunagi::discovery::SharedMemoryDiscovery; +use tsunagi::testing::{TestAgent, config_with, network, wait_for_peers, wait_until}; +use tsunagi::{Agent, NetworkId}; + +#[tokio::test] +async fn bounded_marker_batches_eventually_reach_the_tail_of_a_small_send_queue() { + use tsunagi::config::StoragePaths; + use tsunagi::state::{RecordBody, SignedRecord, SignedRetirement}; + use tsunagi::storage::StateStore; + + let discovery = SharedMemoryDiscovery::new(); + let cleaner = TestAgent::spawn(&discovery).await.unwrap(); + let replica = TestAgent::spawn(&discovery).await.unwrap(); + let (name, secret) = network("forget-batched-history"); + let id = cleaner.agent.join_network(&name, &secret).await.unwrap(); + replica.agent.join_network(&name, &secret).await.unwrap(); + let cleaner_dir = cleaner.stop().await; + let replica_dir = replica.stop().await; + let cleaner_path = StoragePaths::under(cleaner_dir.path()).state_db(); + let replica_path = StoragePaths::under(replica_dir.path()).state_db(); + // Seed genuine signed offline history in closed stores. Delivery still uses + // real agents, authenticated loopback sessions and the persistent stores. + let targets = tokio::task::spawn_blocking(move || { + let cleaner = StateStore::open(&cleaner_path).unwrap(); + let replica = StateStore::open(&replica_path).unwrap(); + let author = cleaner.device_identity().unwrap().unwrap(); + let signing_key = iroh::SecretKey::from_bytes(&author.secret_bytes()); + let mut targets = Vec::new(); + for index in 0..12 { + let target = iroh::SecretKey::generate(); + let record = SignedRecord::sign( + &target, + id, + 1, + RecordBody::Claim { + address: None, + range: None, + hostname: Some(format!("offline-{index}")), + hostname_since: Some(1), + }, + ); + cleaner.put_signed_record(&record).unwrap(); + replica.put_signed_record(&record).unwrap(); + cleaner + .publish_own_retirement(&SignedRetirement::sign( + &signing_key, + id, + 100 + index, + record, + )) + .unwrap(); + targets.push(target.public()); + } + targets + }) + .await + .unwrap(); + let replica = Agent::spawn(config_with(replica_dir.path(), &discovery)) + .await + .unwrap(); + assert_eq!( + replica + .network_status(id) + .await + .unwrap() + .members + .iter() + .filter(|m| targets.contains(&m.endpoint_id)) + .count(), + 12 + ); + let mut config = config_with(cleaner_dir.path(), &discovery); + config.limits.max_state_records = 1; + config.limits.session_send_queue = 2; + let cleaner = Agent::spawn(config).await.unwrap(); + wait_for_peers(&cleaner, id, 1).await; + wait_until( + "all cleanup batches arrive despite send queue pressure", + || async { + replica + .network_status(id) + .await + .ok()? + .members + .iter() + .all(|m| !targets.contains(&m.endpoint_id)) + .then_some(()) + }, + ) + .await; + cleaner.shutdown().await; + replica.shutdown().await; + let replica_path = StoragePaths::under(replica_dir.path()).state_db(); + tokio::task::spawn_blocking(move || { + let replica = StateStore::open(&replica_path).unwrap(); + assert_eq!(replica.retirements(id, 16).unwrap().len(), 12); + }) + .await + .unwrap(); +} + +async fn member( + agent: &Agent, + network: NetworkId, + peer: iroh::EndpointId, +) -> tsunagi::agent::MemberStatus { + wait_until("signed member", || async { + agent + .network_status(network) + .await + .ok()? + .members + .into_iter() + .find(|m| m.endpoint_id == peer) + }) + .await +} + +#[tokio::test] +async fn retirement_survives_restart_and_stale_replicas_but_allows_a_new_claim() { + let discovery = SharedMemoryDiscovery::new(); + let cleaner = TestAgent::spawn_with(|c| c.with_hostname("cleaner"), &discovery) + .await + .unwrap(); + let target = TestAgent::spawn_with(|c| c.with_hostname("old-device"), &discovery) + .await + .unwrap(); + let replica = TestAgent::spawn_with(|c| c.with_hostname("replica"), &discovery) + .await + .unwrap(); + let (name, secret) = network("forget-propagation"); + let id = cleaner.agent.join_network(&name, &secret).await.unwrap(); + target.agent.join_network(&name, &secret).await.unwrap(); + replica.agent.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&cleaner.agent, id, 2).await; + let peer = target.agent.endpoint_id(); + let original = wait_until("all three agents agree on allocated claims", || async { + let a = cleaner.agent.network_status(id).await.ok()?; + let b = target.agent.network_status(id).await.ok()?; + let c = replica.agent.network_status(id).await.ok()?; + for status in [&a, &b, &c] { + if status.members.len() != 3 { + return None; + } + let addresses: std::collections::HashSet<_> = status + .members + .iter() + .filter_map(|m| m.overlay_address_v4) + .collect(); + if addresses.len() != 3 { + return None; + } + } + let record = a.members.iter().find(|m| m.endpoint_id == peer)?; + (b.members + .iter() + .find(|m| m.endpoint_id == peer)? + .record_version + == record.record_version + && c.members + .iter() + .find(|m| m.endpoint_id == peer)? + .record_version + == record.record_version) + .then(|| record.clone()) + }) + .await; + assert!( + cleaner + .agent + .forget_node(id, peer, original.record_version) + .await + .is_err(), + "online nodes are not history" + ); + assert!( + cleaner + .agent + .forget_node(id, cleaner.agent.endpoint_id(), 1) + .await + .is_err() + ); + let replica_dir = replica.stop().await; + let target_dir = target.stop().await; + wait_until("target offline", || async { + cleaner + .agent + .network_status(id) + .await + .ok()? + .members + .iter() + .find(|m| m.endpoint_id == peer) + .filter(|m| !m.online) + .map(|_| ()) + }) + .await; + assert!( + cleaner + .agent + .forget_node(id, peer, original.record_version + 1) + .await + .is_err(), + "stale confirmation changes nothing" + ); + let result = cleaner + .agent + .forget_node(id, peer, original.record_version) + .await + .unwrap(); + assert_eq!(result.target, peer); + assert_eq!(result.record_version, original.record_version); + assert_eq!(result.peers_told, 0); + assert!( + !cleaner + .agent + .network_status(id) + .await + .unwrap() + .members + .iter() + .any(|m| m.endpoint_id == peer) + ); + let cleaner_dir = cleaner.stop().await; + let cleaner = + Agent::spawn(config_with(cleaner_dir.path(), &discovery).with_hostname("cleaner")) + .await + .unwrap(); + let replica = + Agent::spawn(config_with(replica_dir.path(), &discovery).with_hostname("replica")) + .await + .unwrap(); + wait_for_peers(&cleaner, id, 1).await; + wait_until("stale replica learns durable retirement", || async { + replica + .network_status(id) + .await + .ok()? + .members + .iter() + .all(|m| m.endpoint_id != peer) + .then_some(()) + }) + .await; + assert!( + !cleaner + .network_status(id) + .await + .unwrap() + .members + .iter() + .any(|m| m.endpoint_id == peer) + ); + replica.set_hostname("old-device").await.unwrap(); + let replica_id = replica.endpoint_id(); + wait_until("released hostname is reclaimed", || async { + cleaner + .network_status(id) + .await + .ok()? + .members + .iter() + .find(|m| m.endpoint_id == replica_id) + .filter(|m| m.hostname.as_deref() == Some("old-device"))? + .hostname_conflict + .is_none() + .then_some(()) + }) + .await; + let returned = + Agent::spawn(config_with(target_dir.path(), &discovery).with_hostname("old-device")) + .await + .unwrap(); + wait_for_peers(&returned, id, 2).await; + wait_until("returning identity publishes a fresh version", || async { + let m = cleaner + .network_status(id) + .await + .ok()? + .members + .into_iter() + .find(|m| m.endpoint_id == peer)?; + (m.online && m.record_version > original.record_version && m.hostname_conflict.is_some()) + .then_some(()) + }) + .await; + cleaner.shutdown().await; + replica.shutdown().await; + returned.shutdown().await; +} + +#[tokio::test] +async fn forgetting_is_network_scoped_and_every_member_has_the_same_right() { + let discovery = SharedMemoryDiscovery::new(); + let a = TestAgent::spawn(&discovery).await.unwrap(); + let b = TestAgent::spawn(&discovery).await.unwrap(); + let target = TestAgent::spawn(&discovery).await.unwrap(); + let (name, secret) = network("forget-one-network"); + let (other_name, other_secret) = network("keep-other-network"); + let id = a.agent.join_network(&name, &secret).await.unwrap(); + b.agent.join_network(&name, &secret).await.unwrap(); + target.agent.join_network(&name, &secret).await.unwrap(); + let other = a + .agent + .join_network(&other_name, &other_secret) + .await + .unwrap(); + target + .agent + .join_network(&other_name, &other_secret) + .await + .unwrap(); + let peer = target.agent.endpoint_id(); + let record = member(&b.agent, id, peer).await; + member(&a.agent, other, peer).await; + target.agent.shutdown().await; + wait_until("offline on B", || async { + b.agent + .network_status(id) + .await + .ok()? + .members + .iter() + .find(|m| m.endpoint_id == peer) + .filter(|m| !m.online) + .map(|_| ()) + }) + .await; + b.agent + .forget_node(id, peer, record.record_version) + .await + .unwrap(); + wait_until("A learns B's retirement", || async { + a.agent + .network_status(id) + .await + .ok()? + .members + .iter() + .all(|m| m.endpoint_id != peer) + .then_some(()) + }) + .await; + assert!( + a.agent + .network_status(other) + .await + .unwrap() + .members + .iter() + .any(|m| m.endpoint_id == peer) + ); + a.agent.shutdown().await; + b.agent.shutdown().await; +} diff --git a/crates/tsunagi-tray/src/agent.rs b/crates/tsunagi-tray/src/agent.rs index f99a13f..6c386ea 100644 --- a/crates/tsunagi-tray/src/agent.rs +++ b/crates/tsunagi-tray/src/agent.rs @@ -21,6 +21,12 @@ const POLL_INTERVAL: Duration = Duration::from_secs(2); /// A request from the UI to the agent. pub(crate) enum Command { + /// Retire the exact offline-node claim selected in the devices window. + ForgetNode { + network_id: String, + peer: String, + expected_version: u64, + }, /// Replace the network's enabled protocols in preference order. SetProtocols { network_id: String, @@ -192,6 +198,19 @@ async fn run( ctx.request_repaint(); let outcome: Result = match command { + Command::ForgetNode { + network_id, + peer, + expected_version, + } => ipc::forget_node(socket, &network_id, &peer, expected_version) + .await + .map(|report| { + format!( + "forgot node claim v{}; cleanup queued to {} peers", + report.record_version, report.peers_told + ) + }) + .map_err(|err| err.to_string()), Command::SetProtocols { network_id, protocols, diff --git a/crates/tsunagi-tray/src/app.rs b/crates/tsunagi-tray/src/app.rs index 9d9875a..c475949 100644 --- a/crates/tsunagi-tray/src/app.rs +++ b/crates/tsunagi-tray/src/app.rs @@ -116,20 +116,21 @@ impl App { .with_title(devices::title(network)) .with_app_id(crate::icon::APP_ID) .with_icon(crate::icon::window()) - .with_inner_size([940.0, 540.0]) - .with_min_inner_size([480.0, 320.0]); + .with_inner_size([1440.0, 640.0]) + .with_min_inner_size([800.0, 400.0]); let agent = &self.agent; let traffic = &self.traffic; // The window shares the single unit setting, so its toggle and the // main window's stay in step. let unit = &mut self.state.unit; let confirm = &mut self.state.exit_confirm; + let forget_confirm = &mut self.state.forget_confirm; let keep = ctx.show_viewport_immediate(viewport_id, builder, |vctx, _class| { devices::show( vctx, agent, unit, - confirm, + (confirm, forget_confirm), network, &report.endpoint_id, traffic, diff --git a/crates/tsunagi-tray/src/devices.rs b/crates/tsunagi-tray/src/devices.rs index 70f6034..04655e5 100644 --- a/crates/tsunagi-tray/src/devices.rs +++ b/crates/tsunagi-tray/src/devices.rs @@ -27,6 +27,8 @@ pub(crate) fn title(network: &NetworkReport) -> String { /// One row of the device table, built from a tunnel and/or a signed member. struct Device<'a> { + /// Observed claim version, if this row came from signed membership. + record_version: Option, /// The agent's own answer to whether it can be reached by any path. online: bool, /// How packets reach it. @@ -65,6 +67,15 @@ pub(crate) struct ExitConfirm { main: bool, } +/// An exact offline claim selected for durable cleanup. +pub(crate) struct ForgetConfirm { + network_id: String, + peer: String, + version: u64, + name: String, + address: Option, +} + impl ExitConfirm { /// A question about stopping the exit node a network's tile shows. pub(crate) fn stop_from_tile(network: &NetworkReport, peer_id: &str) -> Self { @@ -121,11 +132,12 @@ pub(crate) fn show( ctx: &egui::Context, agent: &AgentClient, unit: &mut Unit, - confirm: &mut Option, + confirmations: (&mut Option, &mut Option), network: &NetworkReport, own_id: &str, traffic: &Traffic, ) { + let (confirm, forget_confirm) = confirmations; egui::CentralPanel::default().show(ctx, |ui| { ui.horizontal(|ui| { ui.heading(&network.name); @@ -158,10 +170,68 @@ pub(crate) fn show( egui::ScrollArea::both() .auto_shrink([false, false]) .show(ui, |ui| { - device_table(ui, unit, network, &devices, traffic, confirm); + device_table( + ui, + unit, + network, + &devices, + traffic, + confirm, + forget_confirm, + ); }); }); confirm_exit(ctx, agent, confirm, Some(&network.network_id)); + confirm_forget(ctx, agent, forget_confirm, &network.network_id); +} + +fn confirm_forget( + ctx: &egui::Context, + agent: &AgentClient, + confirm: &mut Option, + network: &str, +) { + let Some(pending) = confirm.as_ref().filter(|c| c.network_id == network) else { + return; + }; + egui::Area::new(egui::Id::new("forget node backdrop")) + .order(egui::Order::Middle) + .fixed_pos(egui::Pos2::ZERO) + .show(ctx, |ui| { + let screen = ctx.screen_rect(); + ui.allocate_response(screen.size(), egui::Sense::click_and_drag()); + ui.painter() + .rect_filled(screen, 0.0, egui::Color32::from_black_alpha(140)); + }); + let (mut accepted, mut cancelled) = (false, false); + egui::Window::new("Forget offline node") + .order(egui::Order::Foreground) + .collapsible(false) + .resizable(false) + .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) + .show(ctx, |ui| { + ui.set_max_width(440.0); + ui.label(format!("Forget {}?", pending.name)); + ui.monospace(&pending.peer); + if let Some(address) = &pending.address { + ui.label(format!("IP: {address}")); + } + ui.label("The network will forget this claim and free its name and IP. A returning device can publish a new claim; this does not ban it."); + ui.horizontal(|ui| { + accepted = ui.add_enabled(!agent.snapshot().busy, egui::Button::new("Forget")).clicked(); + cancelled = ui.button("Cancel").clicked(); + }); + }); + if accepted { + agent.send(Command::ForgetNode { + network_id: pending.network_id.clone(), + peer: pending.peer.clone(), + expected_version: pending.version, + }); + } + if accepted || cancelled || ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + *confirm = None; + } } /// The confirmation for an exit-node choice, drawn in the window it was @@ -354,6 +424,7 @@ fn devices<'a>(network: &'a NetworkReport, own_id: &str) -> Vec> { }); let hostname = hostnames.get(id).copied(); Device { + record_version: member.map(|m| m.record_version), online, path, via_name, @@ -493,6 +564,7 @@ fn device_table( devices: &[Device<'_>], traffic: &Traffic, confirm: &mut Option, + forget_confirm: &mut Option, ) { let network_id = network.network_id.as_str(); if devices.is_empty() { @@ -542,7 +614,26 @@ fn device_table( ui.end_row(); for device in devices { - cell(ui, 0, &mut widths, |ui| name_cell(ui, device)); + cell(ui, 0, &mut widths, |ui| { + ui.horizontal(|ui| { + name_cell(ui, device); + if !device.online + && !device.in_use + && let Some(version) = device.record_version + && ui.add_enabled(network.active, egui::Button::new("Forget")) + .on_hover_text("Forget this offline claim across the network; returning devices may rejoin") + .clicked() + { + *forget_confirm = Some(ForgetConfirm { + network_id: network.network_id.clone(), + peer: device.id.into(), + version, + name: device.name(), + address: device.address.clone(), + }); + } + }); + }); cell(ui, 1, &mut widths, |ui| { exit_cell(ui, device, network_id, confirm); }); @@ -736,6 +827,7 @@ mod tests { fn member(id: &str, online: bool, path: DataPathReport) -> MemberReport { MemberReport { + record_version: 1, endpoint_id: id.into(), online, path, diff --git a/crates/tsunagi-tray/src/ui.rs b/crates/tsunagi-tray/src/ui.rs index 5d8007b..9432dac 100644 --- a/crates/tsunagi-tray/src/ui.rs +++ b/crates/tsunagi-tray/src/ui.rs @@ -45,6 +45,8 @@ pub(crate) struct UiState { pub(crate) about_open: bool, /// An exit-node choice waiting for the user to confirm it. pub(crate) exit_confirm: Option, + /// A selected offline claim waiting for cleanup confirmation. + pub(crate) forget_confirm: Option, } impl Default for UiState { @@ -56,6 +58,7 @@ impl Default for UiState { open_devices: BTreeSet::new(), about_open: false, exit_confirm: None, + forget_confirm: None, } } } diff --git a/crates/tsunagi/src/main.rs b/crates/tsunagi/src/main.rs index b41609b..93ace3e 100644 --- a/crates/tsunagi/src/main.rs +++ b/crates/tsunagi/src/main.rs @@ -204,6 +204,13 @@ struct JoinArgs { #[derive(Debug, Subcommand)] enum NetworkAction { + /// Forgets an offline node's observed claim on every replica; newer claims may return. + ForgetNode { + /// Exact network name, full id or unique id prefix. + network: String, + /// Exact node hostname, full endpoint id, unique id prefix or overlay IPv4. + node: String, + }, /// Shows or changes this network's local LAN broadcast participation. Broadcast { /// Exact network name, full id or unique id prefix. @@ -1570,6 +1577,31 @@ struct AgentControl { } impl tsunagi::ipc::ReportSource for AgentControl { + fn forget_node( + &self, + network: String, + peer: String, + expected_version: u64, + ) -> tsunagi::BoxFuture<'_, Result> { + Box::pin(async move { + let network = network + .parse() + .map_err(|err| format!("invalid network id: {err}"))?; + let peer = peer + .parse() + .map_err(|err| format!("invalid endpoint id: {err}"))?; + let result = self + .agent + .forget_node(network, peer, expected_version) + .await + .map_err(|err| err.to_string())?; + Ok(tsunagi::ipc::ForgottenNodeReport { + peer: result.target.to_string(), + record_version: result.record_version, + peers_told: result.peers_told as u32, + }) + }) + } fn report(&self) -> tsunagi::BoxFuture<'_, tsunagi::ipc::StatusReport> { Box::pin(async move { let dns = self.dns_state().await; @@ -2035,6 +2067,46 @@ async fn protocols_command(args: ProtocolsArgs) -> Result<(), Box Result<(), Box> { + let report = tsunagi::ipc::request_status(socket) + .await + .map_err(|err| format!("a reachable running agent is required to forget nodes: {err}"))?; + let entries: Vec<_> = report + .networks + .iter() + .map(|n| (n.network_id.clone(), n.name.clone())) + .collect(); + let network = &report.networks[resolve_network_index(&entries, network)?]; + if !network.active { + return Err(format!( + "start `{}` first: tsng network start {}", + network.name, network.network_id + ) + .into()); + } + let member = tsunagi::ipc::resolve_node(network, node)?; + let result = tsunagi::ipc::forget_node( + socket, + &network.network_id, + &member.endpoint_id, + member.record_version, + ) + .await?; + println!( + "forgot {} ({}) claim v{} in `{}`; cleanup queued to {} peers. A newer signed claim can return.", + member.hostname.as_deref().unwrap_or("unnamed"), + result.peer, + result.record_version, + network.name, + result.peers_told + ); + Ok(()) +} + /// `tsng network`: what this device belongs to, and leaving it. async fn network_command(args: NetworkArgs) -> Result<(), Box> { let paths = args.paths.resolve()?; @@ -2042,6 +2114,9 @@ async fn network_command(args: NetworkArgs) -> Result<(), Box { + forget_node_command(&socket, &network, &node).await + } None => show_networks(&paths, &socket).await, Some(NetworkAction::Broadcast { network, choice }) => { broadcast_command(&paths, &socket, &network, choice).await @@ -5259,6 +5334,7 @@ async fn build_report( .members .iter() .map(|member| MemberReport { + record_version: member.record_version, endpoint_id: member.endpoint_id.to_string(), overlay_address_v4: member.overlay_address_v4.map(|addr| addr.to_string()), // What this agent is currently experiencing trying to @@ -5688,6 +5764,7 @@ mod status_tests { }], members: vec![ MemberReport { + record_version: 1, endpoint_id: OWN.into(), overlay_address_v4: Some("10.13.37.69".into()), failed_dials: 0, @@ -5695,6 +5772,7 @@ mod status_tests { ..Default::default() }, MemberReport { + record_version: 1, endpoint_id: ONLINE.into(), overlay_address_v4: Some("10.13.37.237".into()), failed_dials: 0, @@ -5971,6 +6049,7 @@ mod status_tests { fn offline_members_are_listed_after_online_ones() { let mut network = network_after_a_peer_returned(); network.members.push(MemberReport { + record_version: 1, endpoint_id: AWAY.into(), overlay_address_v4: Some("10.13.37.99".into()), failed_dials: 9, @@ -6092,6 +6171,7 @@ mod status_tests { fn an_offline_member_keeps_the_last_name_it_announced() { let mut network = network_after_a_peer_returned(); network.members.push(MemberReport { + record_version: 1, endpoint_id: AWAY.into(), overlay_address_v4: Some("10.13.37.99".into()), failed_dials: 0, @@ -6113,6 +6193,7 @@ mod status_tests { // are attributed to it instead of floating free as a counter. let mut network = network_after_a_peer_returned(); network.members.push(MemberReport { + record_version: 1, endpoint_id: AWAY.into(), overlay_address_v4: Some("10.13.37.99".into()), failed_dials: 9, @@ -6137,6 +6218,7 @@ mod status_tests { // and said to be, rather than being shown as away or as a fault. let mut network = network_after_a_peer_returned(); network.members.push(MemberReport { + record_version: 1, endpoint_id: AWAY.into(), overlay_address_v4: Some("10.13.37.99".into()), hostname: Some("laptop".into()), diff --git a/crates/tsunagi/tests/network_cli.rs b/crates/tsunagi/tests/network_cli.rs index 794de55..d2753b4 100644 --- a/crates/tsunagi/tests/network_cli.rs +++ b/crates/tsunagi/tests/network_cli.rs @@ -58,6 +58,15 @@ fn start_bare_range(port: u16, range: Option<&str>) -> Running { } fn start_bare_config(port: u16, range: Option<&str>, protocols: Option<&str>) -> Running { + start_bare_peers(port, range, protocols, &[]) +} + +fn start_bare_peers( + port: u16, + range: Option<&str>, + protocols: Option<&str>, + peers: &[String], +) -> Running { let dir = TempDir::new().unwrap(); let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_tsng")); command @@ -78,6 +87,9 @@ fn start_bare_config(port: u16, range: Option<&str>, protocols: Option<&str>) -> if let Some(protocols) = protocols { command.args(["--protocol", protocols]); } + for peer in peers { + command.args(["--peer", peer]); + } let child = command.spawn().expect("the agent binary starts"); let mut agent = Running { child, dir }; let runtime = tokio::runtime::Builder::new_current_thread() @@ -119,6 +131,113 @@ fn start(network: &str, port: u16) -> Running { agent } +#[test] +fn cli_forgets_only_the_selected_offline_claim_over_real_control() { + let cleaner = start("forget-cli", 0); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let socket = tsunagi::ipc::control_socket_path(&cleaner.dir.path().join("state")); + let read = || { + runtime + .block_on(tsunagi::ipc::request_status(&socket)) + .unwrap() + }; + let before = read(); + let bootstrap = format!("{}@{}", before.endpoint_id, before.bound_sockets[0]); + let target = start_bare_peers(0, None, Some("none"), &[bootstrap]); + assert!( + target + .run(&["id", "hostname", "old-cli-node"]) + .status + .success() + ); + assert!( + target + .run(&[ + "join", + "--network", + "forget-cli", + "--secret", + "a-secret-for-the-cli-test" + ]) + .status + .success() + ); + runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let status = tsunagi::ipc::request_status(&socket).await.unwrap(); + if status.networks[0] + .members + .iter() + .any(|m| m.hostname.as_deref() == Some("old-cli-node") && m.online) + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .unwrap(); + }); + assert!( + !cleaner + .run(&["network", "forget-node", "forget-cli", "old-cli-node"]) + .status + .success() + ); + assert!( + !cleaner + .run(&["network", "forget-node", "forget-cli", &before.endpoint_id]) + .status + .success() + ); + assert!( + target + .run(&["network", "stop", "forget-cli"]) + .status + .success() + ); + runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let status = tsunagi::ipc::request_status(&socket).await.unwrap(); + if status.networks[0] + .members + .iter() + .any(|m| m.hostname.as_deref() == Some("old-cli-node") && !m.online) + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .unwrap(); + }); + let forgotten = cleaner.run(&["network", "forget-node", "forget-cli", "old-cli-node"]); + assert!( + forgotten.status.success(), + "{}", + String::from_utf8_lossy(&forgotten.stderr) + ); + assert!(String::from_utf8_lossy(&forgotten.stdout).contains("newer signed claim can return")); + assert!( + !read().networks[0] + .members + .iter() + .any(|m| m.hostname.as_deref() == Some("old-cli-node")) + ); + assert!( + !cleaner + .run(&["network", "forget-node", "forget-cli", "old-cli-node"]) + .status + .success() + ); +} + #[test] fn protocols_absent_from_startup_can_be_enabled_and_changed_offline() { let mut agent = start_bare_config(0, None, Some("none")); diff --git a/docs/protocol.md b/docs/protocol.md index c0c007d..cabbf5b 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -9,14 +9,18 @@ Two versions exist and are independent: - **Identity scheme**, `tsunagi-network-id-v1`. Frozen. Changing it creates a different network space for the same name and secret. -- **Control protocol**, ALPN `tsunagi/ctrl/6`, `PROTOCOL_VERSION = 6`. +- **Control protocol**, ALPN `tsunagi/ctrl/7`, `PROTOCOL_VERSION = 7`. Upgrading the crate or bumping the control protocol must never change an existing `NetworkId`. -Control version 6 adds private transport negotiation for pairwise AWG and data -delivery checks. Data ALPN remains `tsunagi/data/5`. All members must upgrade -together; network ids and stored state are unchanged. See [amnezia.md](amnezia.md). +Control version 7 adds signed offline-node retirement messages to version 6's +private AWG transport negotiation and delivery checks. Data ALPN remains +`tsunagi/data/5`. All members must upgrade together; network ids, handshake +encoding and existing record signatures are unchanged. Local CLI/tray IPC is +version 21, so restart the agent with matching clients. SQLite schema 9 adds +retirement storage while preserving existing identities, settings and records. +See [sync-model.md](sync-model.md#offline-node-cleanup) and [amnezia.md](amnezia.md). ## Canonical encoding @@ -238,6 +242,7 @@ not affect other networks. | `Ping { seq, payload }` | small request used to verify the exchange | | `Pong { seq, payload }` | the echoed reply | | `State { records }` | a snapshot of signed records, merged into what the receiver holds | +| `Retirements { markers }` | signed cleanup markers with the target's original signed claim as proof | | `Peers { peers }` | unverified member address hints, requiring their own handshake | | `Reachable { links }` | sender's current direct data links, by kind of transport | | `Bye { reason }` | graceful goodbye; not a revocation of anything | diff --git a/docs/routing.md b/docs/routing.md index d4b2d0a..917069b 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -124,5 +124,5 @@ counts sends without storing frames. Results are CPU forwarding cost, **not** end-to-end network latency or VPN throughput; encryption, fragmentation, sockets, congestion and scheduling contribute separately. -Wire compatibility: control ALPN `tsunagi/ctrl/6`, data ALPN `tsunagi/data/5`. +Wire compatibility: control ALPN `tsunagi/ctrl/7`, data ALPN `tsunagi/data/5`. Upgrade every participant together; saved identities and network state persist. diff --git a/docs/sync-model.md b/docs/sync-model.md index a6f859b..18e3e27 100644 --- a/docs/sync-model.md +++ b/docs/sync-model.md @@ -1,18 +1,9 @@ -# Planned state synchronisation +# State synchronisation -**The first slice of this model is now implemented**, in `crates/tsunagi-core/src/state/`, and is -used for one thing: IPv4 overlay addresses. What follows describes the whole -model; the section at the end says exactly which parts exist. - -The proof of concept still exchanges hostname and capability announcements -over live sessions and keeps no replicated history for those, which is why -WireGuard peer *membership* remains session-scoped even though a peer's -*address* no longer is. This file records the intended direction so the module -boundaries in [architecture.md](architecture.md) stay compatible with it, and so -nobody mistakes the current announcements for synchronisation. - -There is no fake "ready CRDT" here and no snapshots that are not actually -verified. +Signed claims in `crates/tsunagi-core/src/state/` preserve IPv4 reservations and +hostnames across an absence. Signed releases and offline-node cleanup markers +free those reservations durably. Capabilities, links and authenticated sessions +remain volatile; a stored claim never authenticates a live connection. ## The problem @@ -75,6 +66,39 @@ Turning a computer off is not a revocation of its hostname and does not remove the participant. Revocations are not dropped merely because they are old, and no acknowledgement from offline peers is required to keep working. +## Offline-node cleanup + +Any authenticated member can sign a `SignedRetirement` for another member's +observed `Claim`. It includes the original signed claim as proof and uses the +separate `tsunagi-node-retirement/v1` signing domain. Both signatures and both +network scopes are checked. No owner, vote or quorum is involved, and no member +can fabricate the target's release or invent a future version cutoff. + +The strongest observed target version wins; ties use a deterministic canonical +byte order. Claims at or below that version do not occupy an address, hostname +or visible member row. Original signed records remain stored, including their +range, so cleanup does not renumber the network. Markers remain even after a +newer claim returns: absence from a snapshot is never deletion and replaying an +old snapshot cannot restore the retired claim. + +The local `Agent::forget_node` API checks the exact observed version again and +refuses self, online nodes, the selected exit node and missing/changed claims. +CLI and tray share this API. A returning live node can sign a newer claim and +allocate against the current holders; it resets the old hostname priority, so +it cannot reclaim a reused name merely by keeping its old timestamp. Cleanup +is not a permanent identity ban or a change of secret. + +The marker and requester's own author counter commit in one SQLite transaction +before publication. Receivers persist a verified marker before applying it. +Invalid proofs are rejected individually. Periodic bounded marker batches +rotate through each peer's send queue, including when a queue fills, so a +replica that missed an update can catch up without the requester being online. + +One strongest marker per target is retained, with a default limit of 1024 per +network and 1024 bytes per encoded marker. New targets beyond that limit are +refused rather than evicting resurrection protection. Corrupt mandatory marker +storage or a configured limit smaller than the stored set is an explicit error. + ## Storage requirement this creates When signed records land, writing the event and bumping the author's own @@ -104,9 +128,9 @@ Implemented, in `crates/tsunagi-core/src/state/`: * the merge rules above: higher version wins, an older version never rolls back a newer one, duplicates are idempotent, a same-version conflict is resolved identically on every replica and reported; -* a release tombstone, which merges correctly and is not undone by a replica - that has not heard of it — though nothing emits one yet, so freeing an - address still means forgetting the network; +* a release tombstone, emitted when leaving or replacing the device key and + not undone by a replica that has not heard of it; +* durable hostnames and equal-rights offline-node retirement described above; * persistence in `state.sqlite`, with the record and the author's own version counter committed in **one transaction before the record is announced**; * distribution as a `State` control message, merged into what the receiver @@ -114,21 +138,16 @@ Implemented, in `crates/tsunagi-core/src/state/`: * allocation of a free IPv4 address against what everybody else holds, which is what makes an address stable across an absence. -Deliberately not implemented: compaction, revoking a whole author, record -types beyond addressing, and any bound on how large a snapshot may grow -beyond the per-message limit. +Deliberately not implemented: compaction and permanent author revocation. +Claims have a per-message bound; retirement history also has a per-network bound. ## Future tests These are **not implemented and must not be reported as passing**: -- snapshot merge against the rules above, including conflicting same-version - records; -- revocation propagation and resistance to resurrection after compaction; +- resistance to resurrection after compaction (compaction is not implemented); - long network partitions and rejoin after an extended absence; -- hostname rename with atomic revoke-and-announce; -- recovery of a returning participant from a replica that is not the author; - NAT traversal and hole punching between real hosts; - relay fallback behaviour against a self-hosted relay; -- multi-process and multi-host deployment, as opposed to several library - instances inside one test process. +- multi-host deployment. CLI tests do run separate local agent processes; + library tests run several independent instances inside one process. diff --git a/docs/testing.md b/docs/testing.md index 82a8157..2c5dae9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -164,10 +164,9 @@ paused clock. The public Mainline/relay smoke test is ignored by default. ## Not covered, and not claimed to be -Listed in [sync-model.md](sync-model.md#future-tests): snapshots, revocations, -long partitions, hostname renames, recovery of a returning participant, NAT -traversal, relay fallback, and multi-process or multi-host deployment. None of -these are implemented, and none are marked as passing. +Listed in [sync-model.md](sync-model.md#future-tests): compaction, long partitions, +NAT traversal, relay fallback and multi-host deployment. These scenarios are +not established by the default loopback suite. ## Debugging a test @@ -198,3 +197,14 @@ SQLite tests migrate schema 7 to 8 without changing identity, secrets or signed records, preserve order and explicit empty choices, and reject malformed stored selections. These checks use memory TUNs and do not establish GUI rendering or behavior on an actual OS interface. + +`tests/forget_node.rs` uses real agents, handshakes and SQLite to check equal +cleanup rights, network isolation, online/self/stale-version refusal, restart, +recovery of a stale replica, reuse of a hostname and a returning identity's +newer claim. Bounded batch delivery is checked with a two-message send queue +and twelve retained markers. Retirement unit tests check signatures, proof tampering, invented +future cutoffs, bounds, deterministic merge and resistance to stale snapshots. +SQLite tests migrate schema 8 to 9 and inject counter-write failure to prove +atomic rollback. The CLI test starts two real agent processes and exercises +offline cleanup by network and hostname through the control socket. +Tray rendering and physical screen sizing are not exercised automatically. diff --git a/docs/threat-model.md b/docs/threat-model.md index d05c8a7..583f86c 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -38,6 +38,14 @@ Read this before relying on anything here. The protocol is in arbitrarily many identities, flood the network with records and collide with other participants' names. This is why a majority is not a root of trust. Signatures protect authorship; they do not make a participant honest. +- **Cleanup rights are equal.** Any member can retire an observed signed claim + of another device, freeing its hostname and IPv4 reservation. A valid original + claim bounds the cutoff; a member cannot invent a future cutoff or forge the + target's release. The local API refuses online nodes, but a remote signature + cannot prove that a target was offline. A live target may publish a newer + claim; a malicious member can repeatedly disrupt reservations. Cleanup is + not expulsion or secret revocation. See + [sync-model.md](sync-model.md#offline-node-cleanup). - **Weak secrets.** This targets high-entropy secrets. There is no PAKE, so a short human passphrase can be guessed offline by anyone who can reach the handshake. Use `NetworkSecret::generate()`. @@ -55,8 +63,10 @@ Read this before relying on anything here. The protocol is in having iroh under it does not make it so. - **A cloned state directory is a cloned identity.** `state.sqlite` holds the device secret key and the network secrets. Copying it copies the participant. - Restoring an old backup rolls the agent's state back, which — once signed - records exist — can resurrect revoked information or replay stale versions. + Restoring an old backup rolls the agent's state back and can replay stale + versions until it meets a replica holding the newer signed state. Retained + cleanup markers prevent old claims from resurrecting after that merge; they + cannot help if every copy of a marker has been lost. - **A compromised host.** The secret is on disk to survive restarts. File permissions are owner-only where the platform supports it, and the state directory takes an ownership lock, but neither defends against a user who can