diff --git a/README.md b/README.md index b751da0..ad10885 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,25 @@ and join the same network with the secret that was printed: ./target/release/tsunagi join --network lab --secret tsn1... ``` -**`--peer` is how the first meeting happens.** Two agents that have never -met have nothing to go on: this project publishes nothing about who is in -which network, by design. One of them has to be told the other's endpoint -id — after that each remembers the other and finds it again by itself, so -it is needed once. An agent with nobody to contact says so in `status` -rather than sitting there looking patient. +**`--peer` is how the first meeting happens, and only the first.** Two +agents that have never met have nothing to go on: this project publishes +nothing about who is in which network, by design. One of them has to be +told the other's endpoint id — after that each remembers the other and +finds it again by itself. Give several for redundancy; any one of them +getting through is enough. + +**One introduction is enough for the whole network.** Members tell each +other about the members they know, and every author of a signed record is +somebody to try, so a device pointed at one member ends up talking to all +of them rather than to the one that happened to be on its command line. +What travels is a candidate — an address somebody has seen — and it is +authenticated by the handshake like any other; being introduced grants +nothing. It is deliberately the same shape a lookup in a distributed hash +table would return, so that is a source to add beside this one rather than +a redesign. + +An agent with nobody to contact says so in `status` rather than sitting +there looking patient. Within a few seconds both print something like: @@ -538,7 +551,9 @@ explicitly. ### Through somebody in the middle -Two members can both reach a third and not each other: a blocked path, a +Everybody tries everybody first: the mesh is pairwise, and a relay is only +for the pair that cannot manage it. Two members can both reach a third and +not each other: a blocked path, a relay that is unavailable, a network only reachable from inside somebody else's building. When that happens the pair is routed through a member that has both. diff --git a/crates/tsunagi/src/agent/network.rs b/crates/tsunagi/src/agent/network.rs index e5ff483..9dc29e1 100644 --- a/crates/tsunagi/src/agent/network.rs +++ b/crates/tsunagi/src/agent/network.rs @@ -547,6 +547,23 @@ impl Runtime { } } + // Everybody the signed state says belongs here. Their records + // reached us through somebody, so we know they exist and who they + // are, even having never spoken to them; an id with no address is + // still dialable when the endpoint's own discovery can resolve + // one. This is what makes a network a mesh rather than a star + // around whoever was named on a command line. + for record in self.state.records() { + if let Ok(author) = record.author_id() + && author != self.local_id + { + candidates.push(Candidate::new( + iroh::EndpointAddr::new(author), + CandidateSource::Member, + )); + } + } + // A stale or missing cache only changes which candidates we try first. // It never bypasses authentication. for hint in self.params.storage.hints_for_network(self.network_id).await { @@ -574,6 +591,8 @@ impl Runtime { } self.start_dials(); + // Last, so it carries what this round learned. + self.introduce_peers(); } fn start_dials(&mut self) { @@ -1147,6 +1166,112 @@ impl Runtime { } } + /// Takes in what another member knows about the rest. + /// + /// Returns how many of them were new here, which is the only reason to + /// start dialling again straight away. + fn learn_peers(&mut self, hints: &[crate::proto::message::PeerHint]) -> usize { + let mut learned = 0; + for hint in hints { + let Ok(peer) = EndpointId::from_bytes(&hint.endpoint) else { + continue; + }; + if peer == self.local_id || self.sessions.contains_key(&peer) { + continue; + } + let mut addr = iroh::EndpointAddr::new(peer); + for text in &hint.addrs { + // The same decoder as the cache's hints: one spelling, one + // place it is read, nothing to drift. + if let Some(decoded) = decode_hint(peer, text) { + merge_addr(&mut addr, &decoded); + } + } + + let fresh = !self.candidate_addrs.contains_key(&peer); + self.candidate_addrs + .entry(peer) + .and_modify(|existing| merge_addr(existing, &addr)) + .or_insert(addr); + self.dial_states + .entry(peer) + .or_insert_with(|| DialState::new(CandidateSource::Introduced)); + if fresh { + learned += 1; + } + } + learned + } + + /// Tells the peers it is talking to about the peers it knows. + /// + /// A device told about one member ends up talking to all of them: a + /// network is a mesh, and which member happened to be named on a + /// command line should not decide who anybody talks to. Sent on every + /// round, because it is cheap, bounded and self-repairing. + fn introduce_peers(&mut self) { + let mut hints: Vec = Vec::new(); + + // What this agent can see for itself: where each peer it is + // talking to is right now. An agent that only ever accepts has no + // candidates of its own, and it is exactly the one everybody else + // was pointed at — so without this, introductions would come from + // nobody. + let mut observed: HashMap = HashMap::new(); + for session in self.sessions.values() { + let snapshot = crate::net::snapshot_connection(&session.conn); + let mut addr = iroh::EndpointAddr::new(session.peer); + for path in &snapshot.paths { + match &path.remote { + crate::net::PathAddr::Ip(socket) => addr = addr.with_ip_addr(*socket), + crate::net::PathAddr::Relay(url) => { + if let Ok(url) = url.parse() { + addr = addr.with_relay_url(url); + } + } + crate::net::PathAddr::Other(_) => {} + } + } + observed.insert(session.peer, addr); + } + for (peer, addr) in &self.candidate_addrs { + observed + .entry(*peer) + .and_modify(|existing| merge_addr(existing, addr)) + .or_insert_with(|| addr.clone()); + } + + for (peer, addr) in &observed { + if *peer == self.local_id { + continue; + } + // An id with no address at all is still worth passing on: the + // other side's own discovery may be able to resolve it. + hints.push(crate::proto::message::PeerHint { + endpoint: *peer.as_bytes(), + addrs: hint_addrs(addr), + }); + } + // This agent, too: it is the one member the others cannot be told + // about by anybody else. + hints.push(crate::proto::message::PeerHint { + endpoint: *self.local_id.as_bytes(), + addrs: hint_addrs(&self.params.adapter.addr()), + }); + hints.truncate(self.params.limits.max_state_records); + if hints.is_empty() { + return; + } + + let message = ControlMessage::Peers { peers: hints }; + let peers: Vec = self.sessions.keys().copied().collect(); + for peer in peers { + if let Err(err) = self.send_to(peer, message.clone()) { + tracing::debug!(%err, "could not queue an introduction"); + } + } + } + /// Tells peers which peers this agent has a live link with, when that /// has changed. /// @@ -1595,6 +1720,15 @@ impl Runtime { ControlMessage::State { records } => { self.pending_state.push((peer, records.clone())); } + // Members somebody else knows of. Candidates, nothing more: + // each still has to pass the handshake, and each is tried the + // same way as one from any other source. + ControlMessage::Peers { peers } => { + let learned = self.learn_peers(peers); + if learned > 0 { + self.start_dials(); + } + } // What that peer can reach, from that peer. Kept with the time // it was heard, so it can go stale on its own. ControlMessage::Reachable { peers } => { @@ -1820,6 +1954,23 @@ fn encode_hint(addr: &PathAddr) -> Option { } } +/// The addresses of a candidate, in the spelling hints use. +/// +/// Bounded and sorted, so what goes on the wire is the same for the same +/// address whatever order the transport happened to report it in. +fn hint_addrs(addr: &iroh::EndpointAddr) -> Vec { + let mut out: Vec = addr + .addrs + .iter() + .map(|transport| transport.to_string()) + .filter(|text| text.len() <= crate::proto::message::MAX_HINT_ADDR_LEN) + .collect(); + out.sort(); + out.dedup(); + out.truncate(crate::proto::message::MAX_HINT_ADDRS); + out +} + /// Decodes a cache hint back into an address. Malformed hints are ignored. fn decode_hint(endpoint_id: EndpointId, hint: &str) -> Option { if let Some(rest) = hint.strip_prefix("ip:") { diff --git a/crates/tsunagi/src/discovery.rs b/crates/tsunagi/src/discovery.rs index 34c60f6..c8c7239 100644 --- a/crates/tsunagi/src/discovery.rs +++ b/crates/tsunagi/src/discovery.rs @@ -44,6 +44,18 @@ pub enum CandidateSource { Discovery, /// An address hint restored from the disposable cache. Cache, + /// An author of a signed record: somebody who belongs to this network, + /// learned from state that reached us through anybody. + /// + /// It carries no address of its own — the endpoint's own discovery has + /// to resolve it — but knowing that a member exists is what turns a + /// star around whoever was named on the command line into a mesh. + Member, + /// Passed on by a member we are talking to. + /// + /// A candidate like any other: an introduction is not a vouching, and + /// membership is still decided by the handshake. + Introduced, } /// An unverified candidate peer. diff --git a/crates/tsunagi/src/proto/message.rs b/crates/tsunagi/src/proto/message.rs index 9e708bf..d6b1266 100644 --- a/crates/tsunagi/src/proto/message.rs +++ b/crates/tsunagi/src/proto/message.rs @@ -129,6 +129,18 @@ pub enum ControlMessage { /// The records. Bounded by [`crate::config::Limits::max_state_records`]. records: Vec, }, + /// Members this sender knows of, and where it has seen them. + /// + /// An introduction, not a vouching: everything here is an unverified + /// candidate, and membership is still decided by the handshake. It is + /// what turns a star around whoever was named on the command line into + /// a mesh — a device that was told about one member ends up talking to + /// all of them — and it is the same shape a lookup in a distributed + /// hash table would return. + Peers { + /// The members, with whatever addresses the sender has for them. + peers: Vec, + }, /// Which peers this sender has a live data link with, right now. /// /// First-hand and nothing else: a sender speaks only for itself, never @@ -152,6 +164,29 @@ pub enum ControlMessage { }, } +/// Somewhere a member has been seen. +/// +/// Addresses are in iroh's own text form, which is what the endpoint takes +/// back: this is a hint to try, never a fact about where anybody is. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PeerHint { + /// The member's endpoint key. + pub endpoint: [u8; 32], + /// Where the sender has seen it, as `ip:` or `relay:`. + /// + /// The same spelling the disposable cache uses, so one decoder serves + /// both and neither can drift from the other. An empty list is still + /// worth sending: an id alone is enough for a receiver whose endpoint + /// can resolve it. + pub addrs: Vec, +} + +/// Longest address text accepted in a hint. +pub const MAX_HINT_ADDR_LEN: usize = 128; + +/// Most addresses accepted for one member in a hint. +pub const MAX_HINT_ADDRS: usize = 8; + /// A control message together with the network it belongs to. /// /// Every session is bound to exactly one network at handshake time. The @@ -210,6 +245,15 @@ pub fn validate_capability( /// network, the other networks or the agent. pub fn validate(message: &ControlMessage, limits: &Limits) -> Result<(), ProtocolError> { match message { + ControlMessage::Peers { peers } => { + check_len("peers", peers.len(), limits.max_state_records)?; + for hint in peers { + check_len("peers.addrs", hint.addrs.len(), MAX_HINT_ADDRS)?; + for addr in &hint.addrs { + check_len("peers.addr", addr.len(), MAX_HINT_ADDR_LEN)?; + } + } + } ControlMessage::Reachable { peers } => { // One entry per member at most; a sender claiming more than a // network can hold is not describing this network. @@ -254,6 +298,7 @@ pub fn kind(message: &ControlMessage) -> &'static str { ControlMessage::Pong { .. } => "pong", ControlMessage::State { .. } => "state", ControlMessage::Reachable { .. } => "reachable", + ControlMessage::Peers { .. } => "peers", ControlMessage::Bye { .. } => "bye", } } diff --git a/crates/tsunagi/tests/discovery.rs b/crates/tsunagi/tests/discovery.rs index a3c6b5a..07b7864 100644 --- a/crates/tsunagi/tests/discovery.rs +++ b/crates/tsunagi/tests/discovery.rs @@ -137,3 +137,63 @@ async fn forgetting_a_network_removes_it_from_the_state_store() { drop(restarted); drop(dir); } + +#[tokio::test] +async fn being_told_about_one_member_is_enough_to_meet_them_all() { + // A network is a mesh, and which member happened to be named on a + // command line must not decide who a device ends up talking to. The + // newcomer is given one address and nothing else; everybody else is + // learned from the members it meets. + let (name, secret) = network("introductions"); + + // Three that already know each other, and nothing that resolves by + // network: no shared discovery here, only what members pass on. + let first_dir = tempfile::TempDir::new().unwrap(); + let first = Agent::spawn(local_config(first_dir.path())).await.unwrap(); + let network_id = first.join_network(&name, &secret).await.unwrap(); + + let second_dir = tempfile::TempDir::new().unwrap(); + let second = Agent::spawn( + local_config(second_dir.path()) + .with_discovery(Arc::new(StaticBootstrap::new([first.local_addr()]))), + ) + .await + .unwrap(); + second.join_network(&name, &secret).await.unwrap(); + wait_for_peers(&first, network_id, 1).await; + + let third_dir = tempfile::TempDir::new().unwrap(); + let third = Agent::spawn( + local_config(third_dir.path()) + .with_discovery(Arc::new(StaticBootstrap::new([first.local_addr()]))), + ) + .await + .unwrap(); + third.join_network(&name, &secret).await.unwrap(); + + // Each of the two newcomers was given only the first one's address, + // and each ends up with both of the others. + wait_for_peers(&second, network_id, 2).await; + wait_for_peers(&third, network_id, 2).await; + wait_for_peers(&first, network_id, 2).await; + + // And the one they were not told about is there as an introduction: + // a candidate like any other, still authenticated by the handshake. + let status = second.network_status(network_id).await.unwrap(); + let introduced = status + .candidates + .iter() + .find(|candidate| candidate.endpoint_id == third.endpoint_id()) + .map(|candidate| candidate.source); + assert!( + matches!( + introduced, + Some(CandidateSource::Introduced) | Some(CandidateSource::Member) + ), + "the third should have arrived from the others: {introduced:?}" + ); + + second.shutdown().await; + third.shutdown().await; + first.shutdown().await; +} diff --git a/docs/testing.md b/docs/testing.md index db2c92d..3c9d7ee 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -91,6 +91,10 @@ the way past, a name outside every zone is refused, each network gets a zone of its own as it is joined, and the resolver can be switched on and off while the agent runs. +`tests/discovery.rs` also covers introductions: a device given one +member's address meets every other member, and the ones it was not told +about arrive as candidates that still pass the handshake like any other. + `tests/discovery.rs` covers the discovery contract itself: a static bootstrap candidate is enough to join, several backends compose, entries are withdrawn when a network stops, and a forgotten network stays forgotten across a restart.