ab/tsunagi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Relay across any mix of links, one online answer for every client, first claim owns a hostname

Browse Source 
A tunnel no longer depends on the kind of link under it. The relay hub keeps
one forwarding table per network instead of one per protocol, the envelope
names the end-to-end protocol, and a neighbour reached over several kinds of
transport is sent to over the best open one. A member in the middle needs no
protocol in common with either end. Which protocol a pair runs is settled by
the lower endpoint id's announced order, so both ends agree without asking.
A direct link that comes up later is shorter than the path through others, so
traffic moves to it with the same logical link.

The agent now answers "is it online" once: a live control session or any data
path. Members and peers report how they are reached (direct over a named
transport, relayed through a named member, or no data path yet), and `tsng
status` and the tray both read that instead of a session list and a WireGuard
handshake respectively.

A hostname belongs to whoever claimed it first. A claim carries the time its
author first claimed the name, signed with the record, and replicas order by
(time, id) so every member names the same holder. A later claimant keeps
working by address, is not found by the name, and is told so in its status,
`tsng status`, the tray and a HostnameTaken event.

Wire change: control and data ALPN 5, control PROTOCOL_VERSION 5, envelope
version 2, signing domain v3, state schema 7, local control protocol 18.
Upgrade every member together.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Aleksandr BogomiakovandClaude committed 2026-10-05 11:39:33 +01:00
1 parent c184ef0914
commit 97377e3703
22 files changed
+1699 -292
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/tsunagi-core/src/dataplane/relay.rs
+259 -127
View File
@@ -67,10 +67,41 @@ impl Tally {
}
}
/// The links to one neighbour, best first. Which transport carries a frame
/// is decided here and nowhere else: the end-to-end protocol is in the
/// envelope and never depends on it.
#[derive(Debug)]
struct NextHop {
peer: PeerId,
link: SharedLink,
links: Box<[SharedLink]>,
}
impl NextHop {
fn usable(&self) -> bool {
self.links.iter().any(|link| !link.is_closed())
}
fn first_open(&self) -> Option<&SharedLink> {
self.links.iter().find(|link| !link.is_closed())
}
/// Hands a frame to the best open link that accepts it. A link that
/// cannot carry the frame (too large for its path, closed a moment ago)
/// leaves it to the next one; the frame is only copied when there is
/// another to try.
fn send(&self, frame: Bytes) -> Result<(), TransportError> {
let mut open = self.links.iter().filter(|link| !link.is_closed());
let Some(mut current) = open.next() else {
return Err(TransportError::Closed);
};
for next in open {
if current.send(frame.clone()).is_ok() {
return Ok(());
}
current = next;
}
current.send(frame)
}
}
#[derive(Debug)]
@@ -82,21 +113,22 @@ struct ForwardRoute {
impl ForwardRoute {
fn select(&self, source: &PeerId, flow: FlowId) -> Option<&NextHop> {
if self.next.len() == 1 {
return self.next.first().filter(|hop| !hop.link.is_closed());
return self.next.first().filter(|hop| hop.usable());
}
let seed = u64::from_le_bytes(source[..8].try_into().ok()?);
let start = mix64(flow ^ seed) as usize % self.next.len();
// Stable fallback if a link closed just before its table was replaced.
(0..self.next.len())
.map(|i| &self.next[(start + i) % self.next.len()])
.find(|hop| !hop.link.is_closed())
.find(|hop| hop.usable())
}
}
#[derive(Debug, Default)]
struct ForwardingTable {
routes: HashMap<PeerId, ForwardRoute>,
inboxes: HashMap<PeerId, mpsc::Sender<Bytes>>,
/// Local delivery, by who sent it and which end-to-end protocol it is.
inboxes: HashMap<(PeerId, u64), mpsc::Sender<Bytes>>,
members: HashSet<PeerId>,
}
@@ -121,7 +153,7 @@ impl Plane {
return;
}
if header.destination == self.local {
let Some(inbox) = table.inboxes.get(&header.source) else {
let Some(inbox) = table.inboxes.get(&(header.source, header.protocol)) else {
self.tally.dropped_unknown.fetch_add(1, Ordering::Relaxed);
return;
};
@@ -144,7 +176,7 @@ impl Plane {
self.tally.dropped_no_link.fetch_add(1, Ordering::Relaxed);
return;
};
if hop.link.send(envelope::decrement(frame)).is_ok() {
if hop.send(envelope::decrement(frame)).is_ok() {
self.tally.forwarded.fetch_add(1, Ordering::Relaxed);
} else {
self.tally.dropped_no_link.fetch_add(1, Ordering::Relaxed);
@@ -152,27 +184,65 @@ impl Plane {
}
}
/// One physical link to a neighbour, of whatever kind of transport.
#[derive(Debug)]
struct Protocol {
plane: Arc<Plane>,
graph: HashMap<PeerId, Vec<PeerId>>,
members: HashSet<PeerId>,
raw: HashMap<PeerId, SharedLink>,
readers: HashMap<PeerId, JoinHandle<()>>,
links: HashMap<PeerId, Arc<PeerLink>>,
struct RawLink {
kind: String,
link: SharedLink,
reader: JoinHandle<()>,
}
impl Protocol {
/// How a destination is reached right now.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RouteSummary {
/// Number of transport links to it. One is a direct link.
pub hops: u8,
/// The neighbour the first link goes to. For a direct link that is the
/// destination itself.
pub via: EndpointId,
}
#[derive(Debug)]
struct State {
plane: Arc<Plane>,
/// Transport kinds, best first. A neighbour with several links is
/// reached over the first of these it has.
priority: Vec<String>,
graph: HashMap<PeerId, Vec<PeerId>>,
members: HashSet<PeerId>,
raw: HashMap<PeerId, Vec<RawLink>>,
links: HashMap<(PeerId, String), Arc<PeerLink>>,
}
impl State {
fn rank(&self, kind: &str) -> usize {
self.priority
.iter()
.position(|known| known == kind)
.unwrap_or(usize::MAX)
}
/// A neighbour's links, best kind first.
fn ordered(&self, peer: &PeerId) -> Vec<&RawLink> {
let mut links: Vec<&RawLink> = self
.raw
.get(peer)
.map(|links| links.iter().collect())
.unwrap_or_default();
links.sort_by_key(|raw| (self.rank(&raw.kind), raw.kind.clone()));
links
}
fn publish(&self) {
let mut graph = self.graph.clone();
graph.insert(
self.plane.local,
self.raw
.iter()
.filter(|(_, link)| !link.is_closed())
.map(|(peer, _)| *peer)
.collect(),
);
let mut neighbours: Vec<PeerId> = self
.raw
.iter()
.filter(|(_, links)| links.iter().any(|raw| !raw.link.is_closed()))
.map(|(peer, _)| *peer)
.collect();
neighbours.sort_unstable();
graph.insert(self.plane.local, neighbours);
let routing = RoutingTable::build(self.plane.local, &graph, ROUTING_HOP_LIMIT);
let routes = routing
.iter()
@@ -181,10 +251,12 @@ impl Protocol {
.next_hops
.iter()
.filter_map(|id| {
self.raw.get(id).map(|link| NextHop {
peer: *id,
link: link.clone(),
})
let links: Box<[_]> = self
.ordered(id)
.into_iter()
.map(|raw| raw.link.clone())
.collect();
(!links.is_empty()).then_some(NextHop { peer: *id, links })
})
.collect();
(!next.is_empty()).then_some((
@@ -204,17 +276,21 @@ impl Protocol {
inboxes: self
.links
.iter()
.map(|(peer, link)| (*peer, link.inbox_tx.clone()))
.map(|((peer, _), link)| ((*peer, link.tag), link.inbox_tx.clone()))
.collect(),
}));
}
fn stop_readers(&mut self) {
for raw in self.raw.values().flatten() {
raw.reader.abort();
}
}
}
impl Drop for Protocol {
impl Drop for State {
fn drop(&mut self) {
for task in self.readers.values() {
task.abort();
}
self.stop_readers();
for link in self.links.values() {
link.close();
}
@@ -222,12 +298,17 @@ impl Drop for Protocol {
}
}
/// One network's transport links and independently published protocol tables.
/// One network's transport links and its published forwarding table.
///
/// There is one table for the whole network, whatever protocols run over it.
/// A link is a way to reach a neighbour; a protocol is an end-to-end channel
/// to a peer. The two meet only in the envelope, which names the protocol, so
/// a tunnel keeps working over any mix of transports and a middle member
/// needs no protocol in common with either end.
#[derive(Debug)]
pub struct RelayHub {
network: NetworkId,
local: PeerId,
protocols: Mutex<HashMap<String, Protocol>>,
state: Mutex<State>,
tally: Arc<Tally>,
changed: Arc<Notify>,
}
@@ -235,40 +316,38 @@ pub struct RelayHub {
impl RelayHub {
/// Creates the router at this network's local endpoint.
pub fn new(network: NetworkId, local: EndpointId) -> Arc<Self> {
let tally = Arc::new(Tally::default());
Arc::new(Self {
network,
local: *local.as_bytes(),
protocols: Mutex::default(),
tally: Arc::default(),
state: Mutex::new(State {
plane: Arc::new(Plane {
local: *local.as_bytes(),
table: ArcSwap::from_pointee(ForwardingTable::default()),
tally: tally.clone(),
}),
priority: Vec::new(),
graph: HashMap::new(),
members: HashSet::new(),
raw: HashMap::new(),
links: HashMap::new(),
}),
tally,
changed: Arc::default(),
})
}
fn protocols(&self) -> std::sync::MutexGuard<'_, HashMap<String, Protocol>> {
self.protocols
.lock()
.unwrap_or_else(|error| error.into_inner())
fn state(&self) -> std::sync::MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(|error| error.into_inner())
}
fn protocol<'a>(
&self,
protocols: &'a mut HashMap<String, Protocol>,
name: &str,
) -> &'a mut Protocol {
protocols
.entry(name.to_owned())
.or_insert_with(|| Protocol {
plane: Arc::new(Plane {
local: self.local,
table: ArcSwap::from_pointee(ForwardingTable::default()),
tally: self.tally.clone(),
}),
graph: HashMap::new(),
members: HashSet::new(),
raw: HashMap::new(),
readers: HashMap::new(),
links: HashMap::new(),
})
/// Sets which kinds of transport are preferred, best first, when a
/// neighbour can be reached over more than one.
pub fn set_priority(&self, kinds: Vec<String>) {
let mut state = self.state();
if state.priority != kinds {
state.priority = kinds;
state.publish();
}
}
/// Wakes the control loop promptly when a transport reader ends.
@@ -282,14 +361,11 @@ impl RelayHub {
}
/// Publishes authenticated topology. Refreshing identical state is cheap.
pub fn set_topology(
&self,
protocol: &str,
graph: HashMap<PeerId, Vec<PeerId>>,
members: HashSet<PeerId>,
) {
let mut protocols = self.protocols();
let state = self.protocol(&mut protocols, protocol);
///
/// Every row of `graph` is one member's own account of who it has a link
/// to; what kind of link it is does not matter here.
pub fn set_topology(&self, graph: HashMap<PeerId, Vec<PeerId>>, members: HashSet<PeerId>) {
let mut state = self.state();
if state.graph != graph || state.members != members {
state.graph = graph;
state.members = members;
@@ -298,22 +374,44 @@ impl RelayHub {
}
/// Whether the current table has a path to this destination.
pub fn reachable(&self, peer: EndpointId, protocol: &str) -> bool {
self.protocols().get(protocol).is_some_and(|state| {
state
.plane
.table
.load()
.routes
.contains_key(peer.as_bytes())
pub fn reachable(&self, peer: EndpointId) -> bool {
self.state()
.plane
.table
.load()
.routes
.contains_key(peer.as_bytes())
}
/// How a destination is reached right now, if it is.
pub fn route(&self, peer: EndpointId) -> Option<RouteSummary> {
let table = self.state().plane.table.load_full();
let route = table.routes.get(peer.as_bytes())?;
let via = EndpointId::from_bytes(&route.next.first()?.peer).ok()?;
Some(RouteSummary {
hops: route.hops,
via,
})
}
/// Stable logical link, preserved across topology changes.
/// The kinds of transport with an open direct link to a peer, best
/// first.
pub fn direct_kinds(&self, peer: EndpointId) -> Vec<String> {
let state = self.state();
state
.ordered(peer.as_bytes())
.into_iter()
.filter(|raw| !raw.link.is_closed())
.map(|raw| raw.kind.clone())
.collect()
}
/// Stable end-to-end channel for a protocol, preserved across topology
/// and transport changes.
pub fn link(&self, peer: EndpointId, protocol: &str) -> Arc<PeerLink> {
let mut protocols = self.protocols();
let state = self.protocol(&mut protocols, protocol);
if let Some(link) = state.links.get(peer.as_bytes()) {
let mut state = self.state();
let key = (*peer.as_bytes(), protocol.to_owned());
if let Some(link) = state.links.get(&key) {
return link.clone();
}
let (inbox_tx, inbox_rx) = mpsc::channel(INBOX);
@@ -321,88 +419,118 @@ impl RelayHub {
let link = Arc::new(PeerLink {
network: self.network,
peer,
tag: envelope::protocol_tag(protocol),
plane: state.plane.clone(),
inbox_tx,
inbox_rx: tokio::sync::Mutex::new(inbox_rx),
closed,
});
state.links.insert(*peer.as_bytes(), link.clone());
state.links.insert(key, link.clone());
state.publish();
link
}
/// Whether the protocol already owns its logical link.
/// Whether the protocol already owns its end-to-end channel to a peer.
pub fn has_link(&self, peer: EndpointId, protocol: &str) -> bool {
self.protocols()
.get(protocol)
.is_some_and(|state| state.links.contains_key(peer.as_bytes()))
self.state()
.links
.contains_key(&(*peer.as_bytes(), protocol.to_owned()))
}
/// Installs a raw transport link and starts its independent ingress reader.
pub fn set_direct(&self, peer: EndpointId, protocol: &str, raw: SharedLink) {
///
/// A neighbour may have one link per kind of transport; installing a
/// second of the same kind replaces the first.
pub fn set_direct(&self, peer: EndpointId, kind: &str, raw: SharedLink) {
if raw.network() != self.network || raw.peer() != peer {
return;
}
let mut protocols = self.protocols();
let state = self.protocol(&mut protocols, protocol);
let mut state = self.state();
let id = *peer.as_bytes();
if let Some(old) = state.readers.remove(&id) {
old.abort();
}
state.raw.insert(id, raw.clone());
state.publish();
let plane = state.plane.clone();
let changed = self.changed.clone();
state.readers.insert(
id,
tokio::spawn(async move {
let mut batch = 0;
while let Some(frame) = raw.recv().await {
plane.receive(id, frame);
batch += 1;
if batch == 64 {
tokio::task::yield_now().await;
batch = 0;
}
let reader_link = raw.clone();
let reader = tokio::spawn(async move {
let mut batch = 0;
while let Some(frame) = reader_link.recv().await {
plane.receive(id, frame);
batch += 1;
if batch == 64 {
tokio::task::yield_now().await;
batch = 0;
}
changed.notify_one();
}),
);
}
changed.notify_one();
});
let links = state.raw.entry(id).or_default();
if let Some(position) = links.iter().position(|known| known.kind == kind) {
links.remove(position).reader.abort();
}
links.push(RawLink {
kind: kind.to_owned(),
link: raw,
reader,
});
state.publish();
}
/// Removes a physical link without tearing down end-to-end protocol state.
pub fn clear_direct(&self, peer: EndpointId, protocol: &str) {
if let Some(state) = self.protocols().get_mut(protocol) {
state.raw.remove(peer.as_bytes());
if let Some(task) = state.readers.remove(peer.as_bytes()) {
task.abort();
/// Removes one physical link without tearing down end-to-end protocol state.
pub fn clear_direct(&self, peer: EndpointId, kind: &str) {
let mut state = self.state();
let mut changed = false;
if let Some(links) = state.raw.get_mut(peer.as_bytes()) {
if let Some(position) = links.iter().position(|known| known.kind == kind) {
links.remove(position).reader.abort();
changed = true;
}
if links.is_empty() {
state.raw.remove(peer.as_bytes());
}
}
if changed {
state.publish();
}
}
/// Revokes all links and topology involving a departed authenticated peer.
pub fn remove_peer(&self, peer: EndpointId) {
for state in self.protocols().values_mut() {
state.raw.remove(peer.as_bytes());
if let Some(task) = state.readers.remove(peer.as_bytes()) {
task.abort();
let mut state = self.state();
if let Some(links) = state.raw.remove(peer.as_bytes()) {
for raw in links {
raw.reader.abort();
}
if let Some(link) = state.links.remove(peer.as_bytes()) {
}
let gone: Vec<_> = state
.links
.keys()
.filter(|(id, _)| id == peer.as_bytes())
.cloned()
.collect();
for key in gone {
if let Some(link) = state.links.remove(&key) {
link.close();
}
state.members.remove(peer.as_bytes());
state.graph.remove(peer.as_bytes());
for neighbors in state.graph.values_mut() {
neighbors.retain(|id| id != peer.as_bytes());
}
state.publish();
}
state.members.remove(peer.as_bytes());
state.graph.remove(peer.as_bytes());
for neighbors in state.graph.values_mut() {
neighbors.retain(|id| id != peer.as_bytes());
}
state.publish();
}
/// Stops every reader and releases all transport handles.
pub fn close(&self) {
self.protocols().clear();
let mut state = self.state();
state.stop_readers();
state.raw.clear();
for link in state.links.values() {
link.close();
}
state.links.clear();
state.graph.clear();
state.members.clear();
state.plane.table.store(Arc::default());
}
}
@@ -411,6 +539,7 @@ impl RelayHub {
pub struct PeerLink {
network: NetworkId,
peer: EndpointId,
tag: u64,
plane: Arc<Plane>,
inbox_tx: mpsc::Sender<Bytes>,
inbox_rx: tokio::sync::Mutex<mpsc::Receiver<Bytes>>,
@@ -454,10 +583,11 @@ impl PacketLink for PeerLink {
let hop = route
.select(&self.plane.local, flow)
.ok_or(TransportError::Closed)?;
hop.link.send(envelope::encode(
hop.send(envelope::encode(
self.plane.local,
*self.peer.as_bytes(),
flow,
self.tag,
&payload,
))?;
if route.hops > 1 {
@@ -489,7 +619,9 @@ impl PacketLink for PeerLink {
fn path_description(&self) -> String {
let table = self.plane.table.load();
match table.routes.get(self.peer.as_bytes()) {
Some(route) if route.hops == 1 => route.next[0].link.path_description(),
Some(route) if route.hops == 1 => route.next[0]
.first_open()
.map_or_else(|| "unreachable".into(), |link| link.path_description()),
Some(route) => format!(
"relay {} hops via {} ({} equal paths)",
route.hops,
crates/tsunagi-core/src/dataplane/relay_tests.rs
+147 -16
View File
@@ -16,6 +16,9 @@ fn peer(seed: u8) -> EndpointId {
fn id(seed: u8) -> PeerId {
*peer(seed).as_bytes()
}
fn tag() -> u64 {
envelope::protocol_tag("ip")
}
#[derive(Debug)]
struct Wire {
@@ -78,9 +81,8 @@ impl PacketLink for Wire {
}
}
fn topology(hub: &RelayHub, protocol: &str, edges: &[(u8, &[u8])], members: &[u8]) {
fn topology(hub: &RelayHub, edges: &[(u8, &[u8])], members: &[u8]) {
hub.set_topology(
protocol,
edges
.iter()
.map(|(a, bs)| (id(*a), bs.iter().map(|b| id(*b)).collect()))
@@ -96,7 +98,7 @@ async fn ecmp_is_per_flow_direct_wins_and_existing_tunnel_survives_changes() {
let c = Wire::new(3, true);
hub.set_direct(peer(2), "ip", b.clone());
hub.set_direct(peer(3), "ip", c.clone());
topology(&hub, "ip", &[(2, &[4]), (3, &[4])], &[1, 2, 3, 4]);
topology(&hub, &[(2, &[4]), (3, &[4])], &[1, 2, 3, 4]);
let link = hub.link(peer(4), "ip");
let mut chosen = HashMap::new();
for _ in 0..8 {
@@ -132,7 +134,7 @@ async fn ecmp_is_per_flow_direct_wins_and_existing_tunnel_survives_changes() {
}
assert_eq!(c.take().len(), 100);
hub.clear_direct(peer(2), "ip");
topology(&hub, "ip", &[(3, &[])], &[1, 2, 3, 4]);
topology(&hub, &[(3, &[])], &[1, 2, 3, 4]);
assert!(link.send(Bytes::new()).is_err());
assert!(!link.is_closed());
assert!(Arc::ptr_eq(&link, &hub.link(peer(4), "ip")));
@@ -152,8 +154,8 @@ async fn transport_reader_forwards_without_any_plugin_reader_and_isolates_protoc
let c = Wire::new(3, true);
hub.set_direct(peer(1), "ip", a.clone());
hub.set_direct(peer(3), "ip", c.clone());
topology(&hub, "ip", &[(3, &[4])], &[1, 2, 3, 4]);
let frame = envelope::encode(id(1), id(4), 91, b"end-to-end encrypted");
topology(&hub, &[(3, &[4])], &[1, 2, 3, 4]);
let frame = envelope::encode(id(1), id(4), 91, tag(), b"end-to-end encrypted");
let pointer = frame.as_ptr();
a.feed.send(frame).await.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(1), async {
@@ -170,12 +172,136 @@ async fn transport_reader_forwards_without_any_plugin_reader_and_isolates_protoc
ROUTING_HOP_LIMIT - 1
);
assert_eq!(&sent[0][RELAY_OVERHEAD..], b"end-to-end encrypted");
let isolated = hub.link(peer(4), "other-ip");
assert!(isolated.send(Bytes::new()).is_err());
assert_eq!(hub.counters().forwarded, 1);
hub.close();
}
#[tokio::test]
async fn a_frame_is_delivered_only_to_the_protocol_it_names() {
let hub = RelayHub::new(network(), peer(2));
hub.set_direct(peer(1), "ip", Wire::new(1, true));
let ip = hub.link(peer(1), "ip");
let other = hub.link(peer(1), "other-ip");
let plane = ip.plane.clone();
plane.receive(id(1), envelope::encode(id(1), id(2), 0, tag(), b"for ip"));
plane.receive(
id(1),
envelope::encode(
id(1),
id(2),
0,
envelope::protocol_tag("other-ip"),
b"for other",
),
);
plane.receive(
id(1),
envelope::encode(
id(1),
id(2),
0,
envelope::protocol_tag("nobody-runs-this"),
b"for nobody",
),
);
assert_eq!(ip.recv().await.unwrap(), Bytes::from_static(b"for ip"));
assert_eq!(
other.recv().await.unwrap(),
Bytes::from_static(b"for other")
);
assert_eq!(hub.counters().dropped_unknown, 1);
hub.close();
}
/// A middle member joins two neighbours over different transports, and the
/// two ends share a protocol the middle has never heard of.
#[tokio::test]
async fn a_tunnel_crosses_a_middle_member_whose_links_are_of_different_kinds() {
let a = RelayHub::new(network(), peer(1));
let c = RelayHub::new(network(), peer(3));
let b = RelayHub::new(network(), peer(2));
let a_to_c = Wire::new(3, true);
let c_to_a = Wire::new(1, true);
let c_to_b = Wire::new(2, true);
let b_to_c = Wire::new(3, true);
a.set_direct(peer(3), "wg", a_to_c.clone());
c.set_direct(peer(1), "wg", c_to_a);
c.set_direct(peer(2), "wg-quic", c_to_b.clone());
b.set_direct(peer(3), "wg-quic", b_to_c.clone());
topology(&a, &[(3, &[1, 2])], &[1, 2, 3]);
topology(&b, &[(3, &[1, 2])], &[1, 2, 3]);
topology(&c, &[(1, &[3]), (2, &[3])], &[1, 2, 3]);
let outgoing = a.link(peer(2), "tunnel");
let incoming = b.link(peer(1), "tunnel");
outgoing
.send_flow(Bytes::from_static(b"sealed"), 5)
.unwrap();
let frame = a_to_c.take().remove(0);
// The middle forwards it onto its other kind of link without looking
// inside.
c.state().plane.receive(id(1), frame);
let frame = c_to_b.take().remove(0);
assert_eq!(c.counters().forwarded, 1);
b.state().plane.receive(id(3), frame);
assert_eq!(
incoming.recv().await.unwrap(),
Bytes::from_static(b"sealed")
);
assert_eq!(b.counters().received_via, 1);
assert_eq!(
b.route(peer(1)),
Some(RouteSummary {
hops: 2,
via: peer(3)
})
);
a.close();
b.close();
c.close();
}
#[tokio::test]
async fn a_neighbour_is_reached_over_the_preferred_kind_and_falls_back_when_it_is_lost() {
let hub = RelayHub::new(network(), peer(1));
hub.set_priority(vec!["wg".into(), "tcp-tls".into(), "wg-quic".into()]);
let slow = Wire::new(2, true);
let fast = Wire::new(2, true);
hub.set_direct(peer(2), "wg-quic", slow.clone());
let link = hub.link(peer(2), "tunnel");
link.send(Bytes::from_static(b"one")).unwrap();
assert_eq!(slow.take().len(), 1);
assert_eq!(hub.direct_kinds(peer(2)), ["wg-quic"]);
// A better kind appears: traffic moves to it with nothing to redo.
hub.set_direct(peer(2), "wg", fast.clone());
assert_eq!(hub.direct_kinds(peer(2)), ["wg", "wg-quic"]);
link.send(Bytes::from_static(b"two")).unwrap();
assert_eq!(fast.take().len(), 1);
assert!(slow.take().is_empty());
// It dies: the worse one carries on, and the logical link is untouched.
fast.dead.store(true, Ordering::Relaxed);
link.send(Bytes::from_static(b"three")).unwrap();
assert_eq!(slow.take().len(), 1);
assert!(!link.is_closed());
assert_eq!(
hub.route(peer(2)),
Some(RouteSummary {
hops: 1,
via: peer(2)
})
);
hub.clear_direct(peer(2), "wg-quic");
hub.clear_direct(peer(2), "wg");
assert!(!hub.reachable(peer(2)));
assert!(Arc::ptr_eq(&link, &hub.link(peer(2), "tunnel")));
hub.close();
}
#[tokio::test]
async fn local_delivery_is_bounded_and_malformed_or_unknown_sources_are_dropped() {
let hub = RelayHub::new(network(), peer(2));
@@ -184,18 +310,21 @@ async fn local_delivery_is_bounded_and_malformed_or_unknown_sources_are_dropped(
let link = hub.link(peer(1), "ip");
let plane = link.plane.clone();
for _ in 0..INBOX + 1 {
plane.receive(id(1), envelope::encode(id(1), id(2), 0, b"ip"));
plane.receive(id(1), envelope::encode(id(1), id(2), 0, tag(), b"ip"));
}
assert_eq!(hub.counters().dropped_congested, 1);
for _ in 0..INBOX {
assert_eq!(link.recv().await.unwrap(), Bytes::from_static(b"ip"));
}
plane.receive(id(1), Bytes::from_static(b"broken"));
plane.receive(id(1), envelope::encode(id(9), id(2), 0, b"unknown member"));
plane.receive(
id(1),
envelope::encode(id(9), id(2), 0, tag(), b"unknown member"),
);
assert_eq!(hub.counters().dropped_unknown, 2);
plane.receive(
id(1),
envelope::encode(id(1), id(9), 0, b"unknown destination"),
envelope::encode(id(1), id(9), 0, tag(), b"unknown destination"),
);
assert_eq!(hub.counters().dropped_no_link, 1);
hub.close();
@@ -210,11 +339,11 @@ async fn inconsistent_tables_cannot_loop_beyond_the_hop_limit() {
a.set_direct(peer(2), "ip", ab.clone());
b.set_direct(peer(1), "ip", ba.clone());
// Deliberately inconsistent snapshots while updates are in flight.
topology(&a, "ip", &[(2, &[4])], &[1, 2, 3, 4]);
topology(&b, "ip", &[(1, &[4])], &[1, 2, 3, 4]);
topology(&a, &[(2, &[4])], &[1, 2, 3, 4]);
topology(&b, &[(1, &[4])], &[1, 2, 3, 4]);
let ap = a.link(peer(4), "ip").plane.clone();
let bp = b.link(peer(4), "ip").plane.clone();
let mut frame = envelope::encode(id(3), id(4), 7, b"transit");
let mut frame = envelope::encode(id(3), id(4), 7, tag(), b"transit");
for turn in 0..ROUTING_HOP_LIMIT {
let (plane, incoming, wire) = if turn % 2 == 0 {
(&ap, id(2), &ab)
@@ -253,7 +382,7 @@ async fn forwarding_benchmark() {
hub.set_direct(peer(3), "ip", wire.clone());
let alternate = Wire::new(5, false);
hub.set_direct(peer(5), "ip", alternate.clone());
topology(&hub, "ip", &[(3, &[4]), (5, &[4])], &[1, 2, 3, 4, 5]);
topology(&hub, &[(3, &[4]), (5, &[4])], &[1, 2, 3, 4, 5]);
let plane = hub.link(peer(4), "ip").plane.clone();
let source = id(1);
let destination = id(4);
@@ -264,7 +393,9 @@ async fn forwarding_benchmark() {
const BATCHES: usize = 100;
for _ in 0..BATCHES {
let frames: Vec<_> = (0..COUNT)
.map(|flow| envelope::encode(source, destination, flow as u64, &vec![42; size]))
.map(|flow| {
envelope::encode(source, destination, flow as u64, tag(), &vec![42; size])
})
.collect();
let start = std::time::Instant::now();
for frame in frames {
crates/tsunagi-core/src/dataplane/routing/envelope.rs
+25 -5
View File
@@ -3,8 +3,8 @@ use super::{FlowId, PeerId};
use crate::config::{MAX_DATA_DATAGRAM, ROUTING_HOP_LIMIT};
use bytes::{BufMut, Bytes, BytesMut};
pub(crate) const HEADER: usize = 2 + 32 + 32 + 8;
const VERSION: u8 = 1;
pub(crate) const HEADER: usize = 2 + 32 + 32 + 8 + 8;
const VERSION: u8 = 2;
#[derive(Debug, PartialEq, Eq)]
pub(crate) struct Header {
@@ -12,6 +12,15 @@ pub(crate) struct Header {
pub destination: PeerId,
pub remaining: u8,
pub flow: FlowId,
/// Which end-to-end protocol the payload belongs to, whatever links carry it.
pub protocol: u64,
}
/// A stable 64-bit tag for a protocol id (FNV-1a), the same on every member.
pub(crate) fn protocol_tag(name: &str) -> u64 {
name.bytes().fold(0xcbf2_9ce4_8422_2325, |hash, byte| {
(hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
})
}
pub(crate) fn decode(bytes: &[u8]) -> Option<Header> {
@@ -28,16 +37,24 @@ pub(crate) fn decode(bytes: &[u8]) -> Option<Header> {
destination: bytes[34..66].try_into().ok()?,
remaining: bytes[1],
flow: u64::from_be_bytes(bytes[66..74].try_into().ok()?),
protocol: u64::from_be_bytes(bytes[74..82].try_into().ok()?),
})
}
pub(crate) fn encode(source: PeerId, destination: PeerId, flow: FlowId, payload: &[u8]) -> Bytes {
pub(crate) fn encode(
source: PeerId,
destination: PeerId,
flow: FlowId,
protocol: u64,
payload: &[u8],
) -> Bytes {
let mut bytes = BytesMut::with_capacity(HEADER + payload.len());
bytes.put_u8(VERSION);
bytes.put_u8(ROUTING_HOP_LIMIT);
bytes.extend_from_slice(&source);
bytes.extend_from_slice(&destination);
bytes.put_u64(flow);
bytes.put_u64(protocol);
bytes.extend_from_slice(payload);
bytes.freeze()
}
@@ -58,7 +75,7 @@ mod tests {
use super::*;
#[test]
fn transit_reuses_the_buffer_and_rejects_invalid_headers() {
let frame = encode([1; 32], [2; 32], 17, &[42; 1280]);
let frame = encode([1; 32], [2; 32], 17, protocol_tag("wg"), &[42; 1280]);
let address = frame.as_ptr();
let frame = decrement(frame);
assert_eq!(
@@ -66,7 +83,10 @@ mod tests {
address,
"no payload copy for an owned buffer"
);
assert_eq!(decode(&frame).unwrap().remaining, ROUTING_HOP_LIMIT - 1);
let header = decode(&frame).unwrap();
assert_eq!(header.remaining, ROUTING_HOP_LIMIT - 1);
assert_eq!(header.protocol, protocol_tag("wg"));
assert_ne!(protocol_tag("wg"), protocol_tag("wg-quic"));
assert_eq!(&frame[HEADER..], &[42; 1280]);
for length in 0..HEADER {
assert!(decode(&frame[..length]).is_none());