Rename the CLI package to tsunagi and its executable to tsng

The command line package is now `tsunagi` and builds the `tsng` executable,
which the packages, the unit, the installers and the workflows install and
run. The core library package becomes `tsunagi-core`, keeping the library name
`tsunagi`, so no code changes `use tsunagi::...`. Directories follow
(crates/tsunagi-core, crates/tsunagi). Every command shown to a person is now
`tsng ...`.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Aleksandr BogomiakovandClaude committed 2026-10-03 02:14:43 +01:00
1 parent bb38562b0a
commit 9966b50407
120 files changed
+631 -635

No files matched your search

+141
View File
@@ -0,0 +1,141 @@
//! Events published by a running agent.
//!
//! Events are delivered through a bounded [`tokio::sync::broadcast`] channel.
//! A slow subscriber is lagged, never allowed to stall the runtime.
//!
//! Nothing here ever carries a secret, a derived key or a handshake proof.
use std::time::Duration;
use iroh::EndpointId;
use crate::identity::NetworkId;
use crate::net::TransportKind;
use crate::proto::ControlMessage;
use crate::proto::handshake::Role;
/// Something that happened inside the agent.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub enum Event {
/// A network was activated locally.
NetworkActivated {
/// The network.
network: NetworkId,
},
/// A network was deactivated locally.
///
/// This is a local deactivation only. It is not a signed revocation of
/// membership, and it says nothing about the network's other participants.
NetworkDeactivated {
/// The network.
network: NetworkId,
},
/// A peer completed the handshake and has an authenticated session.
PeerConnected {
/// The network the session belongs to.
network: NetworkId,
/// Authenticated peer endpoint id.
peer: EndpointId,
/// Which side this agent played.
role: Role,
/// How the connection currently reaches the peer.
transport: TransportKind,
/// RTT of the selected path, if iroh reported one.
rtt: Option<Duration>,
},
/// A peer's session ended.
PeerDisconnected {
/// The network the session belonged to.
network: NetworkId,
/// The peer.
peer: EndpointId,
/// Why the session ended. Free of secrets.
reason: String,
},
/// A control message arrived on an authenticated session.
MessageReceived {
/// The network.
network: NetworkId,
/// The peer that sent it.
peer: EndpointId,
/// The message.
message: ControlMessage,
},
/// An outbound dial failed.
///
/// A dead candidate produces these and nothing else; other peers keep
/// connecting normally.
DialFailed {
/// The network.
network: NetworkId,
/// The candidate that could not be reached.
peer: EndpointId,
/// Why it failed.
reason: String,
},
/// A handshake was rejected.
///
/// The network is `None` when the failure happened before the peer's
/// requested network could be resolved.
HandshakeRejected {
/// The network, when known.
network: Option<NetworkId>,
/// The peer, when known.
peer: Option<EndpointId>,
/// Why it was rejected.
reason: String,
},
/// A message or session was rejected for violating the protocol.
ProtocolViolation {
/// The network, when known.
network: Option<NetworkId>,
/// The peer, when known.
peer: Option<EndpointId>,
/// What was wrong.
reason: String,
},
/// The disposable cache was discarded and recreated at startup.
CacheReset {
/// Why it was discarded. Free of secrets.
reason: String,
},
/// A data plane link to a peer is up.
///
/// The data plane is a separate connection from the control plane; this
/// says nothing about the control session, and vice versa.
DataLinkUp {
/// The network.
network: NetworkId,
/// The peer.
peer: EndpointId,
/// Plugin protocol the link carries.
protocol: String,
/// What the transport reports about the path in use.
path: String,
/// Largest datagram the link can carry.
max_datagram: usize,
},
/// A data plane link went away or could not be opened.
///
/// Never fatal: the control plane keeps running and the link is retried.
DataLinkDown {
/// The network.
network: NetworkId,
/// The peer.
peer: EndpointId,
/// Plugin protocol the link would have carried.
protocol: String,
/// Why it is not up.
reason: String,
},
/// An IP plugin reported an error. Never fatal.
PluginError {
/// The network the call was scoped to.
network: NetworkId,
/// Plugin protocol id.
protocol: String,
/// The reported error.
reason: String,
},
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+395
View File
@@ -0,0 +1,395 @@
//! One authenticated session with one peer, in one network.
//!
//! A session owns a reader task and a writer task over a single QUIC
//! bidirectional stream. Splitting them keeps both halves simple and avoids
//! cancelling a partially consumed frame, which stream reads do not tolerate.
//!
//! Every inbound message is re-checked against the session's network id, so an
//! authenticated session for one network can never deliver into another.
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use iroh::EndpointId;
use iroh::endpoint::{Connection, RecvStream, SendStream};
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use crate::config::Limits;
use crate::dataplane::PluginCapability;
use crate::error::ProtocolError;
use crate::identity::NetworkId;
use crate::proto::handshake::Role;
use crate::proto::message::{ControlMessage, Envelope, decode, validate};
use crate::proto::{read_frame, write_frame};
use super::shutdown::Shutdown;
/// What a session task reports back to its network runtime.
#[derive(Debug)]
pub(crate) enum SessionEvent {
/// A valid control message arrived.
Message {
/// Which session instance produced this.
session_id: u64,
/// The peer.
peer: EndpointId,
/// The decoded message.
message: ControlMessage,
/// Payload bytes read off the wire.
bytes: usize,
},
/// The peer sent something the protocol does not allow.
Violation {
/// Which session instance produced this.
session_id: u64,
/// The peer.
peer: EndpointId,
/// What was wrong.
error: ProtocolError,
},
/// The session ended.
Closed {
/// Which session instance ended.
session_id: u64,
/// The peer.
peer: EndpointId,
/// Why it ended.
reason: String,
},
}
/// A live session, as held by the network runtime.
#[derive(Debug)]
pub(crate) struct Session {
pub(crate) id: u64,
pub(crate) peer: EndpointId,
pub(crate) role: Role,
pub(crate) established: Instant,
pub(crate) conn: Connection,
/// Already encoded frame payloads, so the runtime can account for the exact
/// number of control bytes it queues.
pub(crate) outbound: mpsc::Sender<Vec<u8>>,
pub(crate) hostname: Option<String>,
pub(crate) capabilities: Vec<PluginCapability>,
pub(crate) broadcast: bool,
/// The peer offers itself as an exit node in this network.
pub(crate) exit_node: bool,
pub(crate) messages_sent: u64,
pub(crate) messages_received: u64,
pub(crate) bytes_sent: u64,
pub(crate) bytes_received: u64,
reader: JoinHandle<()>,
writer: JoinHandle<()>,
path_monitor: JoinHandle<()>,
shutdown: Shutdown,
}
impl Session {
/// Signals all tasks to stop and waits for them, with a bounded grace
/// period.
///
/// A peer that stops reading must not be able to hold up shutdown, so the
/// tasks are aborted if they do not wind down in time.
pub(crate) async fn stop(self) {
let Session {
conn,
reader,
writer,
path_monitor,
shutdown,
..
} = self;
shutdown.trigger();
// Closing the connection unblocks a reader parked on the stream.
conn.close(0u32.into(), b"session stopped by local agent");
let reader_abort = reader.abort_handle();
let writer_abort = writer.abort_handle();
let path_abort = path_monitor.abort_handle();
let joined = tokio::time::timeout(STOP_GRACE, async move {
let _ = reader.await;
let _ = writer.await;
let _ = path_monitor.await;
})
.await;
if joined.is_err() {
tracing::debug!("session tasks did not wind down in time; aborting them");
reader_abort.abort();
writer_abort.abort();
path_abort.abort();
}
}
/// Aborts all tasks without waiting. Used on replacement.
pub(crate) fn abort(&self) {
self.shutdown.trigger();
self.reader.abort();
self.writer.abort();
self.path_monitor.abort();
}
}
/// How long a stopping session may take to wind down before its tasks are
/// aborted. Shutdown must be bounded even if a peer stops reading.
const STOP_GRACE: std::time::Duration = std::time::Duration::from_millis(500);
/// Source of monotonically increasing session instance ids.
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
/// Starts the reader and writer tasks for an authenticated stream.
#[allow(clippy::too_many_arguments)]
pub(crate) fn spawn(
network_id: NetworkId,
peer: EndpointId,
role: Role,
conn: Connection,
send: SendStream,
recv: RecvStream,
limits: Arc<Limits>,
events: mpsc::Sender<SessionEvent>,
parent_shutdown: Shutdown,
) -> Session {
let id = NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed);
let shutdown = Shutdown::new();
let (outbound_tx, outbound_rx) = mpsc::channel(limits.session_send_queue);
let writer = tokio::spawn(writer_task(
send,
outbound_rx,
Arc::clone(&limits),
shutdown.clone(),
parent_shutdown.clone(),
));
let path_conn = conn.clone();
let path_shutdown = shutdown.clone();
let path_parent = parent_shutdown.clone();
let path_monitor = tokio::spawn(async move {
use tokio_stream::StreamExt;
let mut events = path_conn.path_events();
loop {
tokio::select! {
biased;
_ = path_shutdown.wait() => break,
_ = path_parent.wait() => break,
ev = events.next() => {
match ev {
Some(iroh::endpoint::PathEvent::Selected { remote_addr, local_addr, .. }) => {
tracing::debug!(
peer = %peer.fmt_short(),
remote = %remote_addr,
local = ?local_addr,
"Connection path selected (direct / relay / hole punch switch)"
);
}
Some(iroh::endpoint::PathEvent::Opened { remote_addr, .. }) => {
tracing::debug!(
peer = %peer.fmt_short(),
remote = %remote_addr,
"Network path opened to peer"
);
}
Some(iroh::endpoint::PathEvent::Closed { remote_addr, .. }) => {
tracing::debug!(
peer = %peer.fmt_short(),
remote = %remote_addr,
"Network path closed for peer"
);
}
Some(iroh::endpoint::PathEvent::Lagged { missed, .. }) => {
tracing::trace!(peer = %peer.fmt_short(), missed, "Path events lagged");
}
Some(_) => {}
None => break,
}
}
}
}
});
let reader = tokio::spawn(reader_task(
id,
network_id,
peer,
recv,
limits,
events,
shutdown.clone(),
parent_shutdown,
));
Session {
id,
peer,
role,
established: Instant::now(),
conn,
outbound: outbound_tx,
hostname: None,
capabilities: Vec::new(),
broadcast: false,
exit_node: false,
messages_sent: 0,
messages_received: 0,
bytes_sent: 0,
bytes_received: 0,
reader,
writer,
path_monitor,
shutdown,
}
}
async fn writer_task(
mut send: SendStream,
mut outbound: mpsc::Receiver<Vec<u8>>,
limits: Arc<Limits>,
shutdown: Shutdown,
parent: Shutdown,
) {
loop {
let encoded = tokio::select! {
biased;
_ = shutdown.wait() => break,
_ = parent.wait() => break,
encoded = outbound.recv() => match encoded {
Some(encoded) => encoded,
None => break,
},
};
// The write must stay cancellable: shutting the session down cannot
// wait for a peer that has stopped reading. Abandoning a half written
// frame is fine, because the session is going away with it.
let write = tokio::time::timeout(
limits.write_timeout,
write_frame(&mut send, &encoded, limits.max_frame_len),
);
let result = tokio::select! {
biased;
_ = shutdown.wait() => break,
_ = parent.wait() => break,
result = write => result,
};
match result {
Ok(Ok(())) => {}
Ok(Err(err)) => {
tracing::debug!(%err, "control stream write failed");
break;
}
Err(_) => {
tracing::debug!("control stream write timed out");
break;
}
}
}
let _ = send.finish();
}
#[allow(clippy::too_many_arguments)]
async fn reader_task(
session_id: u64,
network_id: NetworkId,
peer: EndpointId,
mut recv: RecvStream,
limits: Arc<Limits>,
events: mpsc::Sender<SessionEvent>,
shutdown: Shutdown,
parent: Shutdown,
) {
let reason = loop {
let frame = tokio::select! {
biased;
_ = shutdown.wait() => break "stopped locally".to_string(),
_ = parent.wait() => break "network deactivated".to_string(),
frame = read_frame(&mut recv, limits.max_frame_len) => frame,
};
let payload = match frame {
Ok(payload) => payload,
Err(ProtocolError::StreamClosed) => break "peer closed the control stream".to_string(),
Err(ProtocolError::Stream(err)) => {
// The transport went away. That is a disconnect, not a peer
// misbehaving, so it ends the session without being counted as
// a protocol violation.
break format!("control stream error: {err}");
}
Err(err) => {
// A framing violation ends this session. Framing errors are not
// recoverable mid-stream: the next bytes have no known meaning.
let text = err.to_string();
let _ = events
.send(SessionEvent::Violation {
session_id,
peer,
error: err,
})
.await;
break text;
}
};
let bytes = payload.len();
let envelope: Envelope = match decode(&payload) {
Ok(envelope) => envelope,
Err(err) => {
let _ = events
.send(SessionEvent::Violation {
session_id,
peer,
error: err,
})
.await;
break "malformed control frame".to_string();
}
};
// Network isolation: a session authenticated for one network must never
// deliver a message belonging to another.
if envelope.network_id != *network_id.as_bytes() {
let _ = events
.send(SessionEvent::Violation {
session_id,
peer,
error: ProtocolError::NetworkMismatch,
})
.await;
break "network id mismatch on an authenticated session".to_string();
}
if let Err(err) = validate(&envelope.message, &limits) {
let _ = events
.send(SessionEvent::Violation {
session_id,
peer,
error: err,
})
.await;
continue;
}
if events
.send(SessionEvent::Message {
session_id,
peer,
message: envelope.message,
bytes,
})
.await
.is_err()
{
break "network runtime stopped".to_string();
}
};
// Best effort: the runtime may already have stopped draining this channel
// while it tears the network down, and a closing session must not block on
// that.
let _ = events.try_send(SessionEvent::Closed {
session_id,
peer,
reason,
});
}
+75
View File
@@ -0,0 +1,75 @@
//! A minimal cancellation primitive.
//!
//! Kept local so the crate does not pull in a utility dependency for one type,
//! and so that no global state is involved: every agent and every network
//! runtime owns its own token.
use std::sync::Arc;
use tokio::sync::watch;
/// A clonable cancellation token.
#[derive(Debug, Clone)]
pub(crate) struct Shutdown {
tx: Arc<watch::Sender<bool>>,
rx: watch::Receiver<bool>,
}
impl Shutdown {
/// Creates an untriggered token.
pub(crate) fn new() -> Self {
let (tx, rx) = watch::channel(false);
Self {
tx: Arc::new(tx),
rx,
}
}
/// Triggers cancellation. Idempotent.
pub(crate) fn trigger(&self) {
let _ = self.tx.send(true);
}
/// Whether cancellation has been triggered.
pub(crate) fn is_triggered(&self) -> bool {
*self.rx.borrow()
}
/// Resolves once cancellation has been triggered.
pub(crate) async fn wait(&self) {
let mut rx = self.rx.clone();
{
if *rx.borrow() {
return;
}
}
while rx.changed().await.is_ok() {
if *rx.borrow() {
return;
}
}
// The sender is gone, which for our purposes means "stop".
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[tokio::test]
async fn waiting_resolves_once_triggered_and_stays_resolved() {
let shutdown = Shutdown::new();
assert!(!shutdown.is_triggered());
let waiter = shutdown.clone();
let waiting = tokio::spawn(async move { waiter.wait().await });
shutdown.trigger();
waiting.await.unwrap();
assert!(shutdown.is_triggered());
// Triggering twice is not an error, and a token cloned afterwards
// sees the trigger that already happened.
shutdown.trigger();
shutdown.clone().wait().await;
}
}
+241
View File
@@ -0,0 +1,241 @@
//! Status snapshots.
//!
//! Metrics are reported at the level they actually belong to. Values that
//! genuinely cannot be attributed to a single network — everything the iroh
//! endpoint aggregates, for instance — stay at the endpoint level rather than
//! being split between networks with invented precision.
use std::time::Duration;
use iroh::{EndpointAddr, EndpointId};
use crate::dataplane::PluginCapability;
use crate::discovery::CandidateSource;
use crate::identity::{NetworkDescriptor, NetworkId, NetworkName};
use crate::net::{ConnectionCounters, PathAddr, PathInfo, TransportKind};
use crate::proto::handshake::Role;
use crate::storage::CacheOutcome;
/// Whether a configured network is running locally.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NetworkState {
/// Configured and running.
Active,
/// Configured but not running.
Inactive,
}
/// An unverified candidate as seen by a network runtime.
#[derive(Debug, Clone)]
pub struct CandidateStatus {
/// Candidate endpoint id.
pub endpoint_id: EndpointId,
/// Where it came from.
pub source: CandidateSource,
/// Consecutive failed dial attempts since the last success.
pub consecutive_failures: u32,
}
/// The overlay interface, as the agent sees it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OverlayStatus {
/// The interface name the operating system gave.
pub interface: String,
/// Whether that interface exists on the host.
///
/// `false` with an in-memory device: the name is real to the agent and
/// to nothing else. Reported, because everything that would configure
/// the operating system for this interface must not when it is this.
pub on_host: bool,
/// Its MTU.
pub mtu: u32,
/// The addresses it should be carrying.
pub addresses: Vec<crate::overlay::Cidr>,
/// What has happened on it.
pub counters: crate::overlay::Counters,
/// The route and firewall allowance installed for LAN broadcast, when
/// some network has broadcast on and the interface is on the host.
/// `None` is "none installed", not "unknown".
pub broadcast_rules: Option<crate::overlay::BroadcastRulesReport>,
}
/// A member the signed state knows about, connected or not.
///
/// This is the durable roster: it comes from signed records, so a member that
/// went away last month is still here. That is what makes it possible to say
/// "this peer is offline" rather than only "nobody is connected".
///
/// It is not a complete membership list, and cannot be. A member is in signed
/// state once it has claimed something — an address, a name, or both — so a
/// member that has joined but never published is visible only while it is
/// connected.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemberStatus {
/// The member's device identity.
pub endpoint_id: EndpointId,
/// The IPv4 overlay address it claimed and signed for.
pub overlay_address_v4: Option<std::net::Ipv4Addr>,
/// The name it claimed, when it holds that name uncontested.
///
/// Signed, so it is still known while the member is away — which is what
/// lets an absent member be named rather than shown as a bare id.
pub hostname: Option<String>,
/// The last name this agent saw it announce, whether or not it is here.
///
/// Not signed, so it is a convenience for naming a member that is away
/// and never a claim: the zone is served from `hostname`.
pub last_hostname: Option<String>,
}
/// Status of one authenticated session.
#[derive(Debug, Clone)]
pub struct PeerStatus {
/// This authenticated peer accepts network broadcasts.
pub broadcast: bool,
/// This authenticated peer offers itself as an exit node.
pub exit_node: bool,
/// Authenticated endpoint id.
pub endpoint_id: EndpointId,
/// Which side this agent played in the handshake.
pub role: Role,
/// Hostname the peer announced, if it has announced one yet.
///
/// A mutable binding, not an identity.
pub hostname: Option<String>,
/// Protocols agreed with this peer: both sides have them at the same
/// version.
///
/// Empty means no data plane with that peer, which leaves the control
/// plane working — messages and state still flow.
pub protocols: Vec<String>,
/// Capabilities the peer announced. Payloads stay opaque.
pub capabilities: Vec<PluginCapability>,
/// How long the session has been up.
pub connected_for: Duration,
/// Verified paths of the underlying connection.
pub paths: Vec<PathInfo>,
/// How the connection currently reaches the peer.
pub transport: TransportKind,
/// RTT of the selected path, when iroh reported one.
pub rtt: Option<Duration>,
/// Per-connection counters.
pub connection: ConnectionCounters,
/// Control messages sent on this session.
pub control_messages_sent: u64,
/// Control messages received on this session.
pub control_messages_received: u64,
/// Control payload bytes queued for this session.
pub control_bytes_sent: u64,
/// Control payload bytes read from this session.
pub control_bytes_received: u64,
}
/// Counters scoped to one logical network.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct NetworkMetrics {
/// Outbound dial attempts started.
pub dial_attempts: u64,
/// Outbound dials that failed before or during the handshake.
pub dial_failures: u64,
/// Handshakes rejected, in either direction.
pub handshake_failures: u64,
/// Sessions that reached the authenticated state.
pub sessions_established: u64,
/// Sessions that ended.
pub disconnects: u64,
/// Control messages sent in this network.
pub control_messages_sent: u64,
/// Control messages received in this network.
pub control_messages_received: u64,
/// Control bytes sent in this network, payload only.
pub control_bytes_sent: u64,
/// Control bytes received in this network, payload only.
pub control_bytes_received: u64,
/// Messages or sessions rejected for protocol violations.
pub protocol_violations: u64,
/// Errors reported by IP plugins. Never fatal.
pub plugin_errors: u64,
/// Data plane links that were established.
pub data_links_established: u64,
/// Attempts to open a data plane link that failed.
pub data_link_failures: u64,
}
/// Status of one network.
#[derive(Debug, Clone)]
pub struct NetworkStatus {
/// Local broadcast participation in this network.
pub broadcast: bool,
/// This agent offers itself as an exit node in this network.
pub exit_offer: bool,
/// The member this device sends all its internet traffic through.
pub exit_via: Option<EndpointId>,
/// Whether that member is connected and still offering to be one.
pub exit_via_online: bool,
/// What the host rules for the exit settings are doing.
pub exit_rules: crate::overlay::ExitRulesReport,
/// Immutable deterministic description of the network space.
pub descriptor: NetworkDescriptor,
/// Network name, for convenience.
pub name: NetworkName,
/// Public network identifier.
pub network_id: NetworkId,
/// Whether the network is running locally.
pub state: NetworkState,
/// Authenticated sessions.
pub peers: Vec<PeerStatus>,
/// Unverified candidates currently known. Not peers.
pub candidates: Vec<CandidateStatus>,
/// Members the signed state knows about, whether connected or not.
pub members: Vec<MemberStatus>,
/// The overlay range this network uses, once it has one.
pub range: Option<crate::state::Ipv4Range>,
/// The range it could not have, because another network on this agent
/// already holds it.
///
/// One agent has one interface, so an address belongs to one network.
/// This network waits to adopt whatever its members settle on instead of
/// proposing something it could not route.
pub range_conflict: Option<crate::state::Ipv4Range>,
/// Per-network counters.
pub metrics: NetworkMetrics,
/// What has gone through a peer in the middle, in both directions.
pub relay: crate::dataplane::relay::RelayCounters,
}
impl NetworkStatus {
/// Endpoint ids of peers with an authenticated session.
pub fn connected_peers(&self) -> Vec<EndpointId> {
self.peers.iter().map(|peer| peer.endpoint_id).collect()
}
}
/// Status of the whole agent.
#[derive(Debug, Clone)]
pub struct AgentStatus {
/// This device's persistent endpoint id.
pub endpoint_id: EndpointId,
/// Hostname announced to peers.
pub hostname: String,
/// Sockets actually bound.
pub bound_sockets: Vec<std::net::SocketAddr>,
/// Addresses iroh believes this endpoint has. Observed, not verified.
pub observed_addrs: Vec<PathAddr>,
/// The dialable address of this endpoint, as iroh currently reports it.
pub endpoint_addr: EndpointAddr,
/// What happened to the disposable cache at startup.
pub cache_outcome: CacheOutcome,
/// Whether the cache is currently usable.
pub cache_healthy: bool,
/// Per-network status, including configured but inactive networks.
pub networks: Vec<NetworkStatus>,
}
impl AgentStatus {
/// Looks up one network's status.
pub fn network(&self, network_id: &NetworkId) -> Option<&NetworkStatus> {
self.networks
.iter()
.find(|status| &status.network_id == network_id)
}
}
+519
View File
@@ -0,0 +1,519 @@
//! Library configuration.
//!
//! Everything the agent needs is passed in explicitly. The library reads no
//! environment variables, installs no global state and picks no default
//! directories behind the caller's back — [`StoragePaths::user_default`] exists
//! but must be called on purpose.
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use crate::dataplane::SharedPlugin;
use crate::discovery::{MainlineDiscovery, NetworkDiscovery};
use crate::error::Result;
/// Qualifier/organisation/application triple used for platform directories.
///
/// Unused on macOS, where the default paths are fixed system locations rather
/// than per-user directories derived from this.
#[cfg_attr(target_os = "macos", allow(dead_code))]
const APP_NAME: &str = "tsunagi";
/// Rendezvous slots per network; this bounds a discovery sample, not membership.
pub const DHT_SLOTS: u8 = 16;
/// BEP44 permits 1000 bencoded bytes; a 996-byte string has a four-byte prefix.
pub const DHT_MAX_VALUE: usize = 996;
/// Maximum direct addresses in one rendezvous record.
pub const DHT_MAX_ADDRS: usize = 8;
/// Maximum relay URL bytes in a rendezvous record.
pub const DHT_MAX_RELAY_LEN: usize = 512;
/// Application freshness, independent of storage nodes' own expiration.
pub const DHT_RECORD_TTL: Duration = Duration::from_secs(900);
/// Clock skew tolerated when reading a recently published record.
pub const DHT_CLOCK_SKEW: Duration = Duration::from_secs(120);
/// Maximum reassembled data-transport payload, independent of the path MTU.
pub const MAX_DATA_DATAGRAM: usize = 64 * 1024;
/// Maximum incomplete datagrams held by one authenticated data link.
pub const MAX_DATA_ASSEMBLIES: usize = 64;
/// Maximum allocated payload bytes for incomplete datagrams on one data link.
pub const MAX_DATA_REASSEMBLY_BYTES: usize = 256 * 1024;
/// Maximum fragments of one logical datagram, including after a path MTU change.
pub const MAX_DATA_FRAGMENTS: usize = 128;
/// Time allowed to assemble a datagram; loss never stalls later datagrams.
pub const DATA_REASSEMBLY_TTL: Duration = Duration::from_secs(5);
/// Completed or discarded packet IDs retained to ignore late duplicate fragments.
pub const DATA_RECENT_IDS: usize = 128;
/// Where the two stores live.
///
/// The mandatory state and the disposable cache are separate both logically and
/// physically, so that the cache can be deleted at any time without touching
/// identity or network configuration.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StoragePaths {
/// Directory holding `state.sqlite` and the ownership lock.
pub state_dir: PathBuf,
/// Directory holding `cache.sqlite`.
pub cache_dir: PathBuf,
}
impl StoragePaths {
/// Uses explicit directories. Tests always use temporary directories.
pub fn new(state_dir: impl Into<PathBuf>, cache_dir: impl Into<PathBuf>) -> Self {
Self {
state_dir: state_dir.into(),
cache_dir: cache_dir.into(),
}
}
/// Puts both stores under one root, in `state/` and `cache/` subdirectories.
pub fn under(root: impl AsRef<Path>) -> Self {
let root = root.as_ref();
Self {
state_dir: root.join("state"),
cache_dir: root.join("cache"),
}
}
/// The default platform directories.
///
/// Per user on Linux and Windows. On macOS the agent runs as a root
/// LaunchDaemon (creating a utun interface needs root, and `$HOME` under
/// `sudo` points at the wrong place), so the stores live in the system
/// locations instead — `/var/db/tsunagi` for the mandatory state and a
/// disposable cache beside it. A caller that wants somewhere else passes
/// explicit paths to [`StoragePaths::new`] or [`StoragePaths::under`].
pub fn user_default() -> Result<Self> {
#[cfg(target_os = "macos")]
{
Ok(Self {
state_dir: PathBuf::from("/var/db/tsunagi"),
cache_dir: PathBuf::from("/var/db/tsunagi/cache"),
})
}
#[cfg(not(target_os = "macos"))]
{
let dirs = directories::ProjectDirs::from("", "", APP_NAME).ok_or_else(|| {
crate::error::Error::Storage(
"no valid home directory for platform config paths".into(),
)
})?;
Ok(Self {
state_dir: dirs.data_dir().to_path_buf(),
cache_dir: dirs.cache_dir().to_path_buf(),
})
}
}
/// Path of the mandatory state database.
pub fn state_db(&self) -> PathBuf {
self.state_dir.join("state.sqlite")
}
/// Path of the disposable cache database.
pub fn cache_db(&self) -> PathBuf {
self.cache_dir.join("cache.sqlite")
}
/// Path of the ownership lock file.
pub fn lock_file(&self) -> PathBuf {
self.state_dir.join("state.lock")
}
}
/// How the iroh endpoint is allowed to reach the outside world.
///
/// The default is [`TransportPolicy::LocalOnly`] so that a plain
/// `AgentConfig::new(...)` never reaches the internet by accident. Callers that
/// want public connectivity must opt in.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum TransportPolicy {
/// No relays, no address lookup service, no port mapping.
///
/// Suitable for tests and for fully local deployments.
#[default]
LocalOnly,
/// Public address lookup, but no relays.
///
/// Enables iroh's DNS/pkarr address lookup against the public service run
/// by Number 0 (the company behind iroh) at `dns.iroh.link`. This endpoint
/// publishes a signed record of its own addresses there, so peers can dial
/// it by endpoint id alone.
DirectOnly,
/// iroh's standard behaviour: public address lookup plus public relays.
///
/// Maps to iroh's own `presets::N0`. As well as the address lookup above,
/// it uses Number 0's public relay servers as a fallback when a direct
/// path cannot be hole punched. They are fine for development and carry no
/// availability guarantee.
N0Defaults,
}
/// Bounds applied to everything that comes off the network.
///
/// Each of these is enforced before memory is allocated for the corresponding
/// object where that is possible (notably [`Limits::max_frame_len`]).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Limits {
/// Largest accepted control frame payload, in bytes.
pub max_frame_len: usize,
/// Largest accepted hostname, in bytes.
pub max_hostname_len: usize,
/// Largest number of plugin capabilities in one announcement.
pub max_capabilities: usize,
/// Largest opaque plugin payload, in bytes.
pub max_capability_data_len: usize,
/// Largest accepted echo payload in a ping/pong exchange, in bytes.
pub max_echo_payload_len: usize,
/// Largest accepted free-text reason string, in bytes.
pub max_reason_len: usize,
/// Largest number of signed records accepted in one snapshot.
pub max_state_records: usize,
/// Deadline for the whole handshake.
pub handshake_timeout: Duration,
/// Deadline for one outbound dial attempt.
pub dial_timeout: Duration,
/// Deadline for writing one control frame.
///
/// Liveness of an established session is delegated to QUIC: iroh configures
/// keep-alives and an idle timeout, so a dead peer surfaces as a read error
/// rather than needing a protocol-level heartbeat here.
pub write_timeout: Duration,
/// Maximum simultaneous outbound dials per network.
pub max_concurrent_dials: usize,
/// Maximum unverified discovery candidates retained per network.
pub max_discovery_candidates: usize,
/// Maximum simultaneous authenticated sessions per network.
pub max_sessions_per_network: usize,
/// Maximum simultaneous inbound connections being handshaken.
pub max_inbound_handshakes: usize,
/// Capacity of a session's outbound queue, providing backpressure.
pub session_send_queue: usize,
/// Capacity of the event broadcast channel.
pub event_buffer: usize,
/// Maximum address hints kept per peer in the cache.
pub max_hints_per_peer: usize,
}
impl Default for Limits {
fn default() -> Self {
Self {
max_frame_len: 64 * 1024,
max_hostname_len: 255,
max_capabilities: 16,
max_capability_data_len: 4 * 1024,
max_echo_payload_len: 4 * 1024,
max_reason_len: 256,
max_state_records: crate::state::MAX_RECORDS_PER_MESSAGE,
handshake_timeout: Duration::from_secs(10),
dial_timeout: Duration::from_secs(10),
write_timeout: Duration::from_secs(30),
max_concurrent_dials: 8,
max_discovery_candidates: 16,
max_sessions_per_network: 64,
max_inbound_handshakes: 32,
session_send_queue: 64,
event_buffer: 512,
max_hints_per_peer: 8,
}
}
}
/// Bounded exponential backoff with jitter for reconnect attempts.
#[derive(Debug, Clone, PartialEq)]
pub struct ReconnectPolicy {
/// Delay before the first retry.
pub initial_delay: Duration,
/// Upper bound on the delay.
pub max_delay: Duration,
/// Multiplier applied after each failed attempt.
pub factor: f64,
/// Fraction of the delay applied as random jitter, in `0.0..=1.0`.
pub jitter: f64,
/// Give up on a peer after this many consecutive failures until it is seen
/// again by discovery. `None` means never give up while the network is up.
pub max_consecutive_failures: Option<u32>,
}
impl Default for ReconnectPolicy {
fn default() -> Self {
Self {
initial_delay: Duration::from_millis(250),
max_delay: Duration::from_secs(30),
factor: 2.0,
jitter: 0.3,
max_consecutive_failures: None,
}
}
}
impl ReconnectPolicy {
/// Delay to wait before retry number `attempt` (1-based), with jitter.
pub(crate) fn delay_for(&self, attempt: u32) -> Duration {
let exp = self.factor.powi(attempt.saturating_sub(1).min(32) as i32);
let base = self.initial_delay.as_secs_f64() * exp;
let capped = base.min(self.max_delay.as_secs_f64());
let jitter = self.jitter.clamp(0.0, 1.0);
let factor = 1.0 - jitter + jitter * 2.0 * rand::random::<f64>();
Duration::from_secs_f64((capped * factor).max(0.0))
}
}
/// Scheduling of candidate lookup and independent self-publication.
#[derive(Debug, Clone)]
pub struct DiscoveryPolicy {
/// Mean interval between successful self-publications (20% jitter).
pub publish_interval: Duration,
/// Initial delay between unsuccessful bootstrap lookups.
pub lookup_interval: Duration,
/// Maximum delay between bootstrap lookups.
pub max_lookup_interval: Duration,
/// Continuous isolation before a previously connected network searches again.
pub reconnect_delay: Duration,
/// Deadline for one backend operation.
pub request_timeout: Duration,
}
impl Default for DiscoveryPolicy {
fn default() -> Self {
Self {
publish_interval: Duration::from_secs(300),
lookup_interval: Duration::from_secs(5),
max_lookup_interval: Duration::from_secs(30),
reconnect_delay: Duration::from_secs(60),
request_timeout: Duration::from_secs(60),
}
}
}
/// Everything needed to start an [`crate::Agent`].
#[derive(Clone)]
pub struct AgentConfig {
/// Where the mandatory state and the disposable cache live.
pub paths: StoragePaths,
/// Explicit local bind addresses. Empty means iroh's defaults.
///
/// Tests bind to `127.0.0.1:0` so each agent gets a dynamic port.
pub bind_addrs: Vec<SocketAddr>,
/// How much external connectivity machinery the endpoint may use.
pub transport: TransportPolicy,
/// Real QUIC transport settings for constrained-path integration tests.
#[cfg(feature = "testing")]
pub test_quic_transport: Option<iroh::endpoint::QuicTransportConfig>,
/// Peers with no direct data path, for tests. See
/// [`AgentConfig::with_unreachable_data_peers`].
#[cfg(feature = "testing")]
pub unreachable_data_peers: Arc<std::sync::Mutex<std::collections::HashSet<iroh::EndpointId>>>,
/// Hostname announced to peers. `None` keeps whatever the state store holds,
/// falling back to the OS hostname and finally to a short endpoint id.
pub hostname: Option<String>,
/// Additional discovery backend, composed with Mainline when it is enabled.
pub discovery: Option<Arc<dyn NetworkDiscovery>>,
/// Optional Mainline client, shared by this agent's networks. The CLI enables
/// it by default except in local-only mode; library callers opt in explicitly.
pub dht: Option<MainlineDiscovery>,
/// Lookup, publication and recovery scheduling.
pub discovery_policy: DiscoveryPolicy,
/// How often each active network maintains known peers and observes address changes.
pub discovery_interval: Duration,
/// Bounds applied to network input.
pub limits: Limits,
/// Reconnect backoff policy.
pub reconnect: ReconnectPolicy,
/// IP plugins whose capabilities are announced and dispatched.
pub plugins: Vec<SharedPlugin>,
/// Custom packet transports registered per protocol.
pub custom_transports:
std::collections::HashMap<String, Arc<dyn crate::dataplane::PacketTransport>>,
/// Where the overlay interface comes from, when the agent should have one.
///
/// One agent, one interface: it belongs here rather than to a protocol,
/// because every protocol carries traffic for the same addresses on it.
/// `None` means no interface — the protocols still run and their packets
/// are discarded.
pub tun_factory: Option<Arc<dyn crate::overlay::TunFactory>>,
/// The name to ask the operating system for.
pub interface_name: String,
/// The interface MTU.
pub interface_mtu: u32,
/// The IPv4 overlay range this agent proposes.
///
/// Addresses are allocated from it and recorded in signed state, so a
/// participant keeps the same one across restarts. A network that has
/// already settled on another range wins: a joining agent adopts what it
/// finds rather than imposing this.
pub overlay_ipv4_range: Option<crate::state::Ipv4Range>,
}
impl AgentConfig {
/// Creates a configuration with local-only transport and default limits.
pub fn new(paths: StoragePaths) -> Self {
Self {
paths,
bind_addrs: Vec::new(),
transport: TransportPolicy::default(),
#[cfg(feature = "testing")]
test_quic_transport: None,
#[cfg(feature = "testing")]
unreachable_data_peers: Arc::new(std::sync::Mutex::new(
std::collections::HashSet::new(),
)),
hostname: None,
discovery: None,
dht: None,
discovery_policy: DiscoveryPolicy::default(),
discovery_interval: Duration::from_secs(5),
limits: Limits::default(),
reconnect: ReconnectPolicy::default(),
plugins: Vec::new(),
custom_transports: std::collections::HashMap::new(),
tun_factory: None,
interface_name: "tsun0".to_string(),
interface_mtu: 1280,
overlay_ipv4_range: Some(crate::state::DEFAULT_IPV4_RANGE),
}
}
/// Binds to loopback with a dynamic port. Used by the test suite.
pub fn with_loopback_bind(mut self) -> Self {
self.bind_addrs = vec![
SocketAddr::from(([127, 0, 0, 1], 0)),
SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 1], 0)),
];
self
}
/// Sets an explicit list of bind addresses.
pub fn with_bind_addrs(mut self, addrs: impl IntoIterator<Item = SocketAddr>) -> Self {
self.bind_addrs = addrs.into_iter().collect();
self
}
/// Peers this agent will not open or accept a **direct** data link with.
///
/// Test-only, and behind the `testing` feature so a release build has
/// no such switch. The one arrangement that cannot be made on a single
/// host is two agents that both reach a third and not each other,
/// which is exactly the case a relay exists for.
#[cfg(feature = "testing")]
pub fn with_unreachable_data_peers(
mut self,
peers: Arc<std::sync::Mutex<std::collections::HashSet<iroh::EndpointId>>>,
) -> Self {
self.unreachable_data_peers = peers;
self
}
/// Sets the transport policy.
pub fn with_transport(mut self, transport: TransportPolicy) -> Self {
self.transport = transport;
self
}
/// Sets the discovery backend.
pub fn with_discovery(mut self, discovery: Arc<dyn NetworkDiscovery>) -> Self {
self.discovery = Some(discovery);
self
}
/// Adds Mainline discovery alongside the configured candidate backend.
pub fn with_dht(mut self, dht: MainlineDiscovery) -> Self {
self.dht = Some(dht);
self
}
/// Sets lookup, publication and isolation recovery timing.
pub fn with_discovery_policy(mut self, policy: DiscoveryPolicy) -> Self {
self.discovery_policy = policy;
self
}
/// Sets the known-peer maintenance and endpoint-address observation interval.
pub fn with_discovery_interval(mut self, interval: Duration) -> Self {
self.discovery_interval = interval;
self
}
/// Sets the announced hostname.
pub fn with_hostname(mut self, hostname: impl Into<String>) -> Self {
self.hostname = Some(hostname.into());
self
}
/// Sets the IPv4 overlay range this agent proposes, or disables IPv4.
pub fn with_overlay_ipv4_range(mut self, range: Option<crate::state::Ipv4Range>) -> Self {
self.overlay_ipv4_range = range;
self
}
/// Registers an IP plugin.
pub fn with_plugin(mut self, plugin: SharedPlugin) -> Self {
self.plugins.push(plugin);
self
}
/// Registers a custom packet transport for a specific protocol.
pub fn with_custom_transport(
mut self,
protocol: impl Into<String>,
transport: Arc<dyn crate::dataplane::PacketTransport>,
) -> Self {
self.custom_transports.insert(protocol.into(), transport);
self
}
/// Gives the agent an overlay interface, from this factory.
pub fn with_interface(
mut self,
factory: Arc<dyn crate::overlay::TunFactory>,
name: impl Into<String>,
mtu: u32,
) -> Self {
self.tun_factory = Some(factory);
self.interface_name = name.into();
self.interface_mtu = mtu;
self
}
/// Replaces the limits.
pub fn with_limits(mut self, limits: Limits) -> Self {
self.limits = limits;
self
}
/// Replaces the reconnect policy.
pub fn with_reconnect(mut self, reconnect: ReconnectPolicy) -> Self {
self.reconnect = reconnect;
self
}
}
impl std::fmt::Debug for AgentConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AgentConfig")
.field("paths", &self.paths)
.field("bind_addrs", &self.bind_addrs)
.field("transport", &self.transport)
.field("hostname", &self.hostname)
.field("discovery", &self.discovery.as_ref().map(|d| d.name()))
.field("dht", &self.dht.is_some())
.field("discovery_policy", &self.discovery_policy)
.field("discovery_interval", &self.discovery_interval)
.field("limits", &self.limits)
.field("reconnect", &self.reconnect)
.field(
"plugins",
&self
.plugins
.iter()
.map(|p| p.protocol_id().to_string())
.collect::<Vec<_>>(),
)
.finish()
}
}
/// Maximum number of transport links a routed datagram may traverse.
pub const ROUTING_HOP_LIMIT: u8 = 16;
+496
View File
@@ -0,0 +1,496 @@
//! Boundary between the control plane core and future IP plugins.
//!
//! The data plane is where actual IP connectivity is created. WireGuard is the
//! first planned plugin; none is implemented here.
//!
//! Two rules shape this module:
//!
//! 1. **The core never parses plugin payloads.** A [`PluginCapability`] carries
//! a protocol id, a version, an enabled flag and a bounded opaque blob. The
//! core transports the blob and hands it to the matching plugin. It does not
//! know what a WireGuard configuration looks like.
//! 2. **Plugin keys and lifecycle are separate from iroh identity and from the
//! network secret.** A plugin owns its own keys and its own system objects.
//!
//! An iroh address is *not* automatically a WireGuard address. A future plugin
//! is expected to gather its own reachability information and ship it through
//! the control plane as its announcement payload.
//!
//! A data plane failure never stops the daemon: errors returned here are
//! recorded and surfaced, the control plane keeps running.
pub mod relay;
pub mod routing;
pub mod transport;
pub mod tunnel;
use std::sync::Arc;
use iroh::EndpointId;
use tokio::sync::mpsc;
use crate::BoxFuture;
use crate::identity::NetworkId;
pub use transport::{
MultiPacketTransport, PacketLink, PacketTransport, SharedLink, TransportError,
};
pub use tunnel::{GenericTunnelPlugin, TunnelCodec};
/// Maximum length of a plugin protocol identifier.
pub const MAX_PROTOCOL_ID_LEN: usize = 32;
/// An announcement of one IP plugin's capability.
///
/// `data` is opaque to the core. Nothing in it may be interpreted as a shell
/// command, a filesystem path or an OS setting by the core; a plugin that
/// chooses to do so must validate it itself.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct PluginCapability {
/// Protocol identifier, e.g. `wg-quic`. Bounded by [`MAX_PROTOCOL_ID_LEN`].
pub protocol: String,
/// Version of the plugin's announcement format.
pub version: u16,
/// Whether the peer currently has this plugin enabled.
pub enabled: bool,
/// Opaque, bounded, plugin-defined payload.
pub data: Vec<u8>,
}
/// Where a protocol hands the packets it has decrypted.
///
/// A protocol proves *who* sent a packet; it does not know what that member
/// is entitled to say, because entitlement is an address claim the system
/// level holds. So a decrypted packet goes here rather than straight to an
/// interface, and is checked on the way.
pub trait PacketSink: Send + Sync + 'static {
/// Hands over one packet, attributed to the peer whose tunnel decrypted
/// it.
fn deliver<'a>(
&'a self,
network: NetworkId,
peer: EndpointId,
packet: bytes::Bytes,
) -> BoxFuture<'a, ()>;
}
/// A sink that drops everything, for a protocol running without an interface.
#[derive(Debug, Clone, Copy, Default)]
pub struct DiscardPackets;
impl PacketSink for DiscardPackets {
fn deliver<'a>(
&'a self,
_network: NetworkId,
_peer: EndpointId,
_packet: bytes::Bytes,
) -> BoxFuture<'a, ()> {
Box::pin(async move {})
}
}
/// One setting a protocol accepts.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ProtocolOption {
/// The key, as written in `key=value`.
pub key: &'static str,
/// A word for the value, for the help line: `BYTES`, `NAME`.
pub value: &'static str,
/// What it does.
pub help: &'static str,
/// What happens when it is not given.
pub default: Option<&'static str>,
}
/// Errors a plugin may return. They are recorded, never fatal for the agent.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum PluginError {
/// The plugin is not currently able to produce or apply configuration.
#[error("plugin unavailable: {0}")]
Unavailable(String),
/// A peer announcement was not acceptable to the plugin.
#[error("rejected peer announcement: {0}")]
Rejected(String),
/// Anything else.
#[error("plugin error: {0}")]
Other(String),
}
/// A request a plugin makes of the agent that owns it.
#[derive(Debug)]
pub enum PluginRequest {
/// Re-send this agent's announcement to every peer of a network.
Reannounce(NetworkId),
/// Surface a plugin error on the agent's event stream.
Error {
/// Network the error is scoped to.
network: NetworkId,
/// Plugin protocol id.
protocol: String,
/// Human readable reason, free of secrets.
reason: String,
},
/// A plugin reported that a data link or tunnel for a peer failed.
ProtocolFailure {
/// Network the failure occurred in.
network: NetworkId,
/// Peer whose protocol failed.
peer: EndpointId,
/// Protocol that failed.
protocol: String,
/// Human readable reason.
reason: String,
},
}
/// The agent-side handle a plugin is given when it is attached.
///
/// It is deliberately tiny: a plugin may ask for its announcement to be resent
/// and may report an error. It cannot reach into agent state, cannot send
/// arbitrary control messages and knows nothing about sessions.
///
/// All calls are non-blocking. If the agent is gone or its queue is full the
/// request is dropped rather than stalling the plugin.
#[derive(Clone)]
pub struct PluginContext {
sender: Option<mpsc::Sender<PluginRequest>>,
local: Option<EndpointId>,
/// Where decrypted packets go. Absent when nothing is carrying traffic,
/// in which case a protocol still runs and its packets are discarded.
sink: Option<Arc<dyn PacketSink>>,
}
impl PluginContext {
pub(crate) fn new(
sender: mpsc::Sender<PluginRequest>,
local: EndpointId,
sink: Option<Arc<dyn PacketSink>>,
) -> Self {
Self {
sender: Some(sender),
local: Some(local),
sink,
}
}
/// Where to hand a decrypted packet.
///
/// Always answers: with no interface to write to, the packets are
/// discarded, which is what `--no-tun` means and is not an error.
pub fn packet_sink(&self) -> Arc<dyn PacketSink> {
match &self.sink {
Some(sink) => Arc::clone(sink),
None => Arc::new(DiscardPackets),
}
}
/// A context that discards everything, for plugins used outside an agent.
pub fn detached() -> Self {
Self {
sender: None,
local: None,
sink: None,
}
}
/// A context configured with a custom packet sink.
pub fn with_sink(sink: Arc<dyn PacketSink>) -> Self {
Self {
sender: None,
local: None,
sink: Some(sink),
}
}
/// A context configured with a channel for plugin requests and custom sink, for tests.
#[cfg(feature = "testing")]
pub fn for_test(
sender: mpsc::Sender<PluginRequest>,
local: EndpointId,
sink: Option<Arc<dyn PacketSink>>,
) -> Self {
Self {
sender: Some(sender),
local: Some(local),
sink,
}
}
/// This agent's own endpoint id, when the context is attached.
///
/// A plugin needs it to find itself in the agreed allocation.
pub fn local_endpoint_id(&self) -> Option<EndpointId> {
self.local
}
fn send(&self, request: PluginRequest) {
let Some(sender) = &self.sender else {
return;
};
if let Err(err) = sender.try_send(request) {
tracing::debug!(%err, "dropping plugin request");
}
}
/// Asks the agent to resend this agent's announcement in `network`.
///
/// A plugin calls this when its own capability changed — it finished
/// starting up, its keys or reachability changed — so that peers learn the
/// new value without waiting for a reconnect.
pub fn request_reannounce(&self, network: NetworkId) {
self.send(PluginRequest::Reannounce(network));
}
/// Reports a plugin error on the agent's event stream.
///
/// Plugin work happens in the plugin's own tasks, so errors cannot always
/// be returned from a trait call. They are never fatal for the agent.
pub fn report_error(
&self,
network: NetworkId,
protocol: impl Into<String>,
reason: impl Into<String>,
) {
self.send(PluginRequest::Error {
network,
protocol: protocol.into(),
reason: reason.into(),
});
}
/// Reports that data plane connectivity or a tunnel for a peer has failed.
pub fn report_peer_protocol_failure(
&self,
network: NetworkId,
peer: EndpointId,
protocol: impl Into<String>,
reason: impl Into<String>,
) {
self.send(PluginRequest::ProtocolFailure {
network,
peer,
protocol: protocol.into(),
reason: reason.into(),
});
}
}
impl std::fmt::Debug for PluginContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PluginContext")
.field("attached", &self.sender.is_some())
.field("local", &self.local.map(|id| id.fmt_short().to_string()))
.finish()
}
}
/// The contract an IP plugin implements.
///
/// Implementations must be cheap and non-blocking: the agent calls them from
/// its runtime tasks. Anything slow belongs in the plugin's own tasks.
pub trait IpPlugin: Send + Sync + std::fmt::Debug + 'static {
/// Stable protocol identifier, e.g. `wg-quic`.
///
/// Must be non-empty and at most [`MAX_PROTOCOL_ID_LEN`] bytes.
fn protocol_id(&self) -> &str;
/// The wire version of this protocol.
///
/// Two peers carry traffic for each other only when they have the same
/// protocol at the same version. There is no negotiating a middle
/// ground: a protocol either speaks the same words at both ends or it
/// does not, and pretending otherwise produces a link that fails later
/// and less clearly.
///
/// **Not the software version.** A plugin crate has its own version and
/// it is nobody else's business: two peers on different builds work
/// together for as long as the wire between them has not changed. This
/// number moves only when the bytes do, so it must never be derived from
/// `CARGO_PKG_VERSION` or anything else that moves with a release.
fn protocol_version(&self) -> u16;
/// The settings this protocol accepts, and what they mean.
///
/// Declared rather than documented elsewhere, so the agent can list them
/// without knowing anything about the protocol.
fn options(&self) -> &'static [ProtocolOption] {
&[]
}
/// Called once, when the agent starts, before any network is activated.
///
/// The plugin keeps the context to ask for re-announcements and to report
/// errors that happen in its own tasks.
fn attach(&self, context: PluginContext) {
let _ = context;
}
/// Produces this agent's announcement for a given network.
///
/// Returning `Ok(None)` means "nothing to announce right now", which is
/// different from an error.
fn local_capability(
&self,
network: NetworkId,
) -> std::result::Result<Option<PluginCapability>, PluginError>;
/// Called when a network is activated locally, before any peer appears.
///
/// A plugin uses it to get its per-network state ready, so that the first
/// announcement already carries its capability.
fn on_network_activated(&self, network: NetworkId) {
let _ = network;
}
/// Called when a peer announces a capability for this plugin's protocol.
///
/// The core has already bounded the payload size but has not interpreted it.
fn on_peer_capability(
&self,
network: NetworkId,
peer: EndpointId,
capability: &PluginCapability,
) -> std::result::Result<(), PluginError>;
/// The overlay addresses the network has agreed on.
///
/// Allocated rather than derived, and backed by the signed records in
/// [`crate::state`], so a participant keeps its address across restarts
/// and long absences. Called whenever the agreed picture changes.
fn on_address_allocation(
&self,
network: NetworkId,
range: crate::state::Ipv4Range,
allocations: &[(EndpointId, std::net::Ipv4Addr)],
) {
let _ = (network, range, allocations);
}
/// Carries one packet to a peer, encrypting it however this protocol
/// does.
///
/// `false` when it cannot right now — no link, no tunnel, not this
/// protocol's peer — which the caller reports rather than treats as an
/// error. The packet came off the one interface the agent owns, and
/// which protocol takes it is settled by asking.
///
/// The default carries nothing, which is right for a plugin that only
/// announces something.
fn carry(&self, _network: NetworkId, _peer: EndpointId, _packet: bytes::Bytes) -> bool {
false
}
/// A data plane link to a peer is available for this plugin's protocol.
///
/// The plugin moves its packets over this link and never learns how the
/// link is carried. A new link for a peer replaces any previous one.
fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) {
let _ = (network, peer, link);
}
/// Called when a peer's session in a network goes away.
///
/// Any link handed to the plugin for that peer must be dropped here.
fn on_peer_gone(&self, network: NetworkId, peer: EndpointId);
/// Called when a network is deactivated locally.
///
/// This is a local deactivation, not a signed revocation of membership.
/// The plugin is expected to remove whatever it created for that network.
fn on_network_deactivated(&self, network: NetworkId);
/// Called when this agent has left a network for good.
///
/// Deactivation is temporary and keeps everything ready for next time;
/// this is the other one. Whatever the plugin holds *durably* for that
/// network — a key of its own, a file, a record — goes now, because the
/// agent is no longer a member and keeping it is keeping a secret for a
/// network it cannot rejoin without being told the secret again.
///
/// Always preceded by [`IpPlugin::on_network_deactivated`], so this is
/// only about what outlives a session. Defaulted to nothing, for a
/// plugin that stores nothing.
fn on_network_forgotten(&self, _network: NetworkId) {}
/// Called once when the agent shuts down.
///
/// The plugin removes the system objects it created and stops its tasks.
/// It must be bounded: the agent awaits it during shutdown.
fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> {
Box::pin(async {})
}
}
/// A shared handle to a plugin.
pub type SharedPlugin = Arc<dyn IpPlugin>;
/// A plugin used in tests and examples.
///
/// It announces an explicitly test-only protocol id, so nothing in this crate
/// ever advertises WireGuard as an available transport before it exists.
#[derive(Debug)]
pub struct TestCapabilityPlugin {
protocol: String,
payload: Vec<u8>,
seen: std::sync::Mutex<Vec<(NetworkId, EndpointId, PluginCapability)>>,
}
impl TestCapabilityPlugin {
/// Creates a plugin announcing `protocol` with a fixed opaque payload.
pub fn new(protocol: impl Into<String>, payload: impl Into<Vec<u8>>) -> Self {
Self {
protocol: protocol.into(),
payload: payload.into(),
seen: std::sync::Mutex::new(Vec::new()),
}
}
/// Returns everything this plugin was handed so far.
pub fn observed(&self) -> Vec<(NetworkId, EndpointId, PluginCapability)> {
match self.seen.lock() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
}
}
}
impl IpPlugin for TestCapabilityPlugin {
fn protocol_id(&self) -> &str {
&self.protocol
}
fn protocol_version(&self) -> u16 {
1
}
fn local_capability(
&self,
_network: NetworkId,
) -> std::result::Result<Option<PluginCapability>, PluginError> {
Ok(Some(PluginCapability {
protocol: self.protocol.clone(),
version: 1,
enabled: true,
data: self.payload.clone(),
}))
}
fn on_peer_capability(
&self,
network: NetworkId,
peer: EndpointId,
capability: &PluginCapability,
) -> std::result::Result<(), PluginError> {
let mut guard = match self.seen.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard.push((network, peer, capability.clone()));
Ok(())
}
fn on_peer_gone(&self, _network: NetworkId, _peer: EndpointId) {}
fn on_network_deactivated(&self, _network: NetworkId) {}
}
+506
View File
@@ -0,0 +1,506 @@
//! Userspace forwarding. Topology updates publish immutable tables; the packet
//! path takes no routing mutex, walks no graph and never inspects plugin bytes.
//! Each transport reader forwards transit immediately, independently of the
//! destination protocol's receive task. Only local delivery enters its inbox.
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use arc_swap::ArcSwap;
use bytes::Bytes;
use iroh::EndpointId;
use tokio::sync::{Notify, mpsc, watch};
use tokio::task::JoinHandle;
use super::routing::{FlowId, PeerId, RoutingTable, envelope, mix64};
use super::transport::{PacketLink, SharedLink, TransportError};
use crate::BoxFuture;
use crate::config::{MAX_DATA_DATAGRAM, ROUTING_HOP_LIMIT};
use crate::identity::NetworkId;
/// Fixed routing header, subtracted from the logical transport MTU.
pub const RELAY_OVERHEAD: usize = envelope::HEADER;
const INBOX: usize = 256;
/// Forwarding counters, shared by all protocols of one network.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct RelayCounters {
/// Datagrams originated here and sent through another member.
pub sent_via: u64,
/// Transit datagrams handed to the next transport link.
pub forwarded: u64,
/// Local datagrams received through another member.
pub received_via: u64,
/// Missing or failed next hop.
pub dropped_no_link: u64,
/// Malformed envelope or unknown member.
pub dropped_unknown: u64,
/// Exhausted hop limit, including loops during topology convergence.
pub dropped_hop_limit: u64,
/// Full local protocol inbox. Transit does not use this queue.
pub dropped_congested: u64,
}
#[derive(Debug, Default)]
struct Tally {
sent_via: AtomicU64,
forwarded: AtomicU64,
received_via: AtomicU64,
dropped_no_link: AtomicU64,
dropped_unknown: AtomicU64,
dropped_hop_limit: AtomicU64,
dropped_congested: AtomicU64,
}
impl Tally {
fn snapshot(&self) -> RelayCounters {
RelayCounters {
sent_via: self.sent_via.load(Ordering::Relaxed),
forwarded: self.forwarded.load(Ordering::Relaxed),
received_via: self.received_via.load(Ordering::Relaxed),
dropped_no_link: self.dropped_no_link.load(Ordering::Relaxed),
dropped_unknown: self.dropped_unknown.load(Ordering::Relaxed),
dropped_hop_limit: self.dropped_hop_limit.load(Ordering::Relaxed),
dropped_congested: self.dropped_congested.load(Ordering::Relaxed),
}
}
}
#[derive(Debug)]
struct NextHop {
peer: PeerId,
link: SharedLink,
}
#[derive(Debug)]
struct ForwardRoute {
hops: u8,
next: Box<[NextHop]>,
}
impl ForwardRoute {
fn select(&self, source: &PeerId, flow: FlowId) -> Option<&NextHop> {
if self.next.len() == 1 {
return self.next.first().filter(|hop| !hop.link.is_closed());
}
let seed = u64::from_le_bytes(source[..8].try_into().ok()?);
let start = mix64(flow ^ seed) as usize % self.next.len();
// Stable fallback if a link closed just before its table was replaced.
(0..self.next.len())
.map(|i| &self.next[(start + i) % self.next.len()])
.find(|hop| !hop.link.is_closed())
}
}
#[derive(Debug, Default)]
struct ForwardingTable {
routes: HashMap<PeerId, ForwardRoute>,
inboxes: HashMap<PeerId, mpsc::Sender<Bytes>>,
members: HashSet<PeerId>,
}
#[derive(Debug)]
struct Plane {
local: PeerId,
table: ArcSwap<ForwardingTable>,
tally: Arc<Tally>,
}
impl Plane {
/// Synchronous hot path: one snapshot, bounded header decode, table lookup,
/// and transport send. An exclusively owned buffer is modified in place.
fn receive(&self, incoming: PeerId, frame: Bytes) {
let Some(header) = envelope::decode(&frame) else {
self.tally.dropped_unknown.fetch_add(1, Ordering::Relaxed);
return;
};
let table = self.table.load();
if !table.members.contains(&header.source) || header.source == self.local {
self.tally.dropped_unknown.fetch_add(1, Ordering::Relaxed);
return;
}
if header.destination == self.local {
let Some(inbox) = table.inboxes.get(&header.source) else {
self.tally.dropped_unknown.fetch_add(1, Ordering::Relaxed);
return;
};
if inbox.try_send(frame.slice(RELAY_OVERHEAD..)).is_err() {
self.tally.dropped_congested.fetch_add(1, Ordering::Relaxed);
} else if incoming != header.source {
self.tally.received_via.fetch_add(1, Ordering::Relaxed);
}
return;
}
if header.remaining <= 1 {
self.tally.dropped_hop_limit.fetch_add(1, Ordering::Relaxed);
return;
}
let hop = table
.routes
.get(&header.destination)
.and_then(|route| route.select(&header.source, header.flow));
let Some(hop) = hop else {
self.tally.dropped_no_link.fetch_add(1, Ordering::Relaxed);
return;
};
if hop.link.send(envelope::decrement(frame)).is_ok() {
self.tally.forwarded.fetch_add(1, Ordering::Relaxed);
} else {
self.tally.dropped_no_link.fetch_add(1, Ordering::Relaxed);
}
}
}
#[derive(Debug)]
struct Protocol {
plane: Arc<Plane>,
graph: HashMap<PeerId, Vec<PeerId>>,
members: HashSet<PeerId>,
raw: HashMap<PeerId, SharedLink>,
readers: HashMap<PeerId, JoinHandle<()>>,
links: HashMap<PeerId, Arc<PeerLink>>,
}
impl Protocol {
fn publish(&self) {
let mut graph = self.graph.clone();
graph.insert(
self.plane.local,
self.raw
.iter()
.filter(|(_, link)| !link.is_closed())
.map(|(peer, _)| *peer)
.collect(),
);
let routing = RoutingTable::build(self.plane.local, &graph, ROUTING_HOP_LIMIT);
let routes = routing
.iter()
.filter_map(|(peer, route)| {
let next: Box<[_]> = route
.next_hops
.iter()
.filter_map(|id| {
self.raw.get(id).map(|link| NextHop {
peer: *id,
link: link.clone(),
})
})
.collect();
(!next.is_empty()).then_some((
*peer,
ForwardRoute {
hops: route.hops,
next,
},
))
})
.collect();
let mut members = self.members.clone();
members.extend(self.raw.keys().copied());
self.plane.table.store(Arc::new(ForwardingTable {
routes,
members,
inboxes: self
.links
.iter()
.map(|(peer, link)| (*peer, link.inbox_tx.clone()))
.collect(),
}));
}
}
impl Drop for Protocol {
fn drop(&mut self) {
for task in self.readers.values() {
task.abort();
}
for link in self.links.values() {
link.close();
}
self.plane.table.store(Arc::default());
}
}
/// One network's transport links and independently published protocol tables.
#[derive(Debug)]
pub struct RelayHub {
network: NetworkId,
local: PeerId,
protocols: Mutex<HashMap<String, Protocol>>,
tally: Arc<Tally>,
changed: Arc<Notify>,
}
impl RelayHub {
/// Creates the router at this network's local endpoint.
pub fn new(network: NetworkId, local: EndpointId) -> Arc<Self> {
Arc::new(Self {
network,
local: *local.as_bytes(),
protocols: Mutex::default(),
tally: Arc::default(),
changed: Arc::default(),
})
}
fn protocols(&self) -> std::sync::MutexGuard<'_, HashMap<String, Protocol>> {
self.protocols
.lock()
.unwrap_or_else(|error| error.into_inner())
}
fn protocol<'a>(
&self,
protocols: &'a mut HashMap<String, Protocol>,
name: &str,
) -> &'a mut Protocol {
protocols
.entry(name.to_owned())
.or_insert_with(|| Protocol {
plane: Arc::new(Plane {
local: self.local,
table: ArcSwap::from_pointee(ForwardingTable::default()),
tally: self.tally.clone(),
}),
graph: HashMap::new(),
members: HashSet::new(),
raw: HashMap::new(),
readers: HashMap::new(),
links: HashMap::new(),
})
}
/// Wakes the control loop promptly when a transport reader ends.
pub async fn changed(&self) {
self.changed.notified().await;
}
/// Current counters.
pub fn counters(&self) -> RelayCounters {
self.tally.snapshot()
}
/// Publishes authenticated topology. Refreshing identical state is cheap.
pub fn set_topology(
&self,
protocol: &str,
graph: HashMap<PeerId, Vec<PeerId>>,
members: HashSet<PeerId>,
) {
let mut protocols = self.protocols();
let state = self.protocol(&mut protocols, protocol);
if state.graph != graph || state.members != members {
state.graph = graph;
state.members = members;
state.publish();
}
}
/// Whether the current table has a path to this destination.
pub fn reachable(&self, peer: EndpointId, protocol: &str) -> bool {
self.protocols().get(protocol).is_some_and(|state| {
state
.plane
.table
.load()
.routes
.contains_key(peer.as_bytes())
})
}
/// Stable logical link, preserved across topology changes.
pub fn link(&self, peer: EndpointId, protocol: &str) -> Arc<PeerLink> {
let mut protocols = self.protocols();
let state = self.protocol(&mut protocols, protocol);
if let Some(link) = state.links.get(peer.as_bytes()) {
return link.clone();
}
let (inbox_tx, inbox_rx) = mpsc::channel(INBOX);
let (closed, _) = watch::channel(false);
let link = Arc::new(PeerLink {
network: self.network,
peer,
plane: state.plane.clone(),
inbox_tx,
inbox_rx: tokio::sync::Mutex::new(inbox_rx),
closed,
});
state.links.insert(*peer.as_bytes(), link.clone());
state.publish();
link
}
/// Whether the protocol already owns its logical link.
pub fn has_link(&self, peer: EndpointId, protocol: &str) -> bool {
self.protocols()
.get(protocol)
.is_some_and(|state| state.links.contains_key(peer.as_bytes()))
}
/// Installs a raw transport link and starts its independent ingress reader.
pub fn set_direct(&self, peer: EndpointId, protocol: &str, raw: SharedLink) {
if raw.network() != self.network || raw.peer() != peer {
return;
}
let mut protocols = self.protocols();
let state = self.protocol(&mut protocols, protocol);
let id = *peer.as_bytes();
if let Some(old) = state.readers.remove(&id) {
old.abort();
}
state.raw.insert(id, raw.clone());
state.publish();
let plane = state.plane.clone();
let changed = self.changed.clone();
state.readers.insert(
id,
tokio::spawn(async move {
let mut batch = 0;
while let Some(frame) = raw.recv().await {
plane.receive(id, frame);
batch += 1;
if batch == 64 {
tokio::task::yield_now().await;
batch = 0;
}
}
changed.notify_one();
}),
);
}
/// Removes a physical link without tearing down end-to-end protocol state.
pub fn clear_direct(&self, peer: EndpointId, protocol: &str) {
if let Some(state) = self.protocols().get_mut(protocol) {
state.raw.remove(peer.as_bytes());
if let Some(task) = state.readers.remove(peer.as_bytes()) {
task.abort();
}
state.publish();
}
}
/// Revokes all links and topology involving a departed authenticated peer.
pub fn remove_peer(&self, peer: EndpointId) {
for state in self.protocols().values_mut() {
state.raw.remove(peer.as_bytes());
if let Some(task) = state.readers.remove(peer.as_bytes()) {
task.abort();
}
if let Some(link) = state.links.remove(peer.as_bytes()) {
link.close();
}
state.members.remove(peer.as_bytes());
state.graph.remove(peer.as_bytes());
for neighbors in state.graph.values_mut() {
neighbors.retain(|id| id != peer.as_bytes());
}
state.publish();
}
}
/// Stops every reader and releases all transport handles.
pub fn close(&self) {
self.protocols().clear();
}
}
/// A protocol's end-to-end channel, independent of its current next hop.
#[derive(Debug)]
pub struct PeerLink {
network: NetworkId,
peer: EndpointId,
plane: Arc<Plane>,
inbox_tx: mpsc::Sender<Bytes>,
inbox_rx: tokio::sync::Mutex<mpsc::Receiver<Bytes>>,
closed: watch::Sender<bool>,
}
impl PeerLink {
fn close(&self) {
self.closed.send_replace(true);
}
}
impl PacketLink for PeerLink {
fn network(&self) -> NetworkId {
self.network
}
fn peer(&self) -> EndpointId {
self.peer
}
fn max_datagram_size(&self) -> usize {
MAX_DATA_DATAGRAM - RELAY_OVERHEAD
}
fn send(&self, payload: Bytes) -> Result<(), TransportError> {
self.send_flow(payload, 0)
}
fn send_flow(&self, payload: Bytes, flow: FlowId) -> Result<(), TransportError> {
if self.is_closed() {
return Err(TransportError::Closed);
}
if payload.len() > self.max_datagram_size() {
return Err(TransportError::TooLarge {
size: payload.len(),
limit: self.max_datagram_size(),
});
}
let table = self.plane.table.load();
let route = table
.routes
.get(self.peer.as_bytes())
.ok_or(TransportError::Closed)?;
let hop = route
.select(&self.plane.local, flow)
.ok_or(TransportError::Closed)?;
hop.link.send(envelope::encode(
self.plane.local,
*self.peer.as_bytes(),
flow,
&payload,
))?;
if route.hops > 1 {
self.plane.tally.sent_via.fetch_add(1, Ordering::Relaxed);
}
Ok(())
}
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
Box::pin(async move {
let mut closed = self.closed.subscribe();
if *closed.borrow_and_update() {
return None;
}
let mut inbox = self.inbox_rx.lock().await;
tokio::select! { biased; _ = closed.changed() => None, frame = inbox.recv() => frame }
})
}
fn closed(&self) -> BoxFuture<'_, ()> {
Box::pin(async move {
let mut closed = self.closed.subscribe();
if !*closed.borrow_and_update() {
let _ = closed.changed().await;
}
})
}
fn is_closed(&self) -> bool {
*self.closed.borrow()
}
fn path_description(&self) -> String {
let table = self.plane.table.load();
match table.routes.get(self.peer.as_bytes()) {
Some(route) if route.hops == 1 => route.next[0].link.path_description(),
Some(route) => format!(
"relay {} hops via {} ({} equal paths)",
route.hops,
hex::encode(route.next[0].peer),
route.next.len()
),
None => "unreachable".into(),
}
}
}
#[cfg(test)]
#[path = "relay_tests.rs"]
mod tests;
@@ -0,0 +1,287 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::identity::{NetworkKeys, NetworkName, NetworkSecret};
use std::sync::atomic::AtomicBool;
fn network() -> NetworkId {
NetworkKeys::derive(
&NetworkName::new("routing-tests").unwrap(),
&NetworkSecret::from_bytes([3; 32]).unwrap(),
)
.network_id()
}
fn peer(seed: u8) -> EndpointId {
iroh::SecretKey::from_bytes(&[seed; 32]).public()
}
fn id(seed: u8) -> PeerId {
*peer(seed).as_bytes()
}
#[derive(Debug)]
struct Wire {
peer: EndpointId,
sent: Mutex<Vec<Bytes>>,
feed: mpsc::Sender<Bytes>,
inbound: tokio::sync::Mutex<mpsc::Receiver<Bytes>>,
dead: AtomicBool,
count: AtomicU64,
record: bool,
}
impl Wire {
fn new(seed: u8, record: bool) -> Arc<Self> {
let (feed, inbound) = mpsc::channel(32);
Arc::new(Self {
peer: peer(seed),
sent: Mutex::default(),
feed,
inbound: tokio::sync::Mutex::new(inbound),
dead: AtomicBool::new(false),
count: AtomicU64::new(0),
record,
})
}
fn take(&self) -> Vec<Bytes> {
std::mem::take(&mut self.sent.lock().unwrap())
}
}
impl PacketLink for Wire {
fn network(&self) -> NetworkId {
network()
}
fn peer(&self) -> EndpointId {
self.peer
}
fn max_datagram_size(&self) -> usize {
MAX_DATA_DATAGRAM
}
fn send(&self, frame: Bytes) -> Result<(), TransportError> {
if self.is_closed() {
return Err(TransportError::Closed);
}
self.count.fetch_add(1, Ordering::Relaxed);
if self.record {
self.sent.lock().unwrap().push(frame);
}
Ok(())
}
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
Box::pin(async move { self.inbound.lock().await.recv().await })
}
fn closed(&self) -> BoxFuture<'_, ()> {
Box::pin(std::future::pending())
}
fn is_closed(&self) -> bool {
self.dead.load(Ordering::Relaxed)
}
fn path_description(&self) -> String {
"test-direct".into()
}
}
fn topology(hub: &RelayHub, protocol: &str, edges: &[(u8, &[u8])], members: &[u8]) {
hub.set_topology(
protocol,
edges
.iter()
.map(|(a, bs)| (id(*a), bs.iter().map(|b| id(*b)).collect()))
.collect(),
members.iter().map(|p| id(*p)).collect(),
);
}
#[tokio::test]
async fn ecmp_is_per_flow_direct_wins_and_existing_tunnel_survives_changes() {
let hub = RelayHub::new(network(), peer(1));
let b = Wire::new(2, true);
let c = Wire::new(3, true);
hub.set_direct(peer(2), "ip", b.clone());
hub.set_direct(peer(3), "ip", c.clone());
topology(&hub, "ip", &[(2, &[4]), (3, &[4])], &[1, 2, 3, 4]);
let link = hub.link(peer(4), "ip");
let mut chosen = HashMap::new();
for _ in 0..8 {
for flow in 0..100 {
link.send_flow(Bytes::from_static(b"ciphertext"), flow)
.unwrap();
}
for (hop, wire) in [(2, &b), (3, &c)] {
for frame in wire.take() {
let header = envelope::decode(&frame).unwrap();
assert_eq!(header.source, id(1));
assert_eq!(header.destination, id(4));
if let Some(old) = chosen.insert(header.flow, hop) {
assert_eq!(old, hop);
}
}
}
}
assert_eq!(
chosen.values().copied().collect::<HashSet<_>>(),
HashSet::from([2, 3])
);
let direct = Wire::new(4, true);
hub.set_direct(peer(4), "ip", direct.clone());
link.send_flow(Bytes::from_static(b"direct"), 8).unwrap();
assert_eq!(direct.take().len(), 1);
assert!(b.take().is_empty() && c.take().is_empty());
hub.clear_direct(peer(4), "ip");
b.dead.store(true, Ordering::Relaxed);
for flow in 0..100 {
link.send_flow(Bytes::from_static(b"fallback"), flow)
.unwrap();
}
assert_eq!(c.take().len(), 100);
hub.clear_direct(peer(2), "ip");
topology(&hub, "ip", &[(3, &[])], &[1, 2, 3, 4]);
assert!(link.send(Bytes::new()).is_err());
assert!(!link.is_closed());
assert!(Arc::ptr_eq(&link, &hub.link(peer(4), "ip")));
hub.remove_peer(peer(4));
assert!(link.is_closed());
tokio::time::timeout(std::time::Duration::from_secs(1), link.closed())
.await
.unwrap();
assert!(link.recv().await.is_none());
hub.close();
}
#[tokio::test]
async fn transport_reader_forwards_without_any_plugin_reader_and_isolates_protocols() {
let hub = RelayHub::new(network(), peer(2));
let a = Wire::new(1, true);
let c = Wire::new(3, true);
hub.set_direct(peer(1), "ip", a.clone());
hub.set_direct(peer(3), "ip", c.clone());
topology(&hub, "ip", &[(3, &[4])], &[1, 2, 3, 4]);
let frame = envelope::encode(id(1), id(4), 91, b"end-to-end encrypted");
let pointer = frame.as_ptr();
a.feed.send(frame).await.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(1), async {
while hub.counters().forwarded == 0 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
let sent = c.take();
assert_eq!(sent[0].as_ptr(), pointer);
assert_eq!(
envelope::decode(&sent[0]).unwrap().remaining,
ROUTING_HOP_LIMIT - 1
);
assert_eq!(&sent[0][RELAY_OVERHEAD..], b"end-to-end encrypted");
let isolated = hub.link(peer(4), "other-ip");
assert!(isolated.send(Bytes::new()).is_err());
assert_eq!(hub.counters().forwarded, 1);
hub.close();
}
#[tokio::test]
async fn local_delivery_is_bounded_and_malformed_or_unknown_sources_are_dropped() {
let hub = RelayHub::new(network(), peer(2));
let a = Wire::new(1, true);
hub.set_direct(peer(1), "ip", a);
let link = hub.link(peer(1), "ip");
let plane = link.plane.clone();
for _ in 0..INBOX + 1 {
plane.receive(id(1), envelope::encode(id(1), id(2), 0, b"ip"));
}
assert_eq!(hub.counters().dropped_congested, 1);
for _ in 0..INBOX {
assert_eq!(link.recv().await.unwrap(), Bytes::from_static(b"ip"));
}
plane.receive(id(1), Bytes::from_static(b"broken"));
plane.receive(id(1), envelope::encode(id(9), id(2), 0, b"unknown member"));
assert_eq!(hub.counters().dropped_unknown, 2);
plane.receive(
id(1),
envelope::encode(id(1), id(9), 0, b"unknown destination"),
);
assert_eq!(hub.counters().dropped_no_link, 1);
hub.close();
}
#[tokio::test]
async fn inconsistent_tables_cannot_loop_beyond_the_hop_limit() {
let a = RelayHub::new(network(), peer(1));
let b = RelayHub::new(network(), peer(2));
let ab = Wire::new(2, true);
let ba = Wire::new(1, true);
a.set_direct(peer(2), "ip", ab.clone());
b.set_direct(peer(1), "ip", ba.clone());
// Deliberately inconsistent snapshots while updates are in flight.
topology(&a, "ip", &[(2, &[4])], &[1, 2, 3, 4]);
topology(&b, "ip", &[(1, &[4])], &[1, 2, 3, 4]);
let ap = a.link(peer(4), "ip").plane.clone();
let bp = b.link(peer(4), "ip").plane.clone();
let mut frame = envelope::encode(id(3), id(4), 7, b"transit");
for turn in 0..ROUTING_HOP_LIMIT {
let (plane, incoming, wire) = if turn % 2 == 0 {
(&ap, id(2), &ab)
} else {
(&bp, id(1), &ba)
};
plane.receive(incoming, frame);
let sent = wire.take();
if turn == ROUTING_HOP_LIMIT - 1 {
assert!(sent.is_empty());
break;
}
assert_eq!(sent.len(), 1);
frame = sent.into_iter().next().unwrap();
}
assert_eq!(
a.counters().forwarded + b.counters().forwarded,
u64::from(ROUTING_HOP_LIMIT - 1)
);
assert_eq!(
a.counters().dropped_hop_limit + b.counters().dropped_hop_limit,
1
);
a.close();
b.close();
}
/// Measures the actual synchronous transit routine, including header checks,
/// snapshot lookup, ECMP, in-place TTL update and mock transport submission.
/// Packet construction, encryption and socket I/O are outside this measurement.
#[tokio::test]
#[ignore = "manual release-mode forwarding microbenchmark"]
async fn forwarding_benchmark() {
let hub = RelayHub::new(network(), peer(2));
let wire = Wire::new(3, false);
hub.set_direct(peer(3), "ip", wire.clone());
let alternate = Wire::new(5, false);
hub.set_direct(peer(5), "ip", alternate.clone());
topology(&hub, "ip", &[(3, &[4]), (5, &[4])], &[1, 2, 3, 4, 5]);
let plane = hub.link(peer(4), "ip").plane.clone();
let source = id(1);
let destination = id(4);
let incoming = id(1);
for size in [64, 1280] {
let mut elapsed = std::time::Duration::ZERO;
const COUNT: usize = 10000;
const BATCHES: usize = 100;
for _ in 0..BATCHES {
let frames: Vec<_> = (0..COUNT)
.map(|flow| envelope::encode(source, destination, flow as u64, &vec![42; size]))
.collect();
let start = std::time::Instant::now();
for frame in frames {
plane.receive(incoming, std::hint::black_box(frame));
}
elapsed += start.elapsed();
}
let count = (COUNT * BATCHES) as f64;
println!(
"transit {size} B: {:.1} ns/packet, {:.2} Mpps (1M packets, two equal paths)",
elapsed.as_nanos() as f64 / count,
count / elapsed.as_secs_f64() / 1e6
);
}
assert_eq!(
wire.count.load(Ordering::Relaxed) + alternate.count.load(Ordering::Relaxed),
2000000
);
hub.close();
}
@@ -0,0 +1,84 @@
//! Fixed header. Transit changes one byte in an owned receive buffer.
use super::{FlowId, PeerId};
use crate::config::{MAX_DATA_DATAGRAM, ROUTING_HOP_LIMIT};
use bytes::{BufMut, Bytes, BytesMut};
pub(crate) const HEADER: usize = 2 + 32 + 32 + 8;
const VERSION: u8 = 1;
#[derive(Debug, PartialEq, Eq)]
pub(crate) struct Header {
pub source: PeerId,
pub destination: PeerId,
pub remaining: u8,
pub flow: FlowId,
}
pub(crate) fn decode(bytes: &[u8]) -> Option<Header> {
if bytes.len() < HEADER
|| bytes.len() > MAX_DATA_DATAGRAM
|| bytes[0] != VERSION
|| bytes[1] == 0
|| bytes[1] > ROUTING_HOP_LIMIT
{
return None;
}
Some(Header {
source: bytes[2..34].try_into().ok()?,
destination: bytes[34..66].try_into().ok()?,
remaining: bytes[1],
flow: u64::from_be_bytes(bytes[66..74].try_into().ok()?),
})
}
pub(crate) fn encode(source: PeerId, destination: PeerId, flow: FlowId, payload: &[u8]) -> Bytes {
let mut bytes = BytesMut::with_capacity(HEADER + payload.len());
bytes.put_u8(VERSION);
bytes.put_u8(ROUTING_HOP_LIMIT);
bytes.extend_from_slice(&source);
bytes.extend_from_slice(&destination);
bytes.put_u64(flow);
bytes.extend_from_slice(payload);
bytes.freeze()
}
pub(crate) fn decrement(bytes: Bytes) -> Bytes {
let mut bytes = match bytes.try_into_mut() {
Ok(bytes) => bytes,
Err(bytes) => BytesMut::from(bytes.as_ref()),
};
// Called only after decode and after excluding remaining <= 1.
bytes[1] -= 1;
bytes.freeze()
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn transit_reuses_the_buffer_and_rejects_invalid_headers() {
let frame = encode([1; 32], [2; 32], 17, &[42; 1280]);
let address = frame.as_ptr();
let frame = decrement(frame);
assert_eq!(
frame.as_ptr(),
address,
"no payload copy for an owned buffer"
);
assert_eq!(decode(&frame).unwrap().remaining, ROUTING_HOP_LIMIT - 1);
assert_eq!(&frame[HEADER..], &[42; 1280]);
for length in 0..HEADER {
assert!(decode(&frame[..length]).is_none());
}
for ttl in [0, ROUTING_HOP_LIMIT + 1] {
let mut bad = frame.to_vec();
bad[1] = ttl;
assert!(decode(&bad).is_none());
}
let mut bad = frame.to_vec();
bad[0] = 99;
assert!(decode(&bad).is_none());
assert!(decode(&vec![1; MAX_DATA_DATAGRAM + 1]).is_none());
}
}
@@ -0,0 +1,125 @@
//! Optional IP flow classifier for IP plugins, called before encryption.
//! The router itself treats this identifier and the payload as opaque.
use super::FlowId;
fn hash(parts: &[&[u8]]) -> FlowId {
let mut value = 0xcbf29ce484222325u64;
for part in parts {
for byte in *part {
value = (value ^ u64::from(*byte)).wrapping_mul(0x100000001b3);
}
}
value
}
/// Stable directional IP flow hash, independent of TCP sequence numbers,
/// checksums and packet contents. Fragmented traffic uses addresses/protocol
/// instead of ports: even non-initial fragments and subsequent fragmented
/// datagrams stay together. This deliberately coalesces fragmented flows.
/// Malformed packets get a coarse hash; parsing is bounded and never allocates.
pub fn ip_flow(packet: &[u8]) -> FlowId {
match packet.first().map(|b| b >> 4) {
Some(4) if packet.len() >= 20 => {
let addresses = &packet[12..20];
let protocol = &packet[9..10];
let offset = usize::from(packet[0] & 15) * 4;
if u16::from_be_bytes([packet[6], packet[7]]) & 0x3fff != 0 {
return hash(&[addresses, protocol]);
}
if offset >= 20
&& matches!(packet[9], 6 | 17)
&& let Some(ports) = packet.get(offset..offset + 4)
{
return hash(&[addresses, protocol, ports]);
}
hash(&[addresses, protocol])
}
Some(6) if packet.len() >= 40 => {
let addresses = &packet[8..40];
let mut protocol = packet[6];
let mut offset = 40;
for _ in 0..8 {
match protocol {
44 => {
if let Some(fragment) = packet.get(offset..offset + 8) {
return hash(&[addresses, &fragment[..1]]);
}
break;
}
0 | 43 | 60 | 51 => {
let Some(header) = packet.get(offset..offset + 2) else {
break;
};
let size = if protocol == 51 {
(usize::from(header[1]) + 2) * 4
} else {
(usize::from(header[1]) + 1) * 8
};
protocol = header[0];
offset += size;
}
6 | 17 => {
if let Some(ports) = packet.get(offset..offset + 4) {
return hash(&[addresses, &[protocol], ports]);
}
break;
}
_ => break,
}
}
hash(&[addresses, &[protocol]])
}
_ => 0,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tcp_flow_ignores_packet_contents_but_distinguishes_ports() {
let mut packet = [0u8; 80];
packet[0] = 0x45;
packet[9] = 6;
packet[12..24].copy_from_slice(&[10, 0, 0, 1, 10, 0, 0, 2, 0, 22, 1, 2]);
let flow = ip_flow(&packet);
packet[4] = 123; // IP id changes for each unfragmented packet.
packet[24..].fill(47);
assert_eq!(flow, ip_flow(&packet));
packet[23] = 3;
assert_ne!(flow, ip_flow(&packet));
packet[6] = 0x20;
let fragment_flow = ip_flow(&packet);
packet[6] = 0;
packet[7] = 1;
packet[20..].fill(99);
assert_eq!(fragment_flow, ip_flow(&packet));
packet[4] = 97;
assert_eq!(fragment_flow, ip_flow(&packet));
}
#[test]
fn ipv6_extensions_and_fragments_are_bounded_and_stable() {
let mut packet = [0u8; 80];
packet[0] = 0x60;
packet[6] = 0;
packet[40] = 17;
packet[48..52].copy_from_slice(&[1, 2, 3, 4]);
let flow = ip_flow(&packet);
packet[52..].fill(5);
assert_eq!(flow, ip_flow(&packet));
packet[50] = 9;
assert_ne!(flow, ip_flow(&packet));
packet[6] = 44;
packet[40] = 6;
let flow = ip_flow(&packet);
packet[42] = 32;
packet[48..].fill(3);
assert_eq!(flow, ip_flow(&packet));
for len in 0..packet.len() {
let _ = ip_flow(&packet[..len]);
}
}
}
@@ -0,0 +1,131 @@
//! Transport-independent routing over opaque identities. Tables are built on
//! topology changes, never on the packet path. No WireGuard or iroh APIs here.
pub(crate) mod envelope;
pub mod flow;
use std::collections::{BTreeSet, HashMap, VecDeque};
/// Opaque authenticated identity; the transport adapter validates its keys.
pub type PeerId = [u8; 32];
/// Stable flow identifier, carried unchanged between transit routers.
pub type FlowId = u64;
/// A shortest route with all equal-cost first hops in stable order.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Route {
/// Number of transport links to the destination.
pub hops: u8,
/// Equal-cost first hops. Direct destinations have one entry.
pub next_hops: Box<[PeerId]>,
}
/// Immutable shortest-path table, replaced when connectivity changes.
#[derive(Debug, Clone, Default)]
pub struct RoutingTable {
routes: HashMap<PeerId, Route>,
}
impl RoutingTable {
/// Computes shortest directed paths bounded by `hop_limit`.
/// The caller supplies only authenticated, fresh, compatible edges.
pub fn build(local: PeerId, graph: &HashMap<PeerId, Vec<PeerId>>, hop_limit: u8) -> Self {
let mut distances = HashMap::from([(local, 0u8)]);
let mut first_hops: HashMap<PeerId, BTreeSet<PeerId>> = HashMap::new();
let mut pending = VecDeque::from([local]);
while let Some(node) = pending.pop_front() {
let distance = distances[&node];
if distance >= hop_limit {
continue;
}
let inherited = first_hops.get(&node).cloned().unwrap_or_default();
for &next in graph.get(&node).into_iter().flatten() {
let next_distance = distance + 1;
match distances.get(&next) {
Some(&known) if known < next_distance => continue,
None => {
distances.insert(next, next_distance);
pending.push_back(next);
}
_ => {}
}
let hops = first_hops.entry(next).or_default();
if node == local {
hops.insert(next);
} else {
hops.extend(inherited.iter().copied());
}
}
}
Self {
routes: first_hops
.into_iter()
.map(|(peer, hops)| {
(
peer,
Route {
hops: distances[&peer],
next_hops: hops.into_iter().collect(),
},
)
})
.collect(),
}
}
/// Precomputed route to a destination.
pub fn get(&self, destination: &PeerId) -> Option<&Route> {
self.routes.get(destination)
}
/// All reachable destinations, excluding ourselves.
pub fn iter(&self) -> impl Iterator<Item = (&PeerId, &Route)> {
self.routes.iter()
}
}
/// Stable ECMP mixing, without random state or hashing the payload per hop.
pub(crate) fn mix64(mut value: u64) -> u64 {
value ^= value >> 30;
value = value.wrapping_mul(0xbf58476d1ce4e5b9);
value ^= value >> 27;
value = value.wrapping_mul(0x94d049bb133111eb);
value ^ (value >> 31)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn p(n: u8) -> PeerId {
[n; 32]
}
#[test]
fn shortest_paths_keep_equal_cost_hops_and_prefer_direct_links() {
let mut graph = HashMap::from([
(p(1), vec![p(3), p(2)]),
(p(2), vec![p(4)]),
(p(3), vec![p(4)]),
(p(4), vec![p(5)]),
(p(5), vec![p(1)]),
]);
let routes = RoutingTable::build(p(1), &graph, 16);
assert_eq!(routes.get(&p(5)).unwrap().hops, 3);
assert_eq!(&*routes.get(&p(5)).unwrap().next_hops, &[p(2), p(3)]);
assert!(routes.get(&p(1)).is_none());
assert!(routes.get(&p(9)).is_none());
graph.get_mut(&p(1)).unwrap().push(p(5));
assert_eq!(
RoutingTable::build(p(1), &graph, 16).get(&p(5)).unwrap(),
&Route {
hops: 1,
next_hops: Box::new([p(5)])
}
);
}
#[test]
fn topology_removal_and_hop_budget_remove_impossible_routes() {
let mut graph = HashMap::from([(p(1), vec![p(2)]), (p(2), vec![p(3)]), (p(3), vec![p(4)])]);
assert!(RoutingTable::build(p(1), &graph, 2).get(&p(4)).is_none());
assert!(RoutingTable::build(p(1), &graph, 3).get(&p(4)).is_some());
graph.remove(&p(2));
assert!(RoutingTable::build(p(1), &graph, 16).get(&p(4)).is_none());
}
}
@@ -0,0 +1,234 @@
//! Bounded fragmentation of opaque transport payloads, below peer relaying.
//!
//! Data ALPN v3: u64 packet ID, u32 total length, u32 byte offset (big endian),
//! followed by payload. Each QUIC connection owns its IDs and reassembly state.
//! There is no retransmission: one missing fragment loses one datagram only.
use std::collections::{HashMap, VecDeque};
use bytes::{BufMut, Bytes, BytesMut};
use tokio::time::Instant;
use crate::config::{
DATA_REASSEMBLY_TTL, DATA_RECENT_IDS, MAX_DATA_ASSEMBLIES, MAX_DATA_DATAGRAM,
MAX_DATA_FRAGMENTS, MAX_DATA_REASSEMBLY_BYTES,
};
pub(super) const HEADER: usize = 16;
pub(super) fn encode(id: u64, total: usize, offset: usize, payload: &[u8]) -> Bytes {
let mut frame = BytesMut::with_capacity(HEADER + payload.len());
frame.put_u64(id);
frame.put_u32(total as u32);
frame.put_u32(offset as u32);
frame.extend_from_slice(payload);
frame.freeze()
}
#[derive(Debug)]
struct Assembly {
started: Instant,
bytes: Vec<u8>,
ranges: Vec<(usize, usize)>,
received: usize,
}
#[derive(Debug, Default)]
pub(super) struct Reassembler {
packets: HashMap<u64, Assembly>,
buffered: usize,
recent: VecDeque<u64>,
}
impl Reassembler {
fn remember(&mut self, id: u64) {
if self.recent.len() == DATA_RECENT_IDS {
self.recent.pop_front();
}
self.recent.push_back(id);
}
fn remove(&mut self, id: u64) -> Option<Assembly> {
let packet = self.packets.remove(&id)?;
self.buffered -= packet.bytes.len();
self.remember(id);
Some(packet)
}
pub(super) fn expire(&mut self, now: Instant) {
let expired: Vec<_> = self
.packets
.iter()
.filter(|(_, p)| now.duration_since(p.started) >= DATA_REASSEMBLY_TTL)
.map(|(id, _)| *id)
.collect();
for id in expired {
self.remove(id);
}
}
pub(super) fn push(&mut self, frame: Bytes, now: Instant) -> Option<Bytes> {
self.expire(now);
let id = u64::from_be_bytes(frame.get(..8)?.try_into().ok()?);
let total = u32::from_be_bytes(frame.get(8..12)?.try_into().ok()?) as usize;
let offset = u32::from_be_bytes(frame.get(12..HEADER)?.try_into().ok()?) as usize;
let payload = frame.get(HEADER..)?;
let end = offset.checked_add(payload.len())?;
if total > MAX_DATA_DATAGRAM
|| end > total
|| (payload.is_empty() && total != 0)
|| self.recent.contains(&id)
{
return None;
}
if !self.packets.contains_key(&id) {
if offset == 0 && end == total {
self.remember(id);
return Some(frame.slice(HEADER..));
}
// The length is validated before allocating. Drop the oldest
// incomplete packet when the per-link byte or entry budget is full.
while self.packets.len() >= MAX_DATA_ASSEMBLIES
|| self.buffered + total > MAX_DATA_REASSEMBLY_BYTES
{
let oldest = self
.packets
.iter()
.min_by_key(|(_, p)| p.started)
.map(|(id, _)| *id)?;
self.remove(oldest);
}
self.packets.insert(
id,
Assembly {
started: now,
bytes: vec![0; total],
ranges: Vec::new(),
received: 0,
},
);
self.buffered += total;
}
let packet = self.packets.get_mut(&id)?;
if packet.bytes.len() != total {
self.remove(id);
return None;
}
for &(start, stop) in &packet.ranges {
if start == offset && stop == end && packet.bytes[offset..end] == *payload {
return None; // a duplicate does not reset the expiry time
}
if offset < stop && end > start {
self.remove(id); // conflicting or overlapping fragments
return None;
}
}
if packet.ranges.len() == MAX_DATA_FRAGMENTS {
self.remove(id);
return None;
}
packet.bytes[offset..end].copy_from_slice(payload);
packet.ranges.push((offset, end));
packet.received += payload.len();
if packet.received == total {
return self.remove(id).map(|p| Bytes::from(p.bytes));
}
None
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn reordered_duplicates_and_changing_fragment_sizes_reassemble_once() {
let mut rx = Reassembler::default();
let now = Instant::now();
let packet: Vec<u8> = (0..9000).map(|i| (i % 251) as u8).collect();
let mut parts = Vec::new();
let mut offset = 0;
for size in [1100, 700, 1100, 1100, 700, 1100, 1100, 1100, 1000] {
parts.push(encode(
7,
packet.len(),
offset,
&packet[offset..offset + size],
));
offset += size;
}
assert_eq!(offset, packet.len());
for part in parts[1..].iter().rev() {
assert!(rx.push(part.clone(), now).is_none());
assert!(rx.push(part.clone(), now).is_none());
}
assert_eq!(rx.push(parts[0].clone(), now).unwrap().as_ref(), packet);
assert!(rx.push(parts[0].clone(), now).is_none());
assert_eq!(rx.buffered, 0);
assert!(rx.packets.is_empty());
}
#[test]
fn a_missing_fragment_never_blocks_the_next_packet_and_expires() {
let mut rx = Reassembler::default();
let now = Instant::now();
assert!(rx.push(encode(1, 1280, 0, &[1; 800]), now).is_none());
assert_eq!(
rx.push(encode(2, 4, 0, b"next"), now).unwrap(),
&b"next"[..]
);
rx.expire(now + DATA_REASSEMBLY_TTL);
assert!(rx.packets.is_empty());
assert_eq!(rx.buffered, 0);
assert!(
rx.push(encode(1, 1280, 800, &[1; 480]), now + DATA_REASSEMBLY_TTL)
.is_none()
);
}
#[test]
fn malformed_conflicting_and_oversized_fragments_are_bounded() {
let now = Instant::now();
let mut rx = Reassembler::default();
for len in 0..HEADER {
assert!(rx.push(Bytes::from(vec![0xff; len]), now).is_none());
}
assert!(
rx.push(encode(1, MAX_DATA_DATAGRAM + 1, 0, b"x"), now)
.is_none()
);
assert!(rx.push(encode(2, 100, 100, b"x"), now).is_none());
assert!(rx.push(encode(3, 100, 0, b""), now).is_none());
assert_eq!(rx.buffered, 0);
for (id, bad) in [
(4, encode(4, 100, 0, b"different")),
(5, encode(5, 100, 3, b"overlap")),
(6, encode(6, 101, 9, b"changed total")),
] {
assert!(rx.push(encode(id, 100, 0, b"123456789"), now).is_none());
assert!(rx.push(bad, now).is_none());
assert_eq!(rx.buffered, 0);
}
for id in 100..400 {
assert!(
rx.push(encode(id, MAX_DATA_DATAGRAM, 0, b"x"), now)
.is_none()
);
assert!(rx.buffered <= MAX_DATA_REASSEMBLY_BYTES);
assert!(rx.packets.len() <= MAX_DATA_ASSEMBLIES);
assert!(rx.recent.len() <= DATA_RECENT_IDS);
}
}
#[test]
fn excessive_fragment_counts_are_discarded() {
let mut rx = Reassembler::default();
let now = Instant::now();
for offset in 0..=MAX_DATA_FRAGMENTS {
assert!(rx.push(encode(1, 1000, offset, b"x"), now).is_none());
}
assert!(rx.packets.is_empty());
assert_eq!(rx.buffered, 0);
}
}
@@ -0,0 +1,389 @@
//! A data plane transport built on iroh.
//!
//! This is where the data plane gets NAT traversal from. iroh hole punches a
//! direct path between two peers when it can and falls back to a relay when it
//! cannot, so every plugin inherits that without implementing STUN, ICE or a
//! relay of its own.
//!
//! Data connections are separate from control connections in every way that
//! matters: their own ALPN ([`DATA_ALPN`]), their own QUIC connection, their
//! own congestion control. They carry one plugin protocol for one network.
//! A data connection that breaks or floods cannot disturb the control plane.
//!
//! Packets travel as QUIC datagrams: unreliable and unordered, which is what a
//! tunnelled UDP protocol wants, and free of the head-of-line blocking a
//! stream would add.
//!
//! The channel is authenticated exactly like a control connection — the same
//! membership handshake, bound to the same network — so a data link cannot be
//! opened by someone who does not know the network secret.
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use bytes::Bytes;
use iroh::EndpointId;
use iroh::endpoint::{Connection, RecvStream, SendStream};
use crate::BoxFuture;
use crate::config::{DATA_REASSEMBLY_TTL, Limits, MAX_DATA_DATAGRAM, MAX_DATA_FRAGMENTS};
use crate::error::ProtocolError;
use crate::identity::{NetworkId, NetworkKeys};
use crate::net::EndpointAdapter;
use crate::proto::handshake;
use crate::proto::message::{
DATA_ALPN, DataOpen, DataOpenAck, MAX_DATA_PROTOCOL_LEN, decode, encode,
};
use crate::proto::{read_frame, write_frame};
use super::fragments::{self, Reassembler};
use super::{InboundLink, PacketLink, PacketTransport, SharedLink, TransportError};
/// What the iroh transport needs from the agent.
///
/// Implemented by the agent, which is the only thing that knows which networks
/// are active and which plugin protocols are served.
pub trait TransportContext: Send + Sync + std::fmt::Debug + 'static {
/// Key material of every network that is active right now.
///
/// Taken as one snapshot because the membership handshake resolves the
/// requested network synchronously, exactly as the control plane's accept
/// path does.
fn snapshot<'a>(&'a self) -> BoxFuture<'a, HashMap<NetworkId, NetworkKeys>>;
/// Whether a plugin protocol is served in a network.
fn serves<'a>(&'a self, network: NetworkId, protocol: &'a str) -> BoxFuture<'a, bool>;
}
/// One authenticated datagram channel over an iroh connection.
#[derive(Debug)]
pub struct IrohLink {
network: NetworkId,
peer: EndpointId,
conn: Connection,
max_datagram: usize,
next_packet: AtomicU64,
reassembly: tokio::sync::Mutex<Reassembler>,
// Kept alive so the peer sees the channel as open; the connection closes
// when the link is dropped.
_send: tokio::sync::Mutex<SendStream>,
_recv: tokio::sync::Mutex<RecvStream>,
}
impl IrohLink {
fn new(
network: NetworkId,
peer: EndpointId,
conn: Connection,
peer_limit: usize,
send: SendStream,
recv: RecvStream,
) -> Self {
// The negotiated limit is for reassembled payloads, never a snapshot
// of the initial path (which may still be a relay or another VPN).
let max_datagram = MAX_DATA_DATAGRAM.min(peer_limit);
Self {
network,
peer,
conn,
max_datagram,
next_packet: AtomicU64::new(0),
reassembly: tokio::sync::Mutex::new(Reassembler::default()),
_send: tokio::sync::Mutex::new(send),
_recv: tokio::sync::Mutex::new(recv),
}
}
}
impl PacketLink for IrohLink {
fn network(&self) -> NetworkId {
self.network
}
fn peer(&self) -> EndpointId {
self.peer
}
fn max_datagram_size(&self) -> usize {
self.max_datagram
}
fn send(&self, payload: Bytes) -> Result<(), TransportError> {
if payload.len() > self.max_datagram {
return Err(TransportError::TooLarge {
size: payload.len(),
limit: self.max_datagram,
});
}
let id = self.next_packet.fetch_add(1, Ordering::Relaxed);
let mut offset: usize = 0;
let mut attempts = 0;
// Re-read the current path's capacity for every fragment. A migration
// can shrink it even between two send_datagram calls.
for _ in 0..MAX_DATA_FRAGMENTS {
loop {
let capacity = self
.conn
.max_datagram_size()
.unwrap_or(0)
.checked_sub(fragments::HEADER)
.filter(|n| *n > 0)
.ok_or_else(|| {
TransportError::Other("QUIC path cannot carry data fragments".into())
})?;
let end = offset.saturating_add(capacity).min(payload.len());
let frame = fragments::encode(id, payload.len(), offset, &payload[offset..end]);
match self.conn.send_datagram(frame) {
Ok(()) => {
offset = end;
break;
}
Err(iroh::endpoint::SendDatagramError::TooLarge) if attempts < 3 => {
attempts += 1;
}
Err(iroh::endpoint::SendDatagramError::ConnectionLost(_)) => {
return Err(TransportError::Closed);
}
Err(err) => return Err(TransportError::Other(err.to_string())),
}
}
if offset == payload.len() {
return Ok(());
}
}
Err(TransportError::Other(
"QUIC path needs too many fragments".into(),
))
}
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
Box::pin(async move {
let mut reassembly = self.reassembly.lock().await;
loop {
match tokio::time::timeout(DATA_REASSEMBLY_TTL, self.conn.read_datagram()).await {
Ok(Ok(frame)) => {
if let Some(packet) = reassembly.push(frame, tokio::time::Instant::now()) {
return Some(packet);
}
}
Ok(Err(_)) => {
*reassembly = Reassembler::default();
return None;
}
Err(_) => reassembly.expire(tokio::time::Instant::now()),
}
}
})
}
fn closed(&self) -> BoxFuture<'_, ()> {
Box::pin(async move {
let _ = self.conn.closed().await;
})
}
fn is_closed(&self) -> bool {
self.conn.close_reason().is_some()
}
fn path_description(&self) -> String {
// Report what iroh actually knows, never a guess.
let snapshot = crate::net::snapshot_connection(&self.conn);
match snapshot.paths.iter().find(|path| path.is_selected) {
Some(path) => format!("{} via {}", snapshot.transport, path.remote),
None => format!("{}, no selected path yet", snapshot.transport),
}
}
}
/// Opens and accepts data plane links over iroh.
#[derive(Debug, Clone)]
pub struct IrohTransport {
adapter: EndpointAdapter,
limits: Arc<Limits>,
lookup: Arc<dyn TransportContext>,
}
impl IrohTransport {
/// Creates a transport on an existing endpoint.
pub fn new(
adapter: EndpointAdapter,
limits: Arc<Limits>,
lookup: Arc<dyn TransportContext>,
) -> Self {
Self {
adapter,
limits,
lookup,
}
}
/// Completes an inbound data connection that the accept loop routed here.
///
/// The membership handshake runs first, exactly as on a control
/// connection, so an unauthenticated caller never reaches a plugin.
pub async fn accept(&self, conn: Connection) -> Result<InboundLink, TransportError> {
let peer = conn.remote_id();
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|err| TransportError::Other(format!("no data channel stream: {err}")))?;
let local_id = self.adapter.endpoint_id();
let known = self.lookup.snapshot().await;
let outcome = handshake::respond(
&conn,
&mut send,
&mut recv,
local_id,
&self.limits,
|network| known.get(&network).cloned(),
)
.await
.map_err(|err| match err {
ProtocolError::UnknownNetwork => {
TransportError::Other("network is not active for the data plane".into())
}
other => TransportError::Other(other.to_string()),
})?;
let open: DataOpen = decode(
&read_frame(&mut recv, self.limits.max_frame_len)
.await
.map_err(|err| TransportError::Other(err.to_string()))?,
)
.map_err(|err| TransportError::Other(err.to_string()))?;
if open.protocol.is_empty() || open.protocol.len() > MAX_DATA_PROTOCOL_LEN {
return Err(TransportError::Other(
"data channel protocol id is out of bounds".into(),
));
}
let serves = self.lookup.serves(outcome.network_id, &open.protocol).await;
if open.max_datagram == 0 || conn.max_datagram_size().is_none() {
return Err(TransportError::Other(
"data channel has no datagram support".into(),
));
}
let ack = DataOpenAck {
accepted: serves,
max_datagram: MAX_DATA_DATAGRAM as u32,
};
write_frame(
&mut send,
&encode(&ack).map_err(|err| TransportError::Other(err.to_string()))?,
self.limits.max_frame_len,
)
.await
.map_err(|err| TransportError::Other(err.to_string()))?;
if !serves {
conn.close(4u32.into(), b"no plugin for this protocol");
return Err(TransportError::Declined(open.protocol));
}
let link = IrohLink::new(
outcome.network_id,
peer,
conn,
open.max_datagram as usize,
send,
recv,
);
Ok(InboundLink {
network: outcome.network_id,
peer,
protocol: open.protocol,
link: Arc::new(link),
})
}
}
impl PacketTransport for IrohTransport {
fn name(&self) -> &str {
"iroh"
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn open<'a>(
&'a self,
network: NetworkId,
peer: EndpointId,
protocol: &'a str,
) -> BoxFuture<'a, Result<SharedLink, TransportError>> {
Box::pin(async move {
if protocol.is_empty() || protocol.len() > MAX_DATA_PROTOCOL_LEN {
return Err(TransportError::Other(
"data channel protocol id is out of bounds".into(),
));
}
let keys = self
.lookup
.snapshot()
.await
.remove(&network)
.ok_or_else(|| TransportError::Other("network is not active".into()))?;
let addr = iroh::EndpointAddr::new(peer);
let conn = self
.adapter
.endpoint()
.connect(addr, DATA_ALPN)
.await
.map_err(|err| TransportError::Unreachable(err.to_string()))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|err| TransportError::Unreachable(err.to_string()))?;
handshake::initiate(
&conn,
&mut send,
&mut recv,
self.adapter.endpoint_id(),
&keys,
&self.limits,
)
.await
.map_err(|err| TransportError::Other(err.to_string()))?;
let open = DataOpen {
protocol: protocol.to_string(),
max_datagram: MAX_DATA_DATAGRAM as u32,
};
write_frame(
&mut send,
&encode(&open).map_err(|err| TransportError::Other(err.to_string()))?,
self.limits.max_frame_len,
)
.await
.map_err(|err| TransportError::Other(err.to_string()))?;
let ack: DataOpenAck = decode(
&read_frame(&mut recv, self.limits.max_frame_len)
.await
.map_err(|err| TransportError::Other(err.to_string()))?,
)
.map_err(|err| TransportError::Other(err.to_string()))?;
if !ack.accepted {
conn.close(4u32.into(), b"declined");
return Err(TransportError::Declined(protocol.to_string()));
}
if ack.max_datagram == 0 || conn.max_datagram_size().is_none() {
return Err(TransportError::Other(
"data channel has no datagram support".into(),
));
}
let link = IrohLink::new(network, peer, conn, ack.max_datagram as usize, send, recv);
Ok(Arc::new(link) as SharedLink)
})
}
}
@@ -0,0 +1,207 @@
//! The data plane transport boundary.
//!
//! This is the seam that keeps the control protocol and the data plane
//! independent. An [`IpPlugin`] never learns how its packets are moved: it is
//! handed a [`PacketLink`] to a peer and writes datagrams into it. Whether
//! that link runs over iroh today, a raw UDP socket, or something else
//! entirely tomorrow is the transport's business alone.
//!
//! [`IpPlugin`]: crate::dataplane::IpPlugin
//!
//! # Why the transport may use iroh
//!
//! The separation between control and data is **logical**, not a ban on
//! sharing technology. Refusing to use iroh for data would throw away exactly
//! what iroh is good at — hole punching a direct path between two peers behind
//! NAT, with a relay as fallback — and force the data plane to reimplement it.
//! So the default transport is [`iroh_link::IrohTransport`], which gives every
//! plugin that connectivity for free.
//!
//! What the separation does buy is that the control protocol in
//! [`crate::proto`] knows nothing about packets, and this module knows nothing
//! about WireGuard. Either side can be replaced on its own.
//!
//! # Semantics
//!
//! A link is an **unreliable, unordered datagram** channel, because that is
//! what a tunnelled UDP protocol needs: no retransmission, no head-of-line
//! blocking, loss is normal rather than an error. It is authenticated and
//! encrypted by the transport, and scoped to exactly one network, one peer and
//! one plugin protocol.
mod fragments;
pub mod iroh_link;
use std::sync::Arc;
use bytes::Bytes;
use iroh::EndpointId;
use crate::BoxFuture;
use crate::identity::NetworkId;
/// Why a data plane link failed.
///
/// None of these ever stop the control plane.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum TransportError {
/// The peer is not reachable for the data plane right now.
#[error("peer is unreachable: {0}")]
Unreachable(String),
/// The peer declined to open a channel for this protocol.
#[error("peer declined a data channel for protocol `{0}`")]
Declined(String),
/// The link is closed.
#[error("data link is closed")]
Closed,
/// A datagram was larger than the link can carry.
#[error("datagram of {size} bytes exceeds the {limit} byte link limit")]
TooLarge {
/// Size that was attempted.
size: usize,
/// Largest datagram this link accepts.
limit: usize,
},
/// Anything else.
#[error("data transport error: {0}")]
Other(String),
}
/// An authenticated datagram channel to one peer, for one plugin protocol.
///
/// Dropping the link closes it.
pub trait PacketLink: Send + Sync + std::fmt::Debug + 'static {
/// The network this link belongs to.
fn network(&self) -> NetworkId;
/// The authenticated peer on the other end.
fn peer(&self) -> EndpointId;
/// The largest datagram this link can carry, in bytes.
///
/// This is the logical payload limit. A transport can fragment underneath
/// it so path MTU changes do not force a plugin to resize the host interface.
fn max_datagram_size(&self) -> usize;
/// Sends one datagram.
///
/// Delivery is not guaranteed. Returning `Ok` means the datagram was
/// handed to the transport, nothing more.
fn send(&self, payload: Bytes) -> Result<(), TransportError>;
/// Sends a datagram with an opaque stable flow identifier. Plugins derive
/// this before encryption; transports that do not route can ignore it.
fn send_flow(
&self,
payload: Bytes,
_flow: super::routing::FlowId,
) -> Result<(), TransportError> {
self.send(payload)
}
/// Receives the next datagram, or `None` once the link is finished.
fn recv(&self) -> BoxFuture<'_, Option<Bytes>>;
/// Resolves once the link is closed, for whatever reason.
fn closed(&self) -> BoxFuture<'_, ()>;
/// Whether the link is already closed.
///
/// Lets the owner notice a dead link and ask for a new one without
/// keeping a task parked on [`PacketLink::closed`].
fn is_closed(&self) -> bool;
/// A short description of the path in use, for diagnostics.
///
/// Reports what the transport actually knows. It must not invent a value.
fn path_description(&self) -> String;
}
/// A shared handle to a link.
pub type SharedLink = std::sync::Arc<dyn PacketLink>;
/// An inbound link a peer opened towards us.
#[derive(Debug)]
pub struct InboundLink {
/// The network it belongs to.
pub network: NetworkId,
/// The peer that opened it.
pub peer: EndpointId,
/// The plugin protocol it carries.
pub protocol: String,
/// The link itself.
pub link: SharedLink,
}
/// Opens and accepts data plane links.
///
/// The agent owns one of these and hands links to plugins; plugins never call
/// it directly.
pub trait PacketTransport: Send + Sync + std::fmt::Debug + 'static {
/// A short name used in diagnostics.
fn name(&self) -> &str;
/// Lets the agent recover the concrete transport to drive its accept side.
///
/// Accepting is inherently transport-specific — it starts from whatever
/// the transport's own listener produced — so it is not part of this
/// trait's uniform interface.
fn as_any(&self) -> &dyn std::any::Any;
/// Opens a link to `peer` in `network` for `protocol`.
fn open<'a>(
&'a self,
network: NetworkId,
peer: EndpointId,
protocol: &'a str,
) -> BoxFuture<'a, Result<SharedLink, TransportError>>;
}
/// A composite packet transport that dispatches requests by protocol id.
#[derive(Debug)]
pub struct MultiPacketTransport {
default: Arc<dyn PacketTransport>,
by_protocol: std::collections::HashMap<String, Arc<dyn PacketTransport>>,
}
impl MultiPacketTransport {
/// Creates a new composite transport with a default fallback transport.
pub fn new(
default: Arc<dyn PacketTransport>,
by_protocol: std::collections::HashMap<String, Arc<dyn PacketTransport>>,
) -> Self {
Self {
default,
by_protocol,
}
}
/// Access the default underlying transport.
pub fn default_transport(&self) -> &Arc<dyn PacketTransport> {
&self.default
}
}
impl PacketTransport for MultiPacketTransport {
fn name(&self) -> &str {
"multi"
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn open<'a>(
&'a self,
network: NetworkId,
peer: EndpointId,
protocol: &'a str,
) -> BoxFuture<'a, Result<SharedLink, TransportError>> {
if let Some(transport) = self.by_protocol.get(protocol) {
transport.open(network, peer, protocol)
} else {
self.default.open(network, peer, protocol)
}
}
}
+380
View File
@@ -0,0 +1,380 @@
//! Generic harness for IP tunnel plugins.
//!
//! Implementing [`super::IpPlugin`] directly requires managing per-peer background
//! tasks, synchronization, packet queues, and graceful shutdown.
//!
//! [`TunnelCodec`] provides a simplified interface where a plugin author only
//! defines cryptographic transformation and handshake payloads.
//! [`GenericTunnelPlugin`] adapts any [`TunnelCodec`] into a full [`super::IpPlugin`].
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use bytes::Bytes;
use iroh::EndpointId;
use tokio::task::JoinHandle;
use super::{IpPlugin, PluginCapability, PluginContext, PluginError, ProtocolOption, SharedLink};
use crate::BoxFuture;
use crate::identity::NetworkId;
/// The simplified contract for an IP tunnel protocol implementation.
///
/// Implementors only handle key announcements, link lifecycle events,
/// and packet encrypt/decrypt. Concurrency and task management are handled
/// by [`GenericTunnelPlugin`].
pub trait TunnelCodec: Send + Sync + 'static {
/// Protocol identifier, e.g. `tcp-tls`.
fn protocol_id(&self) -> &str;
/// Protocol wire version.
fn protocol_version(&self) -> u16;
/// Options accepted by the protocol.
fn options(&self) -> &'static [ProtocolOption] {
&[]
}
/// Generates local capability announcement data for a network.
fn local_capability_data(&self, network: NetworkId) -> Result<Vec<u8>, PluginError>;
/// Handles a peer's announced capability payload.
fn on_peer_capability_data(
&self,
network: NetworkId,
peer: EndpointId,
data: &[u8],
) -> Result<(), PluginError>;
/// Called when an authenticated link to a peer is established.
fn on_peer_link_up(&self, network: NetworkId, peer: EndpointId, link: &SharedLink) {
let _ = (network, peer, link);
}
/// Called when a peer goes away.
fn on_peer_down(&self, network: NetworkId, peer: EndpointId) {
let _ = (network, peer);
}
/// Encrypts an outbound IP packet into a payload to send over the link.
fn encrypt(
&self,
network: NetworkId,
peer: EndpointId,
packet: &[u8],
) -> Result<Bytes, PluginError>;
/// Decrypts an inbound payload from the link into an IP packet.
fn decrypt(
&self,
network: NetworkId,
peer: EndpointId,
payload: &[u8],
) -> Result<Bytes, PluginError>;
/// Called during shutdown.
fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> {
Box::pin(async {})
}
}
struct TunnelPeerState {
link: SharedLink,
tx_packets: AtomicU64,
rx_packets: AtomicU64,
tx_bytes: AtomicU64,
rx_bytes: AtomicU64,
dropped: AtomicU64,
established_at: std::time::Instant,
}
/// Statistics and status for one active peer in [`GenericTunnelPlugin`].
#[derive(Debug, Clone)]
pub struct TunnelPeerReport {
/// Peer endpoint identity.
pub peer: EndpointId,
/// Protocol identifier.
pub protocol: String,
/// Human-readable path description.
pub path: String,
/// Packets successfully encrypted and sent.
pub tx_packets: u64,
/// Packets successfully received and decrypted.
pub rx_packets: u64,
/// Plaintext bytes successfully encrypted and sent.
pub tx_bytes: u64,
/// Plaintext bytes successfully received and decrypted.
pub rx_bytes: u64,
/// Packets dropped due to encryption/decryption/send failures.
pub dropped: u64,
/// Uptime in seconds since the data link was established.
pub uptime_secs: u64,
}
/// Generic wrapper that turns a [`TunnelCodec`] into a full [`IpPlugin`].
pub struct GenericTunnelPlugin<C: TunnelCodec> {
codec: Arc<C>,
context: Mutex<Option<PluginContext>>,
links: RwLock<HashMap<(NetworkId, EndpointId), Arc<TunnelPeerState>>>,
tasks: Mutex<HashMap<(NetworkId, EndpointId), JoinHandle<()>>>,
}
impl<C: TunnelCodec> GenericTunnelPlugin<C> {
/// Wraps `codec` into an [`IpPlugin`].
pub fn new(codec: C) -> Self {
Self {
codec: Arc::new(codec),
context: Mutex::new(None),
links: RwLock::new(HashMap::new()),
tasks: Mutex::new(HashMap::new()),
}
}
/// Returns a reference to the underlying codec.
pub fn codec(&self) -> &C {
&self.codec
}
/// Returns the active peers with established data links for a network.
pub fn active_peers(&self, network: NetworkId) -> Vec<TunnelPeerReport> {
let guard = match self.links.read() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
guard
.iter()
.filter(|((net, _), _)| *net == network)
.map(|((_, peer), state)| TunnelPeerReport {
peer: *peer,
protocol: self.codec.protocol_id().to_string(),
path: state.link.path_description(),
tx_packets: state.tx_packets.load(Ordering::Relaxed),
rx_packets: state.rx_packets.load(Ordering::Relaxed),
tx_bytes: state.tx_bytes.load(Ordering::Relaxed),
rx_bytes: state.rx_bytes.load(Ordering::Relaxed),
dropped: state.dropped.load(Ordering::Relaxed),
uptime_secs: state.established_at.elapsed().as_secs(),
})
.collect()
}
}
impl<C: TunnelCodec> std::fmt::Debug for GenericTunnelPlugin<C> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GenericTunnelPlugin")
.field("protocol_id", &self.codec.protocol_id())
.field("protocol_version", &self.codec.protocol_version())
.finish()
}
}
impl<C: TunnelCodec> IpPlugin for GenericTunnelPlugin<C> {
fn protocol_id(&self) -> &str {
self.codec.protocol_id()
}
fn protocol_version(&self) -> u16 {
self.codec.protocol_version()
}
fn options(&self) -> &'static [ProtocolOption] {
self.codec.options()
}
fn attach(&self, context: PluginContext) {
if let Ok(mut guard) = self.context.lock() {
*guard = Some(context);
}
}
fn local_capability(
&self,
network: NetworkId,
) -> Result<Option<PluginCapability>, PluginError> {
let data = self.codec.local_capability_data(network)?;
Ok(Some(PluginCapability {
protocol: self.codec.protocol_id().to_string(),
version: self.codec.protocol_version(),
enabled: true,
data,
}))
}
fn on_peer_capability(
&self,
network: NetworkId,
peer: EndpointId,
capability: &PluginCapability,
) -> Result<(), PluginError> {
if capability.protocol != self.codec.protocol_id() {
return Ok(());
}
self.codec
.on_peer_capability_data(network, peer, &capability.data)
}
fn carry(&self, network: NetworkId, peer: EndpointId, packet: bytes::Bytes) -> bool {
let state = {
let guard = match self.links.read() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
guard.get(&(network, peer)).cloned()
};
let Some(state) = state else {
return false;
};
match self.codec.encrypt(network, peer, &packet) {
Ok(ciphertext) => {
if state.link.send(ciphertext).is_ok() {
state.tx_packets.fetch_add(1, Ordering::Relaxed);
state
.tx_bytes
.fetch_add(packet.len() as u64, Ordering::Relaxed);
true
} else {
state.dropped.fetch_add(1, Ordering::Relaxed);
false
}
}
Err(err) => {
state.dropped.fetch_add(1, Ordering::Relaxed);
tracing::debug!(%peer, %err, "tunnel encrypt failed");
false
}
}
}
fn on_peer_link(&self, network: NetworkId, peer: EndpointId, link: SharedLink) {
let state = Arc::new(TunnelPeerState {
link: Arc::clone(&link),
tx_packets: AtomicU64::new(0),
rx_packets: AtomicU64::new(0),
tx_bytes: AtomicU64::new(0),
rx_bytes: AtomicU64::new(0),
dropped: AtomicU64::new(0),
established_at: std::time::Instant::now(),
});
{
let mut guard = match self.links.write() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
guard.insert((network, peer), Arc::clone(&state));
}
self.codec.on_peer_link_up(network, peer, &link);
let codec = Arc::clone(&self.codec);
let sink = match self.context.lock() {
Ok(g) => g.as_ref().map(|ctx| ctx.packet_sink()),
Err(p) => p.into_inner().as_ref().map(|ctx| ctx.packet_sink()),
};
let rx_link = Arc::clone(&link);
let rx_state = Arc::clone(&state);
let handle = tokio::spawn(async move {
while let Some(payload) = rx_link.recv().await {
match codec.decrypt(network, peer, &payload) {
Ok(packet) => {
rx_state.rx_packets.fetch_add(1, Ordering::Relaxed);
rx_state
.rx_bytes
.fetch_add(packet.len() as u64, Ordering::Relaxed);
if let Some(ref sink) = sink {
sink.deliver(network, peer, packet).await;
}
}
Err(err) => {
rx_state.dropped.fetch_add(1, Ordering::Relaxed);
tracing::debug!(%peer, %err, "tunnel decrypt failed");
}
}
}
});
let mut guard = match self.tasks.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
if let Some(old) = guard.insert((network, peer), handle) {
old.abort();
}
}
fn on_peer_gone(&self, network: NetworkId, peer: EndpointId) {
{
let mut guard = match self.tasks.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
if let Some(task) = guard.remove(&(network, peer)) {
task.abort();
}
}
{
let mut guard = match self.links.write() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
guard.remove(&(network, peer));
}
self.codec.on_peer_down(network, peer);
}
fn on_network_deactivated(&self, network: NetworkId) {
let keys_to_remove: Vec<(NetworkId, EndpointId)> = {
let guard = match self.tasks.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
guard
.keys()
.filter(|(net, _)| *net == network)
.copied()
.collect()
};
{
let mut guard = match self.tasks.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
for key in &keys_to_remove {
if let Some(task) = guard.remove(key) {
task.abort();
}
}
}
{
let mut guard = match self.links.write() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
guard.retain(|(net, _), _| *net != network);
}
}
fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> {
Box::pin(async move {
let mut tasks = Vec::new();
{
let mut guard = match self.tasks.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
for (_, task) in guard.drain() {
tasks.push(task);
}
}
for task in tasks {
task.abort();
}
self.codec.shutdown().await;
})
}
}
+369
View File
@@ -0,0 +1,369 @@
//! Finding *candidates*, and nothing more.
//!
//! Discovery answers one question: "which iroh endpoints might currently be
//! participating in the network behind this [`DiscoveryKey`], and at which
//! addresses?". Its answers are **unverified candidates**. Membership is decided
//! later, by the control protocol handshake in [`crate::proto::handshake`].
//!
//! A discovery backend must not carry control messages between agents, must not
//! confirm authentication and must not mutate agent state directly.
//!
//! Two concerns are kept apart, and only the first one is this module's:
//!
//! * *Finding members of a network* — [`NetworkDiscovery::resolve`], keyed by
//! the secret-derived [`DiscoveryKey`]. That is what lives here, and today
//! it includes [`StaticBootstrap`], [`MainlineDiscovery`] and a test backend.
//! * *Resolving the address of one iroh endpoint* — **iroh's job, not ours**.
//! With [`crate::config::TransportPolicy::N0Defaults`] or `DirectOnly`, iroh
//! publishes and resolves endpoint addresses through Number 0's public
//! service, so dialling a bare [`EndpointId`] works. With `LocalOnly` there
//! is no lookup, and a candidate must carry addresses of its own.
//!
//! No empty result ever proves a network is empty. It only means "nobody found
//! yet".
//!
//! Mainline rendezvous is opt-in for library callers; the command line enables it.
mod mainline;
pub(crate) mod worker;
pub use mainline::MainlineDiscovery;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use iroh::{EndpointAddr, EndpointId};
use crate::error::{Error, Result};
use crate::identity::DiscoveryKey;
pub use crate::BoxFuture;
/// Where a candidate came from. Purely informational.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CandidateSource {
/// A statically configured bootstrap entry.
Bootstrap,
/// A discovery backend lookup.
Discovery,
/// An address hint restored from the disposable cache.
Cache,
/// An author of a signed record: somebody who belongs to this network,
/// learned from state that reached us through anybody.
///
/// It carries no address of its own — the endpoint's own discovery has
/// to resolve it — but knowing that a member exists is what turns a
/// star around whoever was named on the command line into a mesh.
Member,
/// Passed on by a member we are talking to.
///
/// A candidate like any other: an introduction is not a vouching, and
/// membership is still decided by the handshake.
Introduced,
}
/// An unverified candidate peer.
///
/// Holding one grants nothing: the peer still has to pass the handshake.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Candidate {
/// iroh address of the candidate, including whatever addressing info exists.
pub addr: EndpointAddr,
/// Where this candidate came from.
pub source: CandidateSource,
}
impl Candidate {
/// Creates a candidate.
pub fn new(addr: EndpointAddr, source: CandidateSource) -> Self {
Self { addr, source }
}
/// The candidate's endpoint id.
pub fn endpoint_id(&self) -> EndpointId {
self.addr.id
}
}
/// A replaceable source of candidates.
///
/// Implementations must be cheap to clone behind an [`Arc`] and must never
/// block the async executor.
pub trait NetworkDiscovery: Send + Sync + std::fmt::Debug + 'static {
/// A short name used in diagnostics.
fn name(&self) -> &str;
/// Publishes this agent's address under `key`.
///
/// Backends that cannot publish (static bootstrap lists) return `Ok(())`.
fn publish<'a>(&'a self, key: DiscoveryKey, addr: EndpointAddr) -> BoxFuture<'a, Result<()>>;
/// Withdraws a previously published address.
fn unpublish<'a>(
&'a self,
key: DiscoveryKey,
endpoint: EndpointId,
) -> BoxFuture<'a, Result<()>>;
/// Returns the candidates currently known for `key`.
fn resolve<'a>(&'a self, key: DiscoveryKey) -> BoxFuture<'a, Result<Vec<Candidate>>>;
/// Delivers candidates as they arrive. The default adapts a batch backend;
/// remote backends can override this so the first dial need not wait for all lookups.
fn resolve_into<'a>(
&'a self,
key: DiscoveryKey,
candidates: tokio::sync::mpsc::Sender<Candidate>,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
for candidate in self.resolve(key).await? {
if candidates.send(candidate).await.is_err() {
break;
}
}
Ok(())
})
}
}
/// A statically configured list of bootstrap candidates.
///
/// Each entry must carry enough addressing information to be dialled, i.e. an
/// iroh endpoint id plus direct addresses or a relay URL, unless iroh's own
/// address lookup is enabled in [`crate::config::TransportPolicy`].
#[derive(Debug, Clone, Default)]
pub struct StaticBootstrap {
entries: Vec<EndpointAddr>,
}
impl StaticBootstrap {
/// Creates a bootstrap list.
pub fn new(entries: impl IntoIterator<Item = EndpointAddr>) -> Self {
Self {
entries: entries.into_iter().collect(),
}
}
}
impl NetworkDiscovery for StaticBootstrap {
fn name(&self) -> &str {
"static-bootstrap"
}
fn publish<'a>(&'a self, _key: DiscoveryKey, _addr: EndpointAddr) -> BoxFuture<'a, Result<()>> {
Box::pin(async { Ok(()) })
}
fn unpublish<'a>(
&'a self,
_key: DiscoveryKey,
_endpoint: EndpointId,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async { Ok(()) })
}
fn resolve<'a>(&'a self, _key: DiscoveryKey) -> BoxFuture<'a, Result<Vec<Candidate>>> {
let candidates: Vec<Candidate> = self
.entries
.iter()
.cloned()
.map(|addr| Candidate::new(addr, CandidateSource::Bootstrap))
.collect();
Box::pin(async move { Ok(candidates) })
}
}
/// An in-process discovery backend used by tests and examples.
///
/// It stores a mapping from [`DiscoveryKey`] to endpoint addresses and nothing
/// else. It carries no messages, performs no authentication and cannot touch an
/// agent's state. Clone it to hand the same rendezvous table to several agents;
/// create a new one per test so that tests stay independent — there is no global
/// mutable state here.
#[derive(Debug, Clone, Default)]
pub struct SharedMemoryDiscovery {
inner: Arc<Mutex<HashMap<DiscoveryKey, HashMap<EndpointId, EndpointAddr>>>>,
}
impl SharedMemoryDiscovery {
/// Creates an empty rendezvous table.
pub fn new() -> Self {
Self::default()
}
/// Number of entries published under `key`. Useful in tests.
pub fn len(&self, key: &DiscoveryKey) -> usize {
self.with_inner(|map| map.get(key).map_or(0, HashMap::len))
}
/// Whether nothing is published under `key`.
pub fn is_empty(&self, key: &DiscoveryKey) -> bool {
self.len(key) == 0
}
/// Removes every entry under `key`, simulating a discovery outage.
pub fn clear(&self, key: &DiscoveryKey) {
self.with_inner(|map| {
map.remove(key);
});
}
/// Replaces an entry with a deliberately wrong address, simulating a stale
/// or poisoned record.
pub fn insert_raw(&self, key: DiscoveryKey, addr: EndpointAddr) {
self.with_inner(|map| {
map.entry(key).or_default().insert(addr.id, addr);
});
}
fn with_inner<T>(
&self,
f: impl FnOnce(&mut HashMap<DiscoveryKey, HashMap<EndpointId, EndpointAddr>>) -> T,
) -> T {
let mut guard = match self.inner.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
f(&mut guard)
}
}
impl NetworkDiscovery for SharedMemoryDiscovery {
fn name(&self) -> &str {
"shared-memory"
}
fn publish<'a>(&'a self, key: DiscoveryKey, addr: EndpointAddr) -> BoxFuture<'a, Result<()>> {
self.with_inner(|map| {
map.entry(key).or_default().insert(addr.id, addr);
});
Box::pin(async { Ok(()) })
}
fn unpublish<'a>(
&'a self,
key: DiscoveryKey,
endpoint: EndpointId,
) -> BoxFuture<'a, Result<()>> {
self.with_inner(|map| {
if let Some(entries) = map.get_mut(&key) {
entries.remove(&endpoint);
if entries.is_empty() {
map.remove(&key);
}
}
});
Box::pin(async { Ok(()) })
}
fn resolve<'a>(&'a self, key: DiscoveryKey) -> BoxFuture<'a, Result<Vec<Candidate>>> {
let candidates: Vec<Candidate> = self.with_inner(|map| {
map.get(&key)
.map(|entries| {
entries
.values()
.cloned()
.map(|addr| Candidate::new(addr, CandidateSource::Discovery))
.collect()
})
.unwrap_or_default()
});
Box::pin(async move { Ok(candidates) })
}
}
/// Combines several backends, concatenating their candidates.
#[derive(Debug, Clone)]
pub struct CompositeDiscovery {
backends: Vec<Arc<dyn NetworkDiscovery>>,
}
impl CompositeDiscovery {
/// Creates a composite over the given backends.
pub fn new(backends: impl IntoIterator<Item = Arc<dyn NetworkDiscovery>>) -> Self {
Self {
backends: backends.into_iter().collect(),
}
}
}
impl NetworkDiscovery for CompositeDiscovery {
fn name(&self) -> &str {
"composite"
}
fn publish<'a>(&'a self, key: DiscoveryKey, addr: EndpointAddr) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
let mut tasks = tokio::task::JoinSet::new();
for backend in &self.backends {
let backend = backend.clone();
let addr = addr.clone();
tasks.spawn(async move { backend.publish(key, addr).await });
}
let mut failed = false;
while let Some(result) = tasks.join_next().await {
if !matches!(result, Ok(Ok(()))) {
failed = true;
}
}
if failed {
Err(Error::Discovery(
"a discovery publication failed; will retry".into(),
))
} else {
Ok(())
}
})
}
fn unpublish<'a>(
&'a self,
key: DiscoveryKey,
endpoint: EndpointId,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
let mut tasks = tokio::task::JoinSet::new();
for backend in &self.backends {
let backend = backend.clone();
tasks.spawn(async move { backend.unpublish(key, endpoint).await });
}
while tasks.join_next().await.is_some() {}
Ok(())
})
}
fn resolve<'a>(&'a self, key: DiscoveryKey) -> BoxFuture<'a, Result<Vec<Candidate>>> {
Box::pin(async move {
let mut out = Vec::new();
for backend in &self.backends {
match backend.resolve(key).await {
Ok(mut found) => out.append(&mut found),
Err(err) => {
tracing::debug!(backend = backend.name(), %err, "resolve failed");
}
}
}
Ok(out)
})
}
fn resolve_into<'a>(
&'a self,
key: DiscoveryKey,
candidates: tokio::sync::mpsc::Sender<Candidate>,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
let mut tasks = tokio::task::JoinSet::new();
for backend in &self.backends {
let backend = backend.clone();
let candidates = candidates.clone();
tasks.spawn(async move {
if let Err(err) = backend.resolve_into(key, candidates).await {
tracing::debug!(backend = backend.name(), %err, "resolve failed");
}
});
}
while tasks.join_next().await.is_some() {}
Ok(())
})
}
}
@@ -0,0 +1,516 @@
//! Mainline BEP44 rendezvous. Slots are a changing sample, not a membership list.
use ::mainline::{Dht, MutableItem, SigningKey, async_dht::AsyncDht};
use futures_lite::StreamExt;
use iroh::{EndpointAddr, EndpointId};
use sha2::{Digest, Sha256};
use std::collections::{HashSet, VecDeque};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::{Mutex as AsyncMutex, mpsc};
use tokio::task::JoinSet;
use tokio::time::timeout;
use zeroize::Zeroizing;
use super::{BoxFuture, Candidate, CandidateSource, NetworkDiscovery};
use crate::config::{
DHT_CLOCK_SKEW, DHT_MAX_ADDRS, DHT_MAX_RELAY_LEN, DHT_MAX_VALUE, DHT_RECORD_TTL, DHT_SLOTS,
};
use crate::error::{Error, Result};
use crate::identity::{DiscoveryKey, NetworkKeys};
const QUERY_TIMEOUT: Duration = Duration::from_secs(8);
const MAGIC: &[u8; 5] = b"TSND\x01";
/// One lazily started Mainline client, shared by all networks on one agent.
/// No socket is opened until an active network publishes or looks up candidates.
/// Agent shutdown closes this client and all its clones; a new agent needs a new client.
#[derive(Debug, Clone, Default)]
pub struct MainlineDiscovery {
client: Arc<AsyncMutex<ClientState>>,
allow_loopback: bool,
}
#[derive(Debug, Default)]
enum ClientState {
#[default]
Pending,
Ready(AsyncDht),
Stopped,
}
impl MainlineDiscovery {
/// Uses a caller-supplied Mainline node instead of public bootstrap defaults.
pub fn from_dht(dht: Dht) -> Self {
Self {
client: Arc::new(AsyncMutex::new(ClientState::Ready(dht.as_async()))),
allow_loopback: false,
}
}
/// Creates a loopback-only client for a local Mainline Testnet.
#[cfg(feature = "testing")]
pub fn local_testnet(bootstrap: &[String]) -> Result<Self> {
// Reject anything that could resolve or send outside loopback.
if bootstrap.is_empty()
|| bootstrap.iter().any(|s| {
s.parse::<SocketAddr>()
.map_or(true, |a| !a.ip().is_loopback())
})
{
return Err(Error::Discovery(
"test bootstrap must contain loopback sockets".into(),
));
}
let dht = Dht::builder()
.bootstrap(bootstrap)
.bind_address(Ipv4Addr::LOCALHOST)
.port(0)
.request_timeout(Duration::from_millis(200))
.build()
.map_err(|_| Error::Discovery("cannot bind local DHT".into()))?;
Ok(Self {
allow_loopback: true,
..Self::from_dht(dht)
})
}
/// Binds this client's rendezvous backend to one network's independent keys.
pub fn for_network(&self, keys: &NetworkKeys) -> Arc<dyn NetworkDiscovery> {
Arc::new(NetworkDht {
client: self.clone(),
key: keys.discovery_key(),
seed: Zeroizing::new(*keys.dht_write_key()),
observed: Mutex::new(VecDeque::new()),
})
}
async fn client(&self) -> Result<AsyncDht> {
let mut client = self.client.lock().await;
match &*client {
ClientState::Ready(dht) => return Ok(dht.clone()),
ClientState::Stopped => return Err(Error::Discovery("DHT client is stopped".into())),
ClientState::Pending => {}
}
// Construction may resolve bootstrap hostnames. Keep it off Tokio's
// executor; a construction failure is retried on the next operation.
let dht = tokio::task::spawn_blocking(|| Dht::builder().port(0).build())
.await
.map_err(|_| Error::Discovery("DHT startup task failed".into()))?
.map(Dht::as_async)
.map_err(|_| Error::Discovery("cannot start DHT client".into()))?;
*client = ClientState::Ready(dht.clone());
Ok(dht)
}
pub(crate) async fn shutdown(&self) {
*self.client.lock().await = ClientState::Stopped;
}
}
struct NetworkDht {
client: MainlineDiscovery,
key: DiscoveryKey,
seed: Zeroizing<[u8; 32]>,
// Records encountered while publishing. Only an explicit lookup consumes
// these hints: publication never causes dials on a connected network.
observed: Mutex<VecDeque<Vec<u8>>>,
}
impl std::fmt::Debug for NetworkDht {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("NetworkDht(<redacted>)")
}
}
fn salt(slot: u8) -> Vec<u8> {
let mut salt = b"tsunagi-rendezvous-v1".to_vec();
salt.push(slot);
salt
}
fn slots_for(id: EndpointId) -> [u8; 2] {
let hash = Sha256::digest(id.as_bytes());
let first = hash[0] % DHT_SLOTS;
[first, (first + 1 + hash[1] % (DHT_SLOTS - 1)) % DHT_SLOTS]
}
fn now() -> Result<u64> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.map_err(|_| Error::Discovery("system clock predates Unix epoch".into()))
}
impl NetworkDht {
async fn publish_slot(&self, dht: &AsyncDht, slot: u8, value: &[u8]) -> Result<()> {
let salt = salt(slot);
let signer = SigningKey::from_bytes(&self.seed);
let public = signer.verifying_key().to_bytes();
for attempt in 0..3 {
let recent = timeout(
QUERY_TIMEOUT,
dht.get_mutable_most_recent(&public, Some(&salt)),
)
.await
.map_err(|_| Error::Discovery("DHT read timed out".into()))?;
if let Some(item) = &recent
&& item.value().len() <= DHT_MAX_VALUE
&& let Ok(mut observed) = self.observed.lock()
{
if observed.len() == DHT_SLOTS as usize {
observed.pop_front();
}
observed.push_back(item.value().to_vec());
}
let seq = recent
.as_ref()
.map_or(0, MutableItem::seq)
.checked_add(1)
.ok_or_else(|| Error::Discovery("DHT sequence exhausted".into()))?;
let item = MutableItem::new(signer.clone(), value, seq, Some(&salt));
match timeout(
QUERY_TIMEOUT,
dht.put_mutable(item, recent.as_ref().map(MutableItem::seq)),
)
.await
{
Ok(Ok(outcome)) if outcome.stored_at > 0 => return Ok(()),
_ if attempt < 2 => {
// CAS is per storage node, not a global lock. Read again on
// conflicts; another writer's entry is an acceptable sample.
tokio::time::sleep(Duration::from_millis(rand::random_range(30..150))).await;
}
_ => break,
}
}
Err(Error::Discovery(
"DHT publication was not acknowledged".into(),
))
}
}
impl NetworkDiscovery for NetworkDht {
fn name(&self) -> &str {
"mainline"
}
fn publish<'a>(&'a self, key: DiscoveryKey, addr: EndpointAddr) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
if key != self.key {
return Err(Error::Discovery("wrong rendezvous network".into()));
}
let value = encode_record(&addr, now()?, self.client.allow_loopback)?;
let dht = self.client.client().await?;
let [first, second] = slots_for(addr.id);
let (a, b) = tokio::join!(
self.publish_slot(&dht, first, &value),
self.publish_slot(&dht, second, &value)
);
a.and(b)
})
}
fn unpublish<'a>(&'a self, _: DiscoveryKey, _: EndpointId) -> BoxFuture<'a, Result<()>> {
// Stopping publication lets freshness expire. Deleting a shared slot
// could erase a different writer, and BEP44 has no delete operation.
Box::pin(async { Ok(()) })
}
fn resolve<'a>(&'a self, key: DiscoveryKey) -> BoxFuture<'a, Result<Vec<Candidate>>> {
Box::pin(async move {
let (tx, mut rx) = mpsc::channel(DHT_SLOTS as usize);
let collect = async move {
let mut found = Vec::new();
while let Some(candidate) = rx.recv().await {
found.push(candidate);
}
found
};
let (result, found) = tokio::join!(self.resolve_into(key, tx), collect);
result?;
Ok(found)
})
}
fn resolve_into<'a>(
&'a self,
key: DiscoveryKey,
candidates: mpsc::Sender<Candidate>,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
if key != self.key {
return Err(Error::Discovery("wrong rendezvous network".into()));
}
let dht = self.client.client().await?;
let public = SigningKey::from_bytes(&self.seed)
.verifying_key()
.to_bytes();
let observed = self
.observed
.lock()
.map(|mut values| std::mem::take(&mut *values))
.unwrap_or_default();
let mut seen = HashSet::new();
for value in observed {
if let Some(addr) = decode_record(&value, now()?, self.client.allow_loopback)
&& seen.insert(addr.id)
&& candidates
.send(Candidate::new(addr, CandidateSource::Discovery))
.await
.is_err()
{
return Ok(());
}
}
let mut slots: Vec<_> = (0..DHT_SLOTS).collect();
for i in (1..slots.len()).rev() {
slots.swap(i, rand::random_range(0..=i));
}
let mut queries = JoinSet::new();
loop {
if seen.len() >= DHT_SLOTS as usize {
break;
}
while queries.len() < 4 {
let Some(slot) = slots.pop() else {
break;
};
let dht = dht.clone();
let allow_loopback = self.client.allow_loopback;
queries.spawn(async move {
timeout(QUERY_TIMEOUT, async move {
let salt = salt(slot);
let target = MutableItem::target_from_key(&public, Some(&salt));
let mut items = dht.get_mutable(&public, Some(&salt), None);
while let Some(item) = items.next().await {
// Mainline verifies the signature; also bind the
// returned item to the key and slot we requested.
if item.key() != &public || item.target() != &target {
continue;
}
if let Some(addr) =
decode_record(item.value(), now().ok()?, allow_loopback)
{
return Some(addr);
}
}
None
})
.await
.ok()
.flatten()
});
}
let Some(result) = queries.join_next().await else {
break;
};
if let Ok(Some(addr)) = result
&& seen.insert(addr.id)
&& candidates
.send(Candidate::new(addr, CandidateSource::Discovery))
.await
.is_err()
{
break;
}
}
Ok(())
})
}
}
fn usable(addr: &SocketAddr, allow_loopback: bool) -> bool {
addr.port() != 0
&& !addr.ip().is_unspecified()
&& !addr.ip().is_multicast()
&& (allow_loopback || !addr.ip().is_loopback())
&& match addr.ip() {
IpAddr::V4(ip) => !ip.is_broadcast() && !ip.is_link_local(),
IpAddr::V6(ip) => !ip.is_unicast_link_local(),
}
}
fn encode_record(addr: &EndpointAddr, timestamp: u64, allow_loopback: bool) -> Result<Vec<u8>> {
let ips: Vec<_> = addr
.ip_addrs()
.filter(|ip| usable(ip, allow_loopback))
.take(DHT_MAX_ADDRS)
.collect();
let relay = addr
.relay_urls()
.next()
.map(ToString::to_string)
.unwrap_or_default();
if relay.len() > DHT_MAX_RELAY_LEN || (ips.is_empty() && relay.is_empty()) {
return Err(Error::Discovery("no encodable endpoint address yet".into()));
}
let mut bytes = Vec::with_capacity(DHT_MAX_VALUE);
bytes.extend_from_slice(MAGIC);
bytes.extend_from_slice(&timestamp.to_be_bytes());
bytes.extend_from_slice(addr.id.as_bytes());
bytes.extend_from_slice(&(relay.len() as u16).to_be_bytes());
bytes.extend_from_slice(relay.as_bytes());
bytes.push(ips.len() as u8);
for ip in ips {
match ip.ip() {
IpAddr::V4(v) => {
bytes.push(4);
bytes.extend_from_slice(&v.octets());
}
IpAddr::V6(v) => {
bytes.push(6);
bytes.extend_from_slice(&v.octets());
}
}
bytes.extend_from_slice(&ip.port().to_be_bytes());
}
if bytes.len() > DHT_MAX_VALUE {
return Err(Error::Discovery("DHT record too large".into()));
}
Ok(bytes)
}
fn take<'a>(input: &mut &'a [u8], n: usize) -> Option<&'a [u8]> {
let (head, rest) = input.split_at_checked(n)?;
*input = rest;
Some(head)
}
fn decode_record(mut bytes: &[u8], now: u64, allow_loopback: bool) -> Option<EndpointAddr> {
if bytes.len() > DHT_MAX_VALUE || take(&mut bytes, MAGIC.len())? != MAGIC {
return None;
}
let timestamp = u64::from_be_bytes(take(&mut bytes, 8)?.try_into().ok()?);
if timestamp > now.saturating_add(DHT_CLOCK_SKEW.as_secs())
|| now.saturating_sub(timestamp) >= DHT_RECORD_TTL.as_secs()
{
return None;
}
let id = EndpointId::from_bytes(take(&mut bytes, 32)?.try_into().ok()?).ok()?;
let mut addr = EndpointAddr::new(id);
let len = u16::from_be_bytes(take(&mut bytes, 2)?.try_into().ok()?) as usize;
if len > DHT_MAX_RELAY_LEN {
return None;
}
let relay = std::str::from_utf8(take(&mut bytes, len)?).ok()?;
if !relay.is_empty() {
addr = addr.with_relay_url(relay.parse().ok()?);
}
let count = *take(&mut bytes, 1)?.first()? as usize;
if count > DHT_MAX_ADDRS {
return None;
}
for _ in 0..count {
let ip = match *take(&mut bytes, 1)?.first()? {
4 => IpAddr::V4(Ipv4Addr::from(
<[u8; 4]>::try_from(take(&mut bytes, 4)?).ok()?,
)),
6 => IpAddr::V6(Ipv6Addr::from(
<[u8; 16]>::try_from(take(&mut bytes, 16)?).ok()?,
)),
_ => return None,
};
let port = u16::from_be_bytes(take(&mut bytes, 2)?.try_into().ok()?);
let socket = SocketAddr::new(ip, port);
if usable(&socket, allow_loopback) {
addr = addr.with_ip_addr(socket);
}
}
(bytes.is_empty() && !addr.addrs.is_empty()).then_some(addr)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::testing::network;
fn address() -> EndpointAddr {
EndpointAddr::new(iroh::SecretKey::generate().public())
.with_ip_addr("127.0.0.1:12345".parse().unwrap())
}
#[test]
fn records_reject_expiration_future_dates_truncation_and_oversize() {
let addr = address();
let bytes = encode_record(&addr, 1_000, true).unwrap();
assert_eq!(decode_record(&bytes, 1_100, true).unwrap(), addr);
assert!(decode_record(&bytes, 1_901, true).is_none());
assert!(decode_record(&bytes, 100, true).is_none());
for n in 0..bytes.len() {
assert!(decode_record(&bytes[..n], 1_100, true).is_none());
}
assert!(decode_record(&vec![0; DHT_MAX_VALUE + 1], 1_100, true).is_none());
assert!(encode_record(&addr, 1_000, false).is_err());
assert!(decode_record(&bytes, 1_100, false).is_none());
}
#[tokio::test]
async fn a_stopped_custom_client_never_falls_back_to_public_bootstrap() {
let net = mainline::Testnet::builder(3).build().unwrap();
let client = MainlineDiscovery::local_testnet(&net.bootstrap).unwrap();
let (name, secret) = network("mainline-shutdown");
let keys = NetworkKeys::derive(&name, &secret);
let backend = client.for_network(&keys);
client.shutdown().await;
assert!(backend.resolve(keys.discovery_key()).await.is_err());
}
#[tokio::test]
async fn concurrent_writers_share_slots_and_other_secrets_find_nothing() {
let net = mainline::Testnet::builder(5).build().unwrap();
let client_a = MainlineDiscovery::local_testnet(&net.bootstrap).unwrap();
let client_b = MainlineDiscovery::local_testnet(&net.bootstrap).unwrap();
let (name, secret) = network("mainline-slots");
let keys = NetworkKeys::derive(&name, &secret);
let a = client_a.for_network(&keys);
let b = client_b.for_network(&keys);
let first = address();
let second = loop {
let next = address();
if slots_for(next.id) == slots_for(first.id) {
break next;
}
};
let key = keys.discovery_key();
let (ra, rb) = tokio::join!(
a.publish(key, first.clone()),
b.publish(key, second.clone())
);
assert!(ra.is_ok() || rb.is_ok());
let (found_a, found_b) = tokio::join!(a.resolve(key), b.resolve(key));
let found_a = found_a.unwrap();
let found_b = found_b.unwrap();
assert!(
found_a.iter().any(|c| c.addr == second) || found_b.iter().any(|c| c.addr == first),
"at least one writer must discover the other despite colliding in both slots"
);
assert!(
found_a
.iter()
.chain(&found_b)
.all(|c| c.addr == first || c.addr == second)
);
let other = NetworkKeys::derive(&name, &crate::identity::NetworkSecret::generate());
assert!(
client_a
.for_network(&other)
.resolve(other.discovery_key())
.await
.unwrap()
.is_empty()
);
// Restarting the publisher must read seq from DHT instead of starting at 1.
let restarted = client_a.for_network(&keys);
restarted.publish(key, first.clone()).await.unwrap();
assert!(
b.resolve(key)
.await
.unwrap()
.iter()
.any(|c| c.addr == first)
);
}
}
+251
View File
@@ -0,0 +1,251 @@
//! Per-network discovery tasks. Dropping the owner cancels all backend futures.
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{mpsc, watch};
use tokio::task::JoinSet;
use tokio::time::{Instant, timeout};
use super::{Candidate, NetworkDiscovery};
use crate::config::DiscoveryPolicy;
use crate::identity::DiscoveryKey;
use crate::net::EndpointAdapter;
pub(crate) struct DiscoveryWorker {
connected: watch::Sender<Connectivity>,
tasks: JoinSet<()>,
backend: Arc<dyn NetworkDiscovery>,
key: DiscoveryKey,
endpoint: iroh::EndpointId,
}
#[derive(Clone, Copy)]
enum Connectivity {
Initial,
Connected,
Isolated(Instant),
}
impl DiscoveryWorker {
pub(crate) fn spawn(
backend: Arc<dyn NetworkDiscovery>,
key: DiscoveryKey,
adapter: EndpointAdapter,
policy: DiscoveryPolicy,
address_check: Duration,
candidates: mpsc::Sender<Candidate>,
) -> Self {
let (connected, receiver) = watch::channel(Connectivity::Initial);
let mut tasks = JoinSet::new();
tasks.spawn(publish_loop(
backend.clone(),
key,
adapter.clone(),
policy.clone(),
address_check,
));
tasks.spawn(lookup_loop(
backend.clone(),
key,
policy,
receiver,
candidates,
));
Self {
connected,
tasks,
backend,
key,
endpoint: adapter.endpoint_id(),
}
}
pub(crate) fn set_connected(&self, connected: bool) {
self.connected.send_if_modified(|current| {
if matches!(*current, Connectivity::Connected) == connected {
return false;
}
*current = if connected {
Connectivity::Connected
} else {
Connectivity::Isolated(Instant::now())
};
true
});
}
pub(crate) async fn stop(mut self) {
self.tasks.abort_all();
while self.tasks.join_next().await.is_some() {}
// BEP44 has no deletion. Memory/static backends can withdraw promptly;
// an unresponsive backend must not extend agent shutdown.
let _ = timeout(
Duration::from_millis(250),
self.backend.unpublish(self.key, self.endpoint),
)
.await;
}
}
fn jitter(delay: Duration) -> Duration {
delay
.mul_f64(0.8 + 0.4 * rand::random::<f64>())
.max(Duration::from_millis(10))
}
async fn publish_loop(
backend: Arc<dyn NetworkDiscovery>,
key: DiscoveryKey,
adapter: EndpointAdapter,
policy: DiscoveryPolicy,
address_check: Duration,
) {
let mut previous = None;
let mut due = Instant::now();
let mut ticker = tokio::time::interval(address_check.max(Duration::from_millis(10)));
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
let mut addr = adapter.addr();
addr.addrs.extend(adapter.loopback_addr().addrs);
if previous.as_ref() != Some(&addr) {
previous = Some(addr.clone());
due = Instant::now();
}
if Instant::now() < due {
continue;
}
let result = timeout(policy.request_timeout, backend.publish(key, addr.clone())).await;
match result {
Ok(Ok(())) => {
due = Instant::now() + jitter(policy.publish_interval);
}
result => {
tracing::debug!(?result, "discovery publication failed; will retry");
due = Instant::now() + jitter(policy.lookup_interval);
}
}
}
}
async fn lookup_loop(
backend: Arc<dyn NetworkDiscovery>,
key: DiscoveryKey,
policy: DiscoveryPolicy,
mut connected: watch::Receiver<Connectivity>,
candidates: mpsc::Sender<Candidate>,
) {
let mut last_isolation = None;
let mut due = Instant::now();
let mut delay = policy.lookup_interval;
loop {
let state = *connected.borrow_and_update();
if matches!(state, Connectivity::Connected) {
if connected.changed().await.is_err() {
break;
}
continue;
}
if let Connectivity::Isolated(since) = state
&& last_isolation != Some(since)
{
// Preserve the transition time even if a rapid reconnect/disconnect
// overwrote a watch value before this task had a chance to run.
due = since + policy.reconnect_delay;
delay = policy.lookup_interval;
last_isolation = Some(since);
}
tokio::select! {
biased;
changed = connected.changed() => {
if changed.is_err() { break; }
continue;
}
_ = tokio::time::sleep_until(due) => {}
}
tokio::select! {
biased;
changed = connected.changed() => {
if changed.is_err() { break; }
continue;
}
result = timeout(policy.request_timeout, backend.resolve_into(key, candidates.clone())) => {
if !matches!(result, Ok(Ok(()))) {
tracing::debug!(?result, "discovery lookup failed; will retry");
}
}
}
due = Instant::now() + jitter(delay);
delay = delay.saturating_mul(2).min(policy.max_lookup_interval);
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::Result;
use crate::discovery::BoxFuture;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Debug, Default)]
struct Reads(AtomicUsize);
impl NetworkDiscovery for Reads {
fn name(&self) -> &str {
"reads"
}
fn publish<'a>(
&'a self,
_: DiscoveryKey,
_: iroh::EndpointAddr,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async { Ok(()) })
}
fn unpublish<'a>(
&'a self,
_: DiscoveryKey,
_: iroh::EndpointId,
) -> BoxFuture<'a, Result<()>> {
Box::pin(async { Ok(()) })
}
fn resolve<'a>(&'a self, _: DiscoveryKey) -> BoxFuture<'a, Result<Vec<Candidate>>> {
self.0.fetch_add(1, Ordering::SeqCst);
Box::pin(async { Ok(Vec::new()) })
}
}
#[tokio::test(start_paused = true)]
async fn recovery_requires_a_full_minute_and_reconnection_restarts_that_minute() {
let reads = Arc::new(Reads::default());
let (tx, rx) = watch::channel(Connectivity::Connected);
let (candidates, _receiver) = mpsc::channel(16);
let mut tasks = JoinSet::new();
tasks.spawn(lookup_loop(
reads.clone(),
DiscoveryKey::from_bytes([0; 32]),
DiscoveryPolicy::default(),
rx,
candidates,
));
tokio::task::yield_now().await;
tx.send(Connectivity::Isolated(Instant::now())).unwrap();
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(59)).await;
tokio::task::yield_now().await;
assert_eq!(reads.0.load(Ordering::SeqCst), 0);
tx.send(Connectivity::Connected).unwrap();
tx.send(Connectivity::Isolated(Instant::now())).unwrap();
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(59)).await;
tokio::task::yield_now().await;
assert_eq!(reads.0.load(Ordering::SeqCst), 0);
tokio::time::advance(Duration::from_secs(1)).await;
tokio::task::yield_now().await;
assert_eq!(reads.0.load(Ordering::SeqCst), 1);
tx.send(Connectivity::Connected).unwrap();
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(600)).await;
tokio::task::yield_now().await;
assert_eq!(reads.0.load(Ordering::SeqCst), 1);
}
}
+84
View File
@@ -0,0 +1,84 @@
//! A DNS view of the overlay.
//!
//! The names and addresses of a network, served to the host that runs the
//! agent, so members can be reached by name. Answered from signed state, so a
//! member that is switched off still resolves.
//!
//! Three parts, kept apart on purpose:
//!
//! * [`zone`] decides what the answer is. Pure, and knows nothing about
//! packets or sockets.
//! * [`server`] puts that on the wire.
//! * `publish` tells the operating system where to send its questions,
//! which is the only part that differs between platforms.
pub mod publish;
pub mod server;
pub mod zone;
pub use publish::{DnsPublisher, PublishError, Published};
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
/// Where the server listens, one list per address family.
///
/// Loopback, and only loopback. The zones are a view *for the host running
/// the agent*: it is that host's resolver that is pointed at them. Binding
/// an overlay address instead would put them in front of the whole mesh —
/// and an agent in several networks would then answer one network's
/// questions about another's names, which is precisely what separate
/// networks are for.
///
/// Both families, and independently: a question arrives over whichever one
/// the resolver happens to use, and on a host with one of them switched off
/// the other must still answer. So neither list failing says anything about
/// the other, and a listener is opened from each.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ListenPlan {
/// IPv4 candidates, best first.
pub v4: Vec<SocketAddr>,
/// IPv6 candidates, best first.
pub v6: Vec<SocketAddr>,
}
impl ListenPlan {
/// The two lists, so a caller can bind one listener from each.
pub fn families(&self) -> [&[SocketAddr]; 2] {
[&self.v4, &self.v6]
}
}
/// Builds the listen plan for one port.
pub fn listen_plan(port: u16) -> ListenPlan {
ListenPlan {
v4: vec![SocketAddr::from((Ipv4Addr::LOCALHOST, port))],
v6: vec![SocketAddr::from((Ipv6Addr::LOCALHOST, port))],
}
}
pub use server::{DnsServer, SharedZone};
pub use zone::{Answer, Query, Zone, ZoneError, ZoneName, Zones};
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn both_families_are_planned_for_and_neither_is_reachable_off_this_host() {
let plan = listen_plan(5354);
assert_eq!(
plan.v4,
vec!["127.0.0.1:5354".parse::<SocketAddr>().unwrap()]
);
assert_eq!(plan.v6, vec!["[::1]:5354".parse::<SocketAddr>().unwrap()]);
assert_eq!(plan.families().len(), 2);
// Nothing here is reachable from another member: the zones are a
// view for this host, and an agent in two networks must not answer
// one network's questions about the other's names.
for address in plan.families().concat() {
assert!(address.ip().is_loopback(), "{address} is not loopback");
}
}
}
@@ -0,0 +1,333 @@
//! Telling the macOS resolver to send some questions here, through
//! `/etc/resolver`.
//!
//! macOS reads one file per domain out of `/etc/resolver`: a file named after
//! a domain, containing the `nameserver` and `port` to send that domain's
//! questions to, makes the system resolver a *scoped* resolver for exactly
//! that suffix and nothing else (see `resolver(5)`). That is the same contract
//! the systemd-resolved and NRPT publishers arrange on the other platforms:
//! route these suffixes to this server, and claim nothing else.
//!
//! # Why a port, and so no privileged socket
//!
//! A resolver file carries a `port`, so the server does not have to sit on 53
//! and the agent needs nothing like `CAP_NET_BIND_SERVICE`. It can serve on a
//! high port and still be reached for its suffixes.
//!
//! # Tagged, reversible, and never clobbering
//!
//! Every file this publisher writes begins with a marker line, so it only ever
//! replaces or removes files it wrote itself: a pre-existing `/etc/resolver`
//! entry a user set up by hand is left untouched. The files are removed when
//! the resolver is turned off and on shutdown; a crash leaves them behind, and
//! the next apply for the same domains overwrites them.
//!
//! # Privilege
//!
//! Writing under `/etc/resolver` needs root. A permission error is reported as
//! a refusal — the kind of failure a person has to act on — rather than a
//! transient one, so the agent does not retry it at the ordinary pace.
use std::collections::BTreeSet;
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use crate::BoxFuture;
use super::{DnsPublisher, PublishError, Published};
/// The directory macOS reads scoped resolvers from.
const RESOLVER_DIR: &str = "/etc/resolver";
/// The first line of every file this publisher writes, so it recognises its
/// own and never touches a file a user created.
const MARKER: &str = "# managed by tsunagi — do not edit";
/// Writes and removes `/etc/resolver` files for the overlay zones.
#[derive(Debug)]
pub struct ResolverDirPublisher {
/// The resolver directory. A field rather than a constant so the tests can
/// point it at a temporary directory and never touch the real one.
dir: PathBuf,
/// The domain files written by the last apply, so a later apply can remove
/// the ones no longer wanted and `revert` can remove them all.
written: Mutex<BTreeSet<String>>,
}
impl Default for ResolverDirPublisher {
fn default() -> Self {
Self::new()
}
}
impl ResolverDirPublisher {
/// A publisher writing to the real `/etc/resolver`.
pub fn new() -> Self {
Self::in_dir(RESOLVER_DIR)
}
/// A publisher writing to a specific directory. For the tests.
fn in_dir(dir: impl Into<PathBuf>) -> Self {
Self {
dir: dir.into(),
written: Mutex::new(BTreeSet::new()),
}
}
fn apply_inner(&self, published: &Published) -> Result<(), PublishError> {
let desired = desired_files(published)?;
// The directory may not exist yet; creating it needs root.
std::fs::create_dir_all(&self.dir).map_err(|err| classify(&self.dir, err))?;
// Remove the files from a previous apply that are no longer wanted.
let previous = lock(&self.written).clone();
for name in previous.difference(&keys(&desired)) {
remove_if_ours(&self.dir.join(name));
}
for (name, body) in &desired {
let path = self.dir.join(name);
// Only ever overwrite a file that is already ours, or a name that
// is free. A resolver a user set up by hand is left in place and
// reported, never clobbered.
if path.exists() && !is_ours(&path) {
return Err(PublishError::Failed(format!(
"{} already exists and was not created by tsunagi; \
remove it or rename the network to publish its resolver",
path.display()
)));
}
std::fs::write(&path, body).map_err(|err| classify(&path, err))?;
}
*lock(&self.written) = keys(&desired);
Ok(())
}
fn revert_inner(&self) {
let written = std::mem::take(&mut *lock(&self.written));
for name in &written {
remove_if_ours(&self.dir.join(name));
}
}
}
impl DnsPublisher for ResolverDirPublisher {
fn name(&self) -> &str {
"resolver-dir"
}
fn apply<'a>(&'a self, published: &'a Published) -> BoxFuture<'a, Result<(), PublishError>> {
// A handful of small file writes, short enough to run in place rather
// than hop to the blocking pool.
Box::pin(async move { self.apply_inner(published) })
}
fn revert(&self) -> BoxFuture<'_, Result<(), PublishError>> {
Box::pin(async move {
self.revert_inner();
Ok(())
})
}
}
/// The resolver files a publication implies: a map of file name to contents.
fn desired_files(published: &Published) -> Result<Vec<(String, String)>, PublishError> {
if published.servers.is_empty() {
return Err(PublishError::Failed(
"no server address to point the resolver at".to_string(),
));
}
let body = file_contents(published);
let mut files = Vec::new();
for domain in &published.domains {
let name = resolver_file_name(domain)?;
files.push((name, body.clone()));
}
Ok(files)
}
/// The contents of one resolver file: the marker, every server, one port.
fn file_contents(published: &Published) -> String {
let mut out = String::from(MARKER);
out.push('\n');
for server in &published.servers {
let ip = match server.ip() {
IpAddr::V4(v4) => v4.to_string(),
IpAddr::V6(v6) => v6.to_string(),
};
out.push_str("nameserver ");
out.push_str(&ip);
out.push('\n');
}
// One port for the file; every server listens on the same one. The `port`
// directive is what lets the server stay off 53.
if let Some(first) = published.servers.first() {
out.push_str("port ");
out.push_str(&first.port().to_string());
out.push('\n');
}
out
}
/// The file name for a domain, rejecting anything that is not a plain label
/// path component: a resolver file name must never escape the directory.
fn resolver_file_name(domain: &str) -> Result<String, PublishError> {
let trimmed = domain.trim_end_matches('.');
if trimmed.is_empty()
|| trimmed.contains('/')
|| trimmed.contains('\\')
|| trimmed.contains(std::path::MAIN_SEPARATOR)
|| trimmed == "."
|| trimmed == ".."
|| trimmed.starts_with('.')
{
return Err(PublishError::Failed(format!(
"cannot make a resolver file name for the domain {domain:?}"
)));
}
Ok(trimmed.to_string())
}
/// Whether a file is one this publisher wrote, by its marker first line.
fn is_ours(path: &Path) -> bool {
match std::fs::read_to_string(path) {
Ok(text) => text.lines().next() == Some(MARKER),
Err(_) => false,
}
}
/// Removes a file, but only if it is one of ours. Best effort.
fn remove_if_ours(path: &Path) {
if is_ours(path)
&& let Err(err) = std::fs::remove_file(path)
&& err.kind() != std::io::ErrorKind::NotFound
{
tracing::debug!(path = %path.display(), %err, "cannot remove the resolver file");
}
}
/// The set of file names in a desired-files list.
fn keys(files: &[(String, String)]) -> BTreeSet<String> {
files.iter().map(|(name, _)| name.clone()).collect()
}
/// Turns a filesystem error into the right kind of publish error: a permission
/// problem is a refusal a person must act on, anything else is a plain failure.
fn classify(path: &Path, err: std::io::Error) -> PublishError {
if err.kind() == std::io::ErrorKind::PermissionDenied {
PublishError::Refused(format!(
"writing {} needs root; start the agent with `sudo` or as a root LaunchDaemon",
path.display()
))
} else {
PublishError::Failed(format!("cannot write {}: {err}", path.display()))
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
match mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::net::SocketAddr;
use super::*;
fn published(domains: &[&str]) -> Published {
Published {
interface: "utun5".into(),
servers: vec![
"10.13.37.1:5353".parse::<SocketAddr>().unwrap(),
"[fd00::1]:5353".parse::<SocketAddr>().unwrap(),
],
domains: domains.iter().map(|d| d.to_string()).collect(),
}
}
#[test]
fn a_file_lists_every_server_and_one_port_under_the_marker() {
let body = file_contents(&published(&["mynet."]));
let mut lines = body.lines();
assert_eq!(lines.next(), Some(MARKER));
assert_eq!(lines.next(), Some("nameserver 10.13.37.1"));
assert_eq!(lines.next(), Some("nameserver fd00::1"));
assert_eq!(lines.next(), Some("port 5353"));
assert_eq!(lines.next(), None);
}
#[test]
fn a_domain_becomes_a_plain_file_name_without_its_trailing_dot() {
assert_eq!(resolver_file_name("mynet.").unwrap(), "mynet");
assert_eq!(resolver_file_name("my.net").unwrap(), "my.net");
}
#[test]
fn a_domain_that_would_escape_the_directory_is_refused() {
assert!(resolver_file_name("../etc/passwd").is_err());
assert!(resolver_file_name("a/b").is_err());
assert!(resolver_file_name("").is_err());
assert!(resolver_file_name(".").is_err());
assert!(resolver_file_name(".hidden").is_err());
}
#[tokio::test]
async fn apply_writes_a_file_per_domain_and_revert_removes_them() {
let dir = tempfile::tempdir().unwrap();
let publisher = ResolverDirPublisher::in_dir(dir.path());
publisher
.apply(&published(&["mynet.", "other."]))
.await
.unwrap();
assert!(dir.path().join("mynet").is_file());
assert!(dir.path().join("other").is_file());
publisher.revert().await.unwrap();
assert!(!dir.path().join("mynet").exists());
assert!(!dir.path().join("other").exists());
}
#[tokio::test]
async fn re_applying_removes_a_domain_that_is_no_longer_wanted() {
let dir = tempfile::tempdir().unwrap();
let publisher = ResolverDirPublisher::in_dir(dir.path());
publisher.apply(&published(&["a.", "b."])).await.unwrap();
publisher.apply(&published(&["a."])).await.unwrap();
assert!(dir.path().join("a").is_file());
assert!(
!dir.path().join("b").exists(),
"the dropped domain's file is removed"
);
}
#[tokio::test]
async fn a_file_a_user_created_is_never_clobbered_or_removed() {
let dir = tempfile::tempdir().unwrap();
let foreign = dir.path().join("mynet");
std::fs::write(&foreign, "nameserver 9.9.9.9\n").unwrap();
let publisher = ResolverDirPublisher::in_dir(dir.path());
let err = publisher.apply(&published(&["mynet."])).await.unwrap_err();
assert!(matches!(err, PublishError::Failed(_)), "{err:?}");
// Untouched.
assert_eq!(
std::fs::read_to_string(&foreign).unwrap(),
"nameserver 9.9.9.9\n"
);
// And revert leaves a foreign file alone even if it is in our set.
publisher.revert().await.unwrap();
assert!(foreign.is_file());
}
}
+105
View File
@@ -0,0 +1,105 @@
//! A pretend resolver, so the wiring is tested without touching this host's.
use std::sync::{Arc, Mutex};
use crate::BoxFuture;
use super::{DnsPublisher, PublishError, Published};
/// Records what it was asked to do, and can be told to refuse.
#[derive(Debug, Clone, Default)]
pub struct MockPublisher {
applied: Arc<Mutex<Option<Published>>>,
failure: Option<PublishError>,
}
impl MockPublisher {
/// A publisher that accepts everything.
pub fn new() -> Self {
Self::default()
}
/// A publisher that refuses everything, with this reason.
pub fn failing(error: PublishError) -> Self {
Self {
applied: Arc::new(Mutex::new(None)),
failure: Some(error),
}
}
/// What is currently applied, if anything.
pub fn applied(&self) -> Option<Published> {
match self.applied.lock() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
}
}
}
impl DnsPublisher for MockPublisher {
fn name(&self) -> &str {
"mock"
}
fn apply<'a>(&'a self, published: &'a Published) -> BoxFuture<'a, Result<(), PublishError>> {
Box::pin(async move {
if let Some(failure) = &self.failure {
return Err(failure.clone());
}
match self.applied.lock() {
Ok(mut guard) => *guard = Some(published.clone()),
Err(poisoned) => *poisoned.into_inner() = Some(published.clone()),
}
Ok(())
})
}
fn revert(&self) -> BoxFuture<'_, Result<(), PublishError>> {
Box::pin(async move {
match self.applied.lock() {
Ok(mut guard) => *guard = None,
Err(poisoned) => *poisoned.into_inner() = None,
}
Ok(())
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn published() -> Published {
Published {
interface: "tsundemo".into(),
servers: vec![
"10.13.37.69:5354".parse().unwrap(),
"[::1]:5354".parse().unwrap(),
],
domains: vec!["lab".into()],
}
}
#[tokio::test]
async fn applying_and_reverting_are_both_recorded() {
let publisher = MockPublisher::new();
assert!(publisher.applied().is_none());
publisher.apply(&published()).await.unwrap();
assert_eq!(publisher.applied(), Some(published()));
publisher.revert().await.unwrap();
assert!(publisher.applied().is_none());
// Reverting twice is not an error: shutdown must not fail here.
publisher.revert().await.unwrap();
}
#[tokio::test]
async fn a_refusal_leaves_nothing_applied() {
let publisher = MockPublisher::failing(PublishError::Refused("polkit said no".into()));
assert!(publisher.apply(&published()).await.is_err());
assert!(publisher.applied().is_none());
}
}
+215
View File
@@ -0,0 +1,215 @@
//! Telling the operating system where to send its questions.
//!
//! The server answers whether or not this works. That is the whole reason it
//! is a separate thing: if the resolver cannot be configured — no
//! systemd-resolved, an unwilling polkit, a platform nobody has written this
//! for — the port is still up and the user can point something at it by
//! hand. A failure here is a degraded overlay, not a broken one.
//!
//! Only the *mechanics* differ between systems. What has to be arranged is
//! the same everywhere: send questions for these suffixes to this address,
//! through this interface, and do not make it the resolver for anything
//! else. That is [`Published`]; the rest is behind [`DnsPublisher`].
use std::net::SocketAddr;
use crate::BoxFuture;
mod mock;
pub use mock::MockPublisher;
#[cfg(all(feature = "dns-publish", target_os = "linux"))]
mod resolved;
#[cfg(all(feature = "dns-publish", target_os = "linux"))]
pub use resolved::ResolvedPublisher;
#[cfg(all(feature = "dns-publish", target_os = "windows"))]
mod windows;
#[cfg(all(feature = "dns-publish", target_os = "windows"))]
pub use windows::NrptPublisher;
#[cfg(all(feature = "dns-publish", target_os = "macos"))]
mod macos;
#[cfg(all(feature = "dns-publish", target_os = "macos"))]
pub use macos::ResolverDirPublisher;
mod unsupported;
pub use unsupported::UnsupportedPublisher;
/// What the operating system is asked to do.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Published {
/// The interface questions should be sent through.
///
/// The server listens on an overlay address, which is only reachable
/// over the overlay interface, so the two travel together. It must be an
/// interface this agent created: configuring one it did not is
/// configuring somebody else's.
pub interface: String,
/// Every address the server is listening on, best first.
///
/// One per address family, so a resolver reaches it over whichever it
/// uses. All of them are published together and none is preferred by
/// this: which one a resolver picks is its business.
pub servers: Vec<SocketAddr>,
/// The suffixes that belong to this server.
///
/// Routing suffixes only: they say *which questions* come here, never
/// that this is the resolver for anything else.
pub domains: Vec<String>,
}
/// Why the resolver could not be told.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum PublishError {
/// There is nothing here that can be configured this way.
#[error("{0}")]
Unavailable(String),
/// Something is there, and it declined.
///
/// Separate from a plain failure because the answer is different: this
/// one is about who the agent is running as, not about whether the thing
/// works.
#[error("{0}")]
Refused(String),
/// It was there, it accepted the request, and it went wrong anyway.
#[error("{0}")]
Failed(String),
}
impl PublishError {
/// Whether waiting will fix it.
///
/// A refusal will not change on its own — somebody has to grant
/// permission — so retrying it at the pace of everything else is just
/// noise. Anything else might be a service still starting.
pub fn needs_a_human(&self) -> bool {
matches!(self, PublishError::Refused(_))
}
/// One line for a status table.
pub fn remedy(&self) -> Option<&'static str> {
match self {
PublishError::Refused(_) => Some(
"grant this user the `org.freedesktop.resolve1.set-*` actions in \
/etc/polkit-1/rules.d, or run the agent as a system service",
),
_ => None,
}
}
}
/// The polkit rule that lets this user configure the resolver.
///
/// Printed in full rather than described, because the point of this feature
/// is that the user has as little to do as possible, and "write a polkit
/// rule" is a great deal more work than pasting one.
///
/// It grants exactly the four actions this agent calls and nothing else.
/// polkit decides by user id — a capability does not help here — so there is
/// no way to do this from inside the process.
pub fn polkit_recipe(user: &str) -> String {
format!(
"sudo tee /etc/polkit-1/rules.d/50-tsunagi-resolved.rules > /dev/null <<'RULE'\n\
polkit.addRule(function(action, subject) {{\n\
\x20 var allowed = [\n\
\x20 \"org.freedesktop.resolve1.set-dns-servers\",\n\
\x20 \"org.freedesktop.resolve1.set-domains\",\n\
\x20 \"org.freedesktop.resolve1.set-default-route\",\n\
\x20 \"org.freedesktop.resolve1.revert\"\n\
\x20 ];\n\
\x20 if (allowed.indexOf(action.id) >= 0 && subject.user == \"{user}\") {{\n\
\x20 return polkit.Result.YES;\n\
\x20 }}\n\
}});\n\
RULE"
)
}
/// Arranges for the operating system to ask this server.
pub trait DnsPublisher: Send + Sync + std::fmt::Debug + 'static {
/// A short name used in diagnostics.
fn name(&self) -> &str;
/// Applies the setting, replacing whatever this publisher set before.
fn apply<'a>(&'a self, published: &'a Published) -> BoxFuture<'a, Result<(), PublishError>>;
/// Undoes it.
///
/// Reverting something that was never applied succeeds: this runs on the
/// shutdown path, where the setting being gone is the point.
fn revert(&self) -> BoxFuture<'_, Result<(), PublishError>>;
}
/// The kernel's index for an interface.
///
/// From sysfs, which needs no privileges and no netlink round trip.
#[cfg(target_os = "linux")]
pub fn interface_index(name: &str) -> Option<u32> {
std::fs::read_to_string(format!("/sys/class/net/{name}/ifindex"))
.ok()?
.trim()
.parse()
.ok()
}
/// The kernel's index for an interface.
///
/// Only the systemd-resolved publisher needs one, so off Linux there is
/// nothing to look up and this is always `None`.
#[cfg(not(target_os = "linux"))]
pub fn interface_index(_name: &str) -> Option<u32> {
None
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn only_a_refusal_suggests_what_to_do_about_it() {
// The other two are conditions of the host, not of the user.
assert!(PublishError::Refused("no".into()).remedy().is_some());
assert!(PublishError::Unavailable("none".into()).remedy().is_none());
assert!(PublishError::Failed("bang".into()).remedy().is_none());
}
#[test]
fn only_a_refusal_waits_for_a_person() {
// The rest may come right on their own, so they are worth retrying
// at the ordinary pace; a refusal is not.
assert!(PublishError::Refused("no".into()).needs_a_human());
assert!(!PublishError::Unavailable("none".into()).needs_a_human());
assert!(!PublishError::Failed("bang".into()).needs_a_human());
}
#[test]
fn the_polkit_recipe_grants_what_is_called_and_no_more() {
let recipe = polkit_recipe("ab");
for action in [
"set-dns-servers",
"set-domains",
"set-default-route",
"revert",
] {
assert!(recipe.contains(action), "{action} missing from:\n{recipe}");
}
// Nothing beyond what the agent calls: a rule that granted the lot
// would be handing out more than this feature needs.
for other in ["set-dnssec", "set-mdns", "register-service", "set-llmnr"] {
assert!(!recipe.contains(other), "{other} should not be granted");
}
assert!(recipe.contains("subject.user == \"ab\""));
// ES5: the rules engine is duktape and has no `startsWith`.
assert!(!recipe.contains("startsWith"));
}
#[cfg(target_os = "linux")]
#[test]
fn an_interface_index_is_read_from_the_running_kernel() {
assert_eq!(interface_index("lo"), Some(1));
assert_eq!(interface_index("tsunagi-no-such-interface"), None);
}
}
@@ -0,0 +1,275 @@
//! Telling systemd-resolved to send some questions here.
//!
//! Three calls on `org.freedesktop.resolve1.Manager`, all scoped to the
//! overlay interface:
//!
//! * `SetLinkDNSEx` — where to send them. The `Ex` form carries a **port**,
//! which is why this server does not have to sit on 53 and the agent needs
//! no `CAP_NET_BIND_SERVICE`. Systems before systemd 247 have only
//! `SetLinkDNS`, which has no port; there the fallback only works if the
//! server did get port 53, and it says so rather than appearing to work.
//! * `SetLinkDomains` with `routing_only` set — a *routing* suffix, the
//! `~domain` form. It says which questions come here and claims nothing
//! else.
//! * `SetLinkDefaultRoute(false)` — so this never becomes the resolver for
//! anything outside those suffixes. Without it resolved may fall back to
//! this link for ordinary names, and this server refuses those.
//!
//! # It cleans up by itself
//!
//! resolved keys all of this to the interface, and drops it when the
//! interface goes. The overlay interface belongs to a file descriptor the
//! agent holds, so it goes when the agent does — however the agent goes. The
//! explicit `RevertLink` on shutdown only makes that immediate.
//!
//! # Privilege
//!
//! resolved asks polkit, and polkit decides by user id, not by capability.
//! So `CAP_NET_ADMIN` does not help here: an ordinary user is prompted or
//! refused, while a system service running as root is not. That refusal is
//! reported as its own kind of error, because the answer to it is different
//! from the answer to "resolved is not installed".
use std::net::IpAddr;
use crate::BoxFuture;
use super::{DnsPublisher, PublishError, Published, interface_index};
/// `AF_INET`, as resolved wants it.
const AF_INET: i32 = 2;
/// `AF_INET6`.
const AF_INET6: i32 = 10;
#[zbus::proxy(
interface = "org.freedesktop.resolve1.Manager",
default_service = "org.freedesktop.resolve1",
default_path = "/org/freedesktop/resolve1"
)]
trait Resolved {
/// Servers for a link, with a port and a name. systemd 247 and later.
#[zbus(name = "SetLinkDNSEx")]
fn set_link_dns_ex(
&self,
ifindex: i32,
addresses: &[(i32, Vec<u8>, u16, String)],
) -> zbus::Result<()>;
/// Servers for a link, without a port. Always port 53.
#[zbus(name = "SetLinkDNS")]
fn set_link_dns(&self, ifindex: i32, addresses: &[(i32, Vec<u8>)]) -> zbus::Result<()>;
/// Suffixes for a link. The flag makes one routing-only.
#[zbus(name = "SetLinkDomains")]
fn set_link_domains(&self, ifindex: i32, domains: &[(String, bool)]) -> zbus::Result<()>;
/// Whether this link may answer for names outside its suffixes.
#[zbus(name = "SetLinkDefaultRoute")]
fn set_link_default_route(&self, ifindex: i32, enable: bool) -> zbus::Result<()>;
/// Forgets everything set for a link.
#[zbus(name = "RevertLink")]
fn revert_link(&self, ifindex: i32) -> zbus::Result<()>;
}
/// Configures systemd-resolved over D-Bus.
#[derive(Debug, Default)]
pub struct ResolvedPublisher {
/// The link last configured, so shutdown knows what to undo.
applied: std::sync::Mutex<Option<u32>>,
}
impl ResolvedPublisher {
/// Creates the publisher. Nothing is contacted until [`Self::apply`].
pub fn new() -> Self {
Self::default()
}
async fn proxy() -> Result<ResolvedProxy<'static>, PublishError> {
let connection = zbus::Connection::system().await.map_err(|err| {
PublishError::Unavailable(format!("no system D-Bus to talk to resolved on: {err}"))
})?;
ResolvedProxy::new(&connection).await.map_err(|err| {
PublishError::Unavailable(format!("systemd-resolved is not answering: {err}"))
})
}
}
/// Turns a D-Bus failure into the kind of failure it actually is.
fn classify(err: zbus::Error, what: &str) -> PublishError {
let name = match &err {
zbus::Error::MethodError(name, _, _) => name.as_str().to_string(),
_ => String::new(),
};
match name.as_str() {
"org.freedesktop.DBus.Error.InteractiveAuthorizationRequired"
| "org.freedesktop.DBus.Error.AccessDenied" => {
PublishError::Refused(format!("systemd-resolved refused {what}: {err}"))
}
"org.freedesktop.DBus.Error.UnknownMethod"
| "org.freedesktop.DBus.Error.ServiceUnknown" => {
PublishError::Unavailable(format!("systemd-resolved cannot do {what}: {err}"))
}
_ => PublishError::Failed(format!("systemd-resolved failed {what}: {err}")),
}
}
/// The address in the shape resolved wants: a family and raw octets.
fn wire_address(address: IpAddr) -> (i32, Vec<u8>) {
match address {
IpAddr::V4(address) => (AF_INET, address.octets().to_vec()),
IpAddr::V6(address) => (AF_INET6, address.octets().to_vec()),
}
}
impl DnsPublisher for ResolvedPublisher {
fn name(&self) -> &str {
"systemd-resolved"
}
fn apply<'a>(&'a self, published: &'a Published) -> BoxFuture<'a, Result<(), PublishError>> {
Box::pin(async move {
let ifindex = interface_index(&published.interface).ok_or_else(|| {
PublishError::Unavailable(format!(
"interface `{}` is not on this host, so there is no link to configure",
published.interface
))
})?;
if published.servers.is_empty() {
return Err(PublishError::Unavailable(
"there is no listening address to send questions to".to_string(),
));
}
let proxy = Self::proxy().await?;
let index = ifindex as i32;
// Every family in one call, because this replaces the link's
// whole list: sending them one at a time would leave only the
// last. A resolver then picks whichever it can reach.
let servers: Vec<(i32, Vec<u8>, u16, String)> = published
.servers
.iter()
.map(|server| {
let (family, octets) = wire_address(server.ip());
(family, octets, server.port(), String::new())
})
.collect();
match proxy.set_link_dns_ex(index, &servers).await {
Ok(()) => {}
Err(err) => {
let classified = classify(err, "setting the link's DNS servers");
// Older systemd has no `Ex` form, and the plain one is
// always port 53. Falling back to it when the server is
// somewhere else would point resolved at nothing.
if !matches!(classified, PublishError::Unavailable(_)) {
return Err(classified);
}
if let Some(elsewhere) =
published.servers.iter().find(|server| server.port() != 53)
{
return Err(PublishError::Unavailable(format!(
"this systemd-resolved cannot be given a port, and the server is on \
{}. Run the server on port 53, or point your resolver at \
{elsewhere} yourself.",
elsewhere.port()
)));
}
let plain: Vec<(i32, Vec<u8>)> = servers
.into_iter()
.map(|(family, octets, _, _)| (family, octets))
.collect();
proxy
.set_link_dns(index, &plain)
.await
.map_err(|err| classify(err, "setting the link's DNS servers"))?;
}
}
let domains: Vec<(String, bool)> = published
.domains
.iter()
.map(|domain| (domain.clone(), true))
.collect();
proxy
.set_link_domains(index, &domains)
.await
.map_err(|err| classify(err, "setting the link's search domains"))?;
// Last, and deliberately: until this is off, resolved may send
// ordinary names here, and this server refuses them.
proxy
.set_link_default_route(index, false)
.await
.map_err(|err| classify(err, "clearing the link's default route"))?;
match self.applied.lock() {
Ok(mut guard) => *guard = Some(ifindex),
Err(poisoned) => *poisoned.into_inner() = Some(ifindex),
}
Ok(())
})
}
fn revert(&self) -> BoxFuture<'_, Result<(), PublishError>> {
Box::pin(async move {
let applied = match self.applied.lock() {
Ok(mut guard) => guard.take(),
Err(poisoned) => poisoned.into_inner().take(),
};
let Some(ifindex) = applied else {
return Ok(());
};
let proxy = Self::proxy().await?;
match proxy.revert_link(ifindex as i32).await {
Ok(()) => Ok(()),
// The link going away takes the setting with it, so this is
// the outcome asked for rather than a failure.
Err(zbus::Error::MethodError(name, _, _))
if name.as_str().contains("NoSuchLink") =>
{
Ok(())
}
Err(err) => Err(classify(err, "reverting the link")),
}
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::net::{Ipv4Addr, SocketAddr};
#[test]
fn an_address_is_encoded_the_way_resolved_expects() {
assert_eq!(
wire_address(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 69))),
(AF_INET, vec![10, 13, 37, 69])
);
let (family, octets) = wire_address("fd55::1".parse().unwrap());
assert_eq!(family, AF_INET6);
assert_eq!(octets.len(), 16);
}
#[tokio::test]
async fn an_interface_that_is_not_here_is_unavailable_not_a_failure() {
// Nothing is contacted: there is no link to configure, and saying so
// is the honest answer without bothering the bus.
let publisher = ResolvedPublisher::new();
let published = Published {
interface: "tsunagi-no-such-interface".into(),
servers: vec![SocketAddr::from(([10, 13, 37, 69], 5354))],
domains: vec!["lab".into()],
};
let err = publisher.apply(&published).await.unwrap_err();
assert!(matches!(err, PublishError::Unavailable(_)), "{err}");
}
#[tokio::test]
async fn reverting_without_having_applied_does_nothing_and_succeeds() {
// The shutdown path must not fail because there was nothing to undo.
ResolvedPublisher::new().revert().await.unwrap();
}
}
@@ -0,0 +1,73 @@
//! The publisher for systems nobody has written one for.
//!
//! macOS needs `SystemConfiguration` and Windows the IP Helper API, and
//! neither is written. Saying so is more use than appearing to work: the
//! server is already answering, so all that is missing is the last step, and
//! the user can take it by hand once they know that is what is needed.
use crate::BoxFuture;
use super::{DnsPublisher, PublishError, Published};
/// Refuses to configure anything, with an explanation.
#[derive(Debug, Clone)]
pub struct UnsupportedPublisher {
platform: &'static str,
}
impl Default for UnsupportedPublisher {
fn default() -> Self {
Self::new()
}
}
impl UnsupportedPublisher {
/// A publisher naming the platform it stands in for.
pub fn new() -> Self {
Self {
platform: std::env::consts::OS,
}
}
}
impl DnsPublisher for UnsupportedPublisher {
fn name(&self) -> &str {
"unsupported"
}
fn apply<'a>(&'a self, _published: &'a Published) -> BoxFuture<'a, Result<(), PublishError>> {
Box::pin(async move {
Err(PublishError::Unavailable(format!(
"configuring the system resolver is not implemented on {} yet",
self.platform
)))
})
}
fn revert(&self) -> BoxFuture<'_, Result<(), PublishError>> {
// Nothing was set, so there is nothing to undo and no reason to fail
// a shutdown over it.
Box::pin(async move { Ok(()) })
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[tokio::test]
async fn it_names_the_platform_and_still_reverts_quietly() {
let publisher = UnsupportedPublisher::new();
let published = Published {
interface: "tsundemo".into(),
servers: vec!["10.0.0.1:5354".parse().unwrap()],
domains: vec!["lab".into()],
};
let err = publisher.apply(&published).await.unwrap_err();
assert!(matches!(err, PublishError::Unavailable(_)));
assert!(err.to_string().contains(std::env::consts::OS));
publisher.revert().await.unwrap();
}
}
@@ -0,0 +1,301 @@
//! Telling the Windows DNS client to send some questions here, through the
//! Name Resolution Policy Table.
//!
//! The NRPT is how Windows does split DNS: a rule says "names under this
//! suffix are resolved by these servers", and names outside every rule are
//! resolved the ordinary way. That is exactly the contract of [`Published`] —
//! route these suffixes here and claim nothing else — so one NRPT rule per
//! suffix is the whole of it. The rules are keyed by namespace, not by
//! interface, so the interface name in [`Published`] is not needed here.
//!
//! # Why PowerShell
//!
//! The NRPT lives in the registry, but a rule only takes effect once the DNS
//! client is told to reload its policy, and that notification is an RPC with
//! no safe wrapper this crate may call — it forbids `unsafe`. The
//! `DnsClient` PowerShell module does the write *and* the reload, so it is
//! used by absolute path from `%SystemRoot%`. Every value passed to it is one
//! this agent produced: the suffixes are validated zone names (letters,
//! digits, `-`, `_`, `.` only) and the servers are addresses it is listening
//! on. They are still single-quoted and escaped when built into the script,
//! so nothing could be read as PowerShell rather than as data.
//!
//! # The port
//!
//! The Windows DNS client always asks on port 53; it cannot be told another.
//! So a server on a different port cannot be reached this way, and rather than
//! configure a rule that points at nothing, this says so — the same honest
//! failure the systemd-resolved path gives for an old resolver with no port.
//!
//! # Privilege
//!
//! Writing an NRPT rule needs an elevated process. An ordinary user is
//! refused, which is reported as its own kind of error because the answer to
//! it — run elevated, or as a service — differs from "this is not Windows".
use std::collections::BTreeSet;
use std::net::IpAddr;
use std::sync::Mutex;
use crate::BoxFuture;
use super::{DnsPublisher, PublishError, Published};
/// The port the Windows DNS client is fixed to.
const DNS_PORT: u16 = 53;
/// Configures the Name Resolution Policy Table through PowerShell.
#[derive(Debug, Default)]
pub struct NrptPublisher {
/// The namespaces last configured, so shutdown knows what to undo.
applied: Mutex<Vec<String>>,
}
impl NrptPublisher {
/// Creates the publisher. Nothing is contacted until [`Self::apply`].
pub fn new() -> Self {
Self::default()
}
}
/// A suffix as the NRPT wants it: a leading dot means "this and everything
/// under it".
fn namespace(domain: &str) -> String {
let trimmed = domain.trim_matches('.');
format!(".{trimmed}")
}
/// A PowerShell single-quoted literal, with any embedded quote doubled.
fn ps_literal(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
/// A PowerShell array literal, `@('a','b')`, or `@()` when empty.
fn ps_array(values: &[String]) -> String {
let items: Vec<String> = values.iter().map(|value| ps_literal(value)).collect();
format!("@({})", items.join(","))
}
/// The script that clears our rules for these namespaces and then adds them.
fn apply_script(namespaces: &[String], servers: &[String]) -> String {
format!(
"$ErrorActionPreference = 'Stop'\n\
$ns = {ns}\n\
$srv = {srv}\n\
foreach ($n in $ns) {{ Get-DnsClientNrptRule | Where-Object {{ $_.Namespace -eq $n }} | \
ForEach-Object {{ Remove-DnsClientNrptRule -Name $_.Name -Force }} }}\n\
foreach ($n in $ns) {{ Add-DnsClientNrptRule -Namespace $n -NameServers $srv }}\n",
ns = ps_array(namespaces),
srv = ps_array(servers),
)
}
/// The script that clears our rules for these namespaces.
fn revert_script(namespaces: &[String]) -> String {
format!(
"$ErrorActionPreference = 'Stop'\n\
$ns = {ns}\n\
foreach ($n in $ns) {{ Get-DnsClientNrptRule | Where-Object {{ $_.Namespace -eq $n }} | \
ForEach-Object {{ Remove-DnsClientNrptRule -Name $_.Name -Force }} }}\n",
ns = ps_array(namespaces),
)
}
/// Turns what PowerShell said on failure into the kind of failure it is.
fn classify(text: &str) -> PublishError {
let lower = text.to_ascii_lowercase();
if lower.contains("access is denied")
|| lower.contains("requires elevation")
|| lower.contains("run as administrator")
|| lower.contains("administrator privilege")
|| lower.contains("permissiondenied")
{
PublishError::Refused(format!("the Windows DNS client refused the change: {text}"))
} else if lower.contains("is not recognized")
|| lower.contains("commandnotfoundexception")
|| lower.contains("not recognized as the name of a cmdlet")
{
PublishError::Unavailable(format!(
"the DnsClient PowerShell module is not available: {text}"
))
} else {
PublishError::Failed(format!("the Windows DNS client failed the change: {text}"))
}
}
/// Runs a PowerShell script from `%SystemRoot%` and maps a failure to a
/// [`PublishError`].
async fn powershell(script: String) -> Result<(), PublishError> {
let program = powershell_path();
let output = tokio::task::spawn_blocking(move || {
std::process::Command::new(program)
.args([
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-Command",
&script,
])
.output()
})
.await
.map_err(|err| PublishError::Failed(format!("could not run PowerShell: {err}")))?
.map_err(|err| PublishError::Unavailable(format!("could not run PowerShell: {err}")))?;
if output.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&output.stderr);
let text = if stderr.trim().is_empty() {
String::from_utf8_lossy(&output.stdout).trim().to_string()
} else {
stderr.trim().to_string()
};
Err(classify(if text.is_empty() { "no output" } else { &text }))
}
/// The absolute path to Windows PowerShell, so nothing on `PATH` can stand in
/// for it.
fn powershell_path() -> std::path::PathBuf {
let root = std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into());
std::path::Path::new(&root)
.join("System32")
.join("WindowsPowerShell")
.join("v1.0")
.join("powershell.exe")
}
impl DnsPublisher for NrptPublisher {
fn name(&self) -> &str {
"windows-nrpt"
}
fn apply<'a>(&'a self, published: &'a Published) -> BoxFuture<'a, Result<(), PublishError>> {
Box::pin(async move {
if published.domains.is_empty() {
return Err(PublishError::Unavailable(
"there are no suffixes to route, so there is no rule to write".to_string(),
));
}
// Windows can only ask on port 53, so a server anywhere else is
// unreachable this way. Keep the ones it can use.
let servers: BTreeSet<IpAddr> = published
.servers
.iter()
.filter(|server| server.port() == DNS_PORT)
.map(|server| server.ip())
.collect();
if servers.is_empty() {
let elsewhere = published
.servers
.iter()
.map(|server| server.to_string())
.collect::<Vec<_>>()
.join(", ");
return Err(PublishError::Unavailable(format!(
"the Windows DNS client only asks on port {DNS_PORT}, and the server is on \
{elsewhere}. Run the server on port {DNS_PORT}, or point your resolver at it \
yourself."
)));
}
let namespaces: Vec<String> = published.domains.iter().map(|d| namespace(d)).collect();
let servers: Vec<String> = servers.iter().map(|ip| ip.to_string()).collect();
powershell(apply_script(&namespaces, &servers)).await?;
match self.applied.lock() {
Ok(mut guard) => *guard = namespaces,
Err(poisoned) => *poisoned.into_inner() = namespaces,
}
Ok(())
})
}
fn revert(&self) -> BoxFuture<'_, Result<(), PublishError>> {
Box::pin(async move {
let namespaces = match self.applied.lock() {
Ok(mut guard) => std::mem::take(&mut *guard),
Err(poisoned) => std::mem::take(&mut *poisoned.into_inner()),
};
if namespaces.is_empty() {
return Ok(());
}
powershell(revert_script(&namespaces)).await
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::net::SocketAddr;
#[test]
fn a_suffix_gets_a_leading_dot() {
assert_eq!(namespace("lab"), ".lab");
assert_eq!(namespace(".lab"), ".lab");
assert_eq!(namespace("a.b"), ".a.b");
}
#[test]
fn a_quote_in_a_value_cannot_escape_the_literal() {
// Zone names never contain a quote, but the escaping is what makes
// that a guarantee rather than a hope.
assert_eq!(ps_literal("a'b"), "'a''b'");
assert_eq!(ps_array(&["x".into(), "y".into()]), "@('x','y')");
assert_eq!(ps_array(&[]), "@()");
}
#[test]
fn the_apply_script_removes_before_it_adds() {
let script = apply_script(&[".lab".into()], &["10.13.37.69".into()]);
let remove = script.find("Remove-DnsClientNrptRule").unwrap();
let add = script.find("Add-DnsClientNrptRule").unwrap();
assert!(
remove < add,
"a stale rule must go before the new one:\n{script}"
);
assert!(script.contains("@('.lab')"));
assert!(script.contains("@('10.13.37.69')"));
}
#[test]
fn an_access_denied_is_a_refusal_a_missing_cmdlet_is_unavailable() {
assert!(matches!(
classify("Access is denied"),
PublishError::Refused(_)
));
assert!(matches!(
classify("The term 'Add-DnsClientNrptRule' is not recognized"),
PublishError::Unavailable(_)
));
assert!(matches!(
classify("something else"),
PublishError::Failed(_)
));
}
#[tokio::test]
async fn a_server_only_on_a_nonstandard_port_is_unavailable_not_a_failure() {
// Nothing is run: Windows cannot ask there, and saying so is the
// honest answer without touching the DNS client.
let publisher = NrptPublisher::new();
let published = Published {
interface: "tsun0".into(),
servers: vec![SocketAddr::from(([10, 13, 37, 69], 5354))],
domains: vec!["lab".into()],
};
let err = publisher.apply(&published).await.unwrap_err();
assert!(matches!(err, PublishError::Unavailable(_)), "{err}");
}
#[tokio::test]
async fn reverting_without_having_applied_does_nothing_and_succeeds() {
NrptPublisher::new().revert().await.unwrap();
}
}
+625
View File
@@ -0,0 +1,625 @@
//! Answering DNS questions about the overlay, over UDP and TCP.
//!
//! Authoritative for one zone and nothing else. There is no recursion, no
//! forwarding and no cache: a question this agent cannot answer from the
//! signed roster is refused rather than passed anywhere, so pointing a
//! resolver at this server can never make it a path to the outside.
//!
//! The decision of what to answer lives in [`super::zone`]; this module is
//! only the wire format and the sockets. [`respond`] sits between them and
//! takes bytes to bytes, so everything the server does to a packet is
//! testable without opening a socket.
use std::net::SocketAddr;
use std::sync::{Arc, RwLock};
use simple_dns::rdata::{A, PTR, RData, SOA};
use simple_dns::{Name, PacketFlag, QCLASS, QTYPE, RCODE, ResourceRecord, TYPE};
use super::zone::{Answer, Query, Zone, Zones};
/// How long an answer may be cached.
///
/// Short, because the roster changes when members come and go and a stale
/// answer is worse than another question.
pub const TTL: u32 = 30;
/// The largest question this server will read.
///
/// A DNS message is 512 bytes without EDNS and 4096 with it; anything past
/// that is not a question worth answering.
pub const MAX_MESSAGE_LEN: usize = 4096;
/// The largest answer sent over UDP without the client offering EDNS.
const CLASSIC_UDP_LIMIT: usize = 512;
/// How long a TCP client may take over one question.
const TCP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// How many TCP questions may be in flight at once.
const MAX_TCP_CONNECTIONS: usize = 32;
/// The zones the server answers from, swapped as the rosters change.
///
/// Shared rather than copied into the server so that a roster change — or a
/// network joining or leaving — is one write, not a restart: rebinding the
/// socket would drop questions in flight for no reason.
#[derive(Debug, Clone, Default)]
pub struct SharedZone(Arc<RwLock<Arc<Zones>>>);
impl SharedZone {
/// Wraps a set of zones.
pub fn new(zones: Zones) -> Self {
Self(Arc::new(RwLock::new(Arc::new(zones))))
}
/// Wraps a single zone, which is the common case in a test.
pub fn one(zone: Zone) -> Self {
Self::new(Zones::new([zone]))
}
/// Replaces them.
pub fn set(&self, zones: Zones) {
match self.0.write() {
Ok(mut guard) => *guard = Arc::new(zones),
Err(poisoned) => *poisoned.into_inner() = Arc::new(zones),
}
}
/// The zones as they are now.
pub fn get(&self) -> Arc<Zones> {
match self.0.read() {
Ok(guard) => Arc::clone(&guard),
Err(poisoned) => Arc::clone(&poisoned.into_inner()),
}
}
}
/// Builds the answer to one question.
///
/// `None` means say nothing at all: the message was not a question this
/// server should reply to, and replying anyway would make this a useful
/// amplifier for somebody spoofing a source address.
pub fn respond(zones: &Zones, query: &[u8]) -> Option<Vec<u8>> {
let packet = simple_dns::Packet::parse(query).ok()?;
if packet.has_flags(PacketFlag::RESPONSE) {
return None;
}
let mut reply = simple_dns::Packet::new_reply(packet.id());
// Recursion is not available here and the flag says so honestly; the
// desired bit is echoed because a resolver compares it.
if packet.has_flags(PacketFlag::RECURSION_DESIRED) {
reply.set_flags(PacketFlag::RECURSION_DESIRED);
}
if packet.opcode() != simple_dns::OPCODE::StandardQuery {
*reply.rcode_mut() = RCODE::NotImplemented;
return reply.build_bytes_vec().ok();
}
// Exactly one question. Zero is nothing to answer; more than one has no
// agreed meaning and every real server rejects it.
let [question] = packet.questions.as_slice() else {
*reply.rcode_mut() = RCODE::FormatError;
return reply.build_bytes_vec().ok();
};
if !matches!(question.qclass, QCLASS::CLASS(simple_dns::CLASS::IN)) {
*reply.rcode_mut() = RCODE::Refused;
return reply.build_bytes_vec().ok();
}
let qname = question.qname.to_string();
// Which zone answers is part of the answer: a denial is bounded by the
// SOA of the zone that denied it, and this server may hold several.
let (zone, answer) = match zones.lookup(&qname, query_kind(question.qtype)) {
Some(answered) => answered,
None => {
reply.questions.push(question.clone());
*reply.rcode_mut() = RCODE::Refused;
return reply.build_bytes_vec_compressed().ok();
}
};
reply.questions.push(question.clone());
let name = Name::new(&qname).ok()?;
match answer {
Answer::NotOurs => *reply.rcode_mut() = RCODE::Refused,
Answer::Addresses(addresses) => {
reply.set_flags(PacketFlag::AUTHORITATIVE_ANSWER);
for address in addresses {
reply.answers.push(ResourceRecord::new(
name.clone(),
simple_dns::CLASS::IN,
TTL,
RData::A(A::from(address)),
));
}
}
Answer::Name(target) => {
reply.set_flags(PacketFlag::AUTHORITATIVE_ANSWER);
let target = Name::new(&target).ok()?.into_owned();
reply.answers.push(ResourceRecord::new(
name.clone(),
simple_dns::CLASS::IN,
TTL,
RData::PTR(PTR(target)),
));
}
Answer::Soa => {
reply.set_flags(PacketFlag::AUTHORITATIVE_ANSWER);
reply.answers.push(soa_record(zone)?);
}
Answer::NoData => {
reply.set_flags(PacketFlag::AUTHORITATIVE_ANSWER);
// The authority section carries the SOA so a resolver knows how
// long it may remember that there is nothing here.
reply.name_servers.push(soa_record(zone)?);
}
Answer::NoSuchName => {
reply.set_flags(PacketFlag::AUTHORITATIVE_ANSWER);
*reply.rcode_mut() = RCODE::NameError;
reply.name_servers.push(soa_record(zone)?);
}
}
reply.build_bytes_vec_compressed().ok()
}
/// The zone's start of authority.
fn soa_record(zone: &Zone) -> Option<ResourceRecord<'static>> {
let origin = Name::new(zone.origin().as_str()).ok()?.into_owned();
Some(ResourceRecord::new(
origin.clone(),
simple_dns::CLASS::IN,
TTL,
RData::SOA(SOA {
mname: origin.clone(),
// There is no mailbox behind this zone and inventing one would
// be a fiction; the origin itself is the honest answer.
rname: origin,
serial: zone.serial(),
refresh: TTL as i32,
retry: TTL as i32,
expire: 86_400,
minimum: TTL,
}),
))
}
fn query_kind(qtype: QTYPE) -> Query {
match qtype {
QTYPE::TYPE(TYPE::A) => Query::A,
QTYPE::TYPE(TYPE::PTR) => Query::Ptr,
QTYPE::TYPE(TYPE::SOA) => Query::Soa,
QTYPE::TYPE(TYPE::NS) => Query::Ns,
// ANY is answered as an address question rather than by dumping the
// zone: an ANY that returns everything is an amplification gift.
QTYPE::ANY => Query::A,
_ => Query::Other,
}
}
/// Whether the client offered EDNS, and so how large an answer it will take.
fn udp_limit(query: &[u8]) -> usize {
simple_dns::Packet::parse(query)
.ok()
.and_then(|packet| packet.opt().map(|opt| opt.udp_packet_size as usize))
.unwrap_or(CLASSIC_UDP_LIMIT as u16 as usize)
.clamp(CLASSIC_UDP_LIMIT, MAX_MESSAGE_LEN)
}
/// Cuts an answer down to what the client said it would take.
///
/// The records are dropped and the truncated bit set, which tells a resolver
/// to ask again over TCP. Sending a reply it cannot reassemble would just
/// look like packet loss.
fn truncate_for_udp(query: &[u8], reply: Vec<u8>) -> Vec<u8> {
let limit = udp_limit(query);
if reply.len() <= limit {
return reply;
}
let Ok(parsed) = simple_dns::Packet::parse(&reply) else {
return reply;
};
let mut short = simple_dns::Packet::new_reply(parsed.id());
short.set_flags(PacketFlag::TRUNCATION | PacketFlag::AUTHORITATIVE_ANSWER);
*short.rcode_mut() = parsed.rcode();
for question in &parsed.questions {
short.questions.push(question.clone());
}
short.build_bytes_vec().unwrap_or(reply)
}
/// A running DNS server.
#[derive(Debug)]
pub struct DnsServer {
local_addr: SocketAddr,
tasks: Vec<tokio::task::JoinHandle<()>>,
}
impl DnsServer {
/// Binds and starts answering.
///
/// Both transports on the same address and port, as a resolver expects:
/// it falls back to TCP when an answer does not fit, and a server that
/// only listened on UDP would leave it with nowhere to go.
pub async fn bind(addr: SocketAddr, zone: SharedZone) -> std::io::Result<Self> {
let udp = tokio::net::UdpSocket::bind(addr).await?;
let local_addr = udp.local_addr()?;
let tcp = tokio::net::TcpListener::bind(local_addr).await?;
let udp_zone = zone.clone();
let udp_task = tokio::spawn(async move {
let mut buffer = vec![0u8; MAX_MESSAGE_LEN];
loop {
let (read, from) = match udp.recv_from(&mut buffer).await {
Ok(pair) => pair,
Err(err) => {
tracing::debug!(%err, "dns udp receive failed");
continue;
}
};
let query = &buffer[..read];
let Some(reply) = respond(&udp_zone.get(), query) else {
continue;
};
let reply = truncate_for_udp(query, reply);
if let Err(err) = udp.send_to(&reply, from).await {
tracing::debug!(%err, "dns udp reply failed");
}
}
});
let tcp_zone = zone.clone();
let tcp_task = tokio::spawn(async move {
let permits = Arc::new(tokio::sync::Semaphore::new(MAX_TCP_CONNECTIONS));
loop {
let (stream, _) = match tcp.accept().await {
Ok(pair) => pair,
Err(err) => {
tracing::debug!(%err, "dns tcp accept failed");
continue;
}
};
let Ok(permit) = Arc::clone(&permits).acquire_owned().await else {
return;
};
let zone = tcp_zone.clone();
tokio::spawn(async move {
let _permit = permit;
// Bounded, so a client that connects and says nothing
// cannot hold a slot open.
let _ = tokio::time::timeout(TCP_TIMEOUT, serve_tcp(stream, zone)).await;
});
}
});
Ok(Self {
local_addr,
tasks: vec![udp_task, tcp_task],
})
}
/// The address it is answering on.
pub fn local_addr(&self) -> SocketAddr {
self.local_addr
}
}
impl Drop for DnsServer {
fn drop(&mut self) {
for task in &self.tasks {
task.abort();
}
}
}
async fn serve_tcp(mut stream: tokio::net::TcpStream, zone: SharedZone) -> std::io::Result<()> {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
loop {
let mut header = [0u8; 2];
if stream.read_exact(&mut header).await.is_err() {
return Ok(());
}
// Checked before the buffer is allocated, as everywhere else that
// reads a length off a wire.
let len = u16::from_be_bytes(header) as usize;
if len == 0 || len > MAX_MESSAGE_LEN {
return Ok(());
}
let mut query = vec![0u8; len];
stream.read_exact(&mut query).await?;
let Some(reply) = respond(&zone.get(), &query) else {
return Ok(());
};
let Ok(len) = u16::try_from(reply.len()) else {
return Ok(());
};
stream.write_all(&len.to_be_bytes()).await?;
stream.write_all(&reply).await?;
stream.flush().await?;
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::dns::zone::ZoneName;
use std::net::Ipv4Addr;
fn zone() -> Zones {
Zones::new([one_zone()])
}
fn one_zone() -> Zone {
Zone::new(
ZoneName::new("lab").unwrap(),
[
("music".to_string(), Ipv4Addr::new(10, 13, 37, 237)),
("ai".to_string(), Ipv4Addr::new(10, 13, 37, 69)),
],
)
}
fn ask(name: &str, qtype: TYPE) -> Vec<u8> {
let mut packet = simple_dns::Packet::new_query(0x1234);
packet.questions.push(simple_dns::Question::new(
Name::new(name).unwrap(),
qtype.into(),
QCLASS::CLASS(simple_dns::CLASS::IN),
false,
));
packet.build_bytes_vec().unwrap()
}
/// The bytes of a reply. Parsed by each caller, because a parsed packet
/// borrows from them.
fn answer(query: &[u8]) -> Vec<u8> {
respond(&zone(), query).expect("a reply")
}
#[test]
fn a_member_is_answered_authoritatively() {
let bytes = answer(&ask("music.lab", TYPE::A));
let reply = simple_dns::Packet::parse(&bytes).unwrap();
assert_eq!(reply.rcode(), RCODE::NoError);
assert!(reply.has_flags(PacketFlag::RESPONSE));
assert!(reply.has_flags(PacketFlag::AUTHORITATIVE_ANSWER));
assert!(!reply.has_flags(PacketFlag::RECURSION_AVAILABLE));
assert_eq!(reply.answers.len(), 1);
assert_eq!(reply.questions.len(), 1, "the question is echoed");
match &reply.answers[0].rdata {
RData::A(a) => assert_eq!(Ipv4Addr::from(a.address), Ipv4Addr::new(10, 13, 37, 237)),
other => panic!("expected an A record, got {other:?}"),
}
}
#[test]
fn an_absent_name_is_denied_with_a_soa_to_cache_the_denial() {
let bytes = answer(&ask("nobody.lab", TYPE::A));
let reply = simple_dns::Packet::parse(&bytes).unwrap();
assert_eq!(reply.rcode(), RCODE::NameError);
assert!(reply.answers.is_empty());
assert_eq!(
reply.name_servers.len(),
1,
"a SOA bounds the negative cache"
);
}
#[test]
fn a_name_that_exists_without_that_record_is_not_denied() {
// NODATA, not NXDOMAIN: denying the name would stop a resolver
// asking for the A record it could have had.
let bytes = answer(&ask("music.lab", TYPE::AAAA));
let reply = simple_dns::Packet::parse(&bytes).unwrap();
assert_eq!(reply.rcode(), RCODE::NoError);
assert!(reply.answers.is_empty());
assert_eq!(reply.name_servers.len(), 1);
}
#[test]
fn anything_outside_the_zone_is_refused_and_never_forwarded() {
for name in ["example.com", "evillab", "google.com"] {
let bytes = answer(&ask(name, TYPE::A));
let reply = simple_dns::Packet::parse(&bytes).unwrap();
assert_eq!(reply.rcode(), RCODE::Refused, "{name}");
assert!(reply.answers.is_empty());
}
}
#[test]
fn an_address_is_answered_backwards() {
let bytes = answer(&ask("237.37.13.10.in-addr.arpa", TYPE::PTR));
let reply = simple_dns::Packet::parse(&bytes).unwrap();
assert_eq!(reply.rcode(), RCODE::NoError);
match &reply.answers[0].rdata {
RData::PTR(ptr) => assert_eq!(ptr.0.to_string(), "music.lab"),
other => panic!("expected a PTR record, got {other:?}"),
}
}
#[test]
fn a_reply_is_never_sent_to_something_that_was_not_a_question() {
// Answering a response would make this a reflector for anyone who
// can spoof a source address.
let mut packet = simple_dns::Packet::new_reply(1);
packet.set_flags(PacketFlag::RESPONSE);
assert!(respond(&zone(), &packet.build_bytes_vec().unwrap()).is_none());
assert!(respond(&zone(), b"").is_none());
assert!(respond(&zone(), b"not a dns packet at all").is_none());
}
#[test]
fn a_question_with_no_question_in_it_is_a_format_error() {
let packet = simple_dns::Packet::new_query(7);
let bytes = respond(&zone(), &packet.build_bytes_vec().unwrap()).unwrap();
let reply = simple_dns::Packet::parse(&bytes).unwrap();
assert_eq!(reply.rcode(), RCODE::FormatError);
}
#[test]
fn a_class_other_than_internet_is_refused() {
let mut packet = simple_dns::Packet::new_query(9);
packet.questions.push(simple_dns::Question::new(
Name::new("music.lab").unwrap(),
TYPE::A.into(),
QCLASS::CLASS(simple_dns::CLASS::CH),
false,
));
let bytes = respond(&zone(), &packet.build_bytes_vec().unwrap()).unwrap();
assert_eq!(
simple_dns::Packet::parse(&bytes).unwrap().rcode(),
RCODE::Refused
);
}
#[test]
fn an_answer_too_large_for_udp_is_truncated_rather_than_dropped() {
// A resolver that gets a truncated reply asks again over TCP; one
// that gets nothing back just waits.
let many: Vec<(String, Ipv4Addr)> = (0..200)
.map(|i| ("host".to_string(), Ipv4Addr::new(10, 13, 37, i as u8)))
.collect();
let wide = Zones::new([Zone::new(ZoneName::new("lab").unwrap(), many)]);
let query = ask("host.lab", TYPE::A);
let full = respond(&wide, &query).unwrap();
assert!(
full.len() > CLASSIC_UDP_LIMIT,
"the test needs a big answer"
);
let short = truncate_for_udp(&query, full);
assert!(short.len() <= CLASSIC_UDP_LIMIT);
let parsed = simple_dns::Packet::parse(&short).unwrap();
assert!(parsed.has_flags(PacketFlag::TRUNCATION));
assert_eq!(parsed.questions.len(), 1);
}
#[tokio::test]
async fn the_server_answers_over_udp_and_tcp_on_one_address() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let shared = SharedZone::new(zone());
let server = DnsServer::bind("127.0.0.1:0".parse().unwrap(), shared.clone())
.await
.unwrap();
let addr = server.local_addr();
let client = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
client
.send_to(&ask("music.lab", TYPE::A), addr)
.await
.unwrap();
let mut buffer = vec![0u8; MAX_MESSAGE_LEN];
let read = client.recv(&mut buffer).await.unwrap();
let reply = simple_dns::Packet::parse(&buffer[..read]).unwrap();
assert_eq!(reply.answers.len(), 1);
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
let query = ask("ai.lab", TYPE::A);
stream
.write_all(&(query.len() as u16).to_be_bytes())
.await
.unwrap();
stream.write_all(&query).await.unwrap();
let mut header = [0u8; 2];
stream.read_exact(&mut header).await.unwrap();
let mut body = vec![0u8; u16::from_be_bytes(header) as usize];
stream.read_exact(&mut body).await.unwrap();
let reply = simple_dns::Packet::parse(&body).unwrap();
assert_eq!(reply.answers.len(), 1);
}
#[tokio::test]
async fn the_same_zone_is_answered_over_ipv6_as_over_ipv4() {
// A question arrives over whichever family the resolver chooses, so
// a listener is opened from each list in the plan and both have to
// answer the same thing. Answering one family only leaves the other
// timing out, which looks like a broken overlay.
let shared = SharedZone::new(zone());
let plan = crate::dns::listen_plan(0);
let mut answered = 0;
for family in plan.families() {
let candidate = family[0];
let server = match DnsServer::bind(candidate, shared.clone()).await {
Ok(server) => server,
// A host with IPv6 switched off in the kernel cannot bind
// `::1`, and the agent copes with that by design; so does
// this. The other family still has to work.
Err(err) if candidate.is_ipv6() => {
eprintln!("no IPv6 loopback on this host ({err}); skipping that family");
continue;
}
Err(err) => panic!("cannot listen on {candidate}: {err}"),
};
let addr = server.local_addr();
assert_eq!(addr.is_ipv6(), candidate.is_ipv6());
let client = tokio::net::UdpSocket::bind(if addr.is_ipv6() {
"[::1]:0"
} else {
"127.0.0.1:0"
})
.await
.unwrap();
client
.send_to(&ask("music.lab", TYPE::A), addr)
.await
.unwrap();
let mut buffer = vec![0u8; MAX_MESSAGE_LEN];
let read = client.recv(&mut buffer).await.unwrap();
let reply = simple_dns::Packet::parse(&buffer[..read]).unwrap();
assert_eq!(reply.rcode(), RCODE::NoError, "over {addr}");
match &reply.answers[0].rdata {
// The same IPv4 answer either way: the family a question
// travelled over says nothing about what the answer is.
RData::A(a) => {
assert_eq!(Ipv4Addr::from(a.address), Ipv4Addr::new(10, 13, 37, 237))
}
other => panic!("expected an A record over {addr}, got {other:?}"),
}
answered += 1;
}
assert!(answered > 0, "at least one family must have answered");
}
#[tokio::test]
async fn replacing_the_zone_changes_what_the_running_server_answers() {
let shared = SharedZone::one(Zone::new(ZoneName::new("lab").unwrap(), []));
let server = DnsServer::bind("127.0.0.1:0".parse().unwrap(), shared.clone())
.await
.unwrap();
let addr = server.local_addr();
let client = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
let mut buffer = vec![0u8; MAX_MESSAGE_LEN];
client
.send_to(&ask("music.lab", TYPE::A), addr)
.await
.unwrap();
let read = client.recv(&mut buffer).await.unwrap();
assert_eq!(
simple_dns::Packet::parse(&buffer[..read]).unwrap().rcode(),
RCODE::NameError
);
// A member joins: no rebind, no dropped socket.
shared.set(zone());
client
.send_to(&ask("music.lab", TYPE::A), addr)
.await
.unwrap();
let read = client.recv(&mut buffer).await.unwrap();
let reply = simple_dns::Packet::parse(&buffer[..read]).unwrap();
assert_eq!(reply.rcode(), RCODE::NoError);
assert_eq!(reply.answers.len(), 1);
}
}
+749
View File
@@ -0,0 +1,749 @@
//! What the overlay answers to, as a DNS zone.
//!
//! Nothing here touches a socket or the operating system, and it does not
//! depend on a DNS wire library either: it takes a roster and a question and
//! says what the answer is. That is what makes the interesting parts — which
//! names exist, what "does not exist" means as against "exists with nothing
//! of that type", and what is outside the zone entirely — testable on their
//! own.
//!
//! # Where the names come from
//!
//! From signed state, which is why a member that is switched off still
//! resolves. Its claim outlived the session, so the name and the address are
//! both still there to answer with. Nothing is invented for a member that
//! claimed neither.
//!
//! Only IPv4 is served. The IPv6 overlay address is derived from a
//! WireGuard key that travels in live announcements and is not in signed
//! state, so it cannot be answered for a member that is away — and answering
//! for some members and not others depending on whether they happen to be
//! online is worse than not answering at all.
use std::collections::BTreeMap;
use std::net::Ipv4Addr;
/// Longest a DNS name may be, in the presentation form used here.
const MAX_NAME_LEN: usize = 253;
/// Longest one label may be.
const MAX_LABEL_LEN: usize = 63;
/// Why a zone name cannot be used.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum ZoneError {
/// The name was empty, or became empty once normalised.
#[error("a zone name must have at least one label")]
Empty,
/// One label was unusable.
#[error("`{label}` is not a usable DNS label: {reason}")]
Label {
/// The offending label.
label: String,
/// What is wrong with it.
reason: &'static str,
},
/// The whole name is too long.
#[error("a zone name must be at most {MAX_NAME_LEN} characters")]
TooLong,
}
/// A validated, canonical zone name.
///
/// Held without a trailing dot and lower-cased, so comparison is a plain
/// string comparison rather than a special case at every use.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ZoneName(String);
impl ZoneName {
/// Validates and normalises a zone name.
pub fn new(raw: &str) -> Result<Self, ZoneError> {
let trimmed = raw.trim().trim_end_matches('.').to_ascii_lowercase();
if trimmed.is_empty() {
return Err(ZoneError::Empty);
}
if trimmed.len() > MAX_NAME_LEN {
return Err(ZoneError::TooLong);
}
for label in trimmed.split('.') {
let reason = if label.is_empty() {
Some("it is empty")
} else if label.len() > MAX_LABEL_LEN {
Some("it is longer than 63 characters")
} else if !label
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
{
Some("only letters, digits, `-` and `_` are allowed")
} else if label.starts_with('-') || label.ends_with('-') {
Some("a label may not start or end with `-`")
} else {
None
};
if let Some(reason) = reason {
return Err(ZoneError::Label {
label: label.to_string(),
reason,
});
}
}
Ok(Self(trimmed))
}
/// The name, without a trailing dot.
pub fn as_str(&self) -> &str {
&self.0
}
/// The last label, which is what would collide with a real top-level
/// domain.
pub fn top_label(&self) -> &str {
self.0.rsplit('.').next().unwrap_or(&self.0)
}
/// What is worrying about this zone name, if anything.
///
/// A warning and never a refusal: the name is the user's to choose, and
/// a private zone that shadows a public one is a decision, not a
/// mistake. Saying nothing would let it be an accident.
pub fn collision(&self) -> Option<String> {
let top = self.top_label();
// Reserved for exactly this use and never delegated, so nothing to
// say. See RFC 6761 and RFC 8375.
const RESERVED: &[&str] = &[
"internal",
"home",
"test",
"example",
"invalid",
"localhost",
];
if RESERVED.contains(&top) {
return None;
}
if top == "local" {
return Some(
"`.local` belongs to multicast DNS: on a host running Avahi or \
systemd-resolved's mDNS, names under it are resolved by that and \
not by this agent"
.to_string(),
);
}
// A real top-level domain is the whole of it. A name that is not
// delegated today might be one day — and saying so fires on every
// private name anybody ever picks, which is a warning about
// nothing that teaches people to ignore warnings.
tld::exist_case_insensitive(top).then(|| {
format!(
"`.{top}` is a real top-level domain, so every public name under it \
becomes unreachable from this host while the overlay is up"
)
})
}
/// Whether `name` is this zone or sits under it.
///
/// Compared label-wise, so `evilzone` does not count as being under
/// `zone`.
pub fn covers(&self, name: &str) -> bool {
let name = name.trim_end_matches('.').to_ascii_lowercase();
name == self.0
|| name
.strip_suffix(&self.0)
.is_some_and(|rest| rest.ends_with('.'))
}
/// The part of `name` below this zone, if it is under it.
fn relative(&self, name: &str) -> Option<String> {
let name = name.trim_end_matches('.').to_ascii_lowercase();
if name == self.0 {
return Some(String::new());
}
let rest = name.strip_suffix(&self.0)?;
let rest = rest.strip_suffix('.')?;
(!rest.is_empty()).then(|| rest.to_string())
}
}
/// The kinds of question this zone knows how to answer.
///
/// Its own enum rather than the wire library's, so the decision of what to
/// answer does not depend on how a packet is encoded.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Query {
/// An IPv4 address.
A,
/// A name for an address.
Ptr,
/// The zone's start of authority.
Soa,
/// The zone's name servers.
Ns,
/// Anything else, including AAAA.
Other,
}
/// What the zone has to say.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Answer {
/// Addresses for the name asked about.
Addresses(Vec<Ipv4Addr>),
/// A name for the address asked about.
Name(String),
/// The zone's start of authority.
Soa,
/// The name exists here but has nothing of the type asked for.
///
/// Distinct from [`Answer::NoSuchName`] because the two are different
/// answers on the wire: this one is a success with no records, and a
/// resolver must not take it as proof the name is absent. Getting them
/// the wrong way round teaches a resolver to cache the wrong thing.
NoData,
/// No such name in this zone.
NoSuchName,
/// Not a name this zone is responsible for.
NotOurs,
}
/// The names and addresses of one network, ready to answer questions.
#[derive(Debug, Clone)]
pub struct Zone {
origin: ZoneName,
/// Name relative to the origin, to the addresses it answers to.
hosts: BTreeMap<String, Vec<Ipv4Addr>>,
/// Address to the name that holds it.
names: BTreeMap<Ipv4Addr, String>,
/// Changes whenever the contents do.
serial: u32,
}
impl Zone {
/// Builds a zone from the members that have both a name and an address.
///
/// A member with one but not the other contributes nothing: a name with
/// no address cannot be answered, and an address with no name has nothing
/// to be asked about.
pub fn new(origin: ZoneName, members: impl IntoIterator<Item = (String, Ipv4Addr)>) -> Self {
let mut hosts: BTreeMap<String, Vec<Ipv4Addr>> = BTreeMap::new();
let mut names: BTreeMap<Ipv4Addr, String> = BTreeMap::new();
for (hostname, address) in members {
let hostname = hostname.trim_matches('.').to_ascii_lowercase();
if hostname.is_empty() {
continue;
}
let addresses = hosts.entry(hostname.clone()).or_default();
if !addresses.contains(&address) {
addresses.push(address);
}
// First name wins, and the map is ordered, so the reverse answer
// is the same on every replica rather than depending on the order
// records happened to arrive in.
names.entry(address).or_insert(hostname);
}
for addresses in hosts.values_mut() {
addresses.sort();
}
let serial = content_serial(&hosts);
Self {
origin,
hosts,
names,
serial,
}
}
/// The zone's origin.
pub fn origin(&self) -> &ZoneName {
&self.origin
}
/// A number that changes whenever the contents do.
pub fn serial(&self) -> u32 {
self.serial
}
/// How many names it answers for.
pub fn len(&self) -> usize {
self.hosts.len()
}
/// Whether it answers for nothing.
pub fn is_empty(&self) -> bool {
self.hosts.is_empty()
}
/// Every name it answers for, with its addresses.
pub fn entries(&self) -> impl Iterator<Item = (&str, &[Ipv4Addr])> {
self.hosts
.iter()
.map(|(name, addresses)| (name.as_str(), addresses.as_slice()))
}
/// The reverse zones this zone is authoritative for.
///
/// Only when the range lands on an octet boundary. Claiming a reverse
/// zone larger than the range would shadow reverse lookups for addresses
/// that are nothing to do with us, which is worse than not answering.
pub fn reverse_origin(base: Ipv4Addr, prefix_len: u8) -> Option<String> {
let octets = base.octets();
match prefix_len {
8 => Some(format!("{}.in-addr.arpa", octets[0])),
16 => Some(format!("{}.{}.in-addr.arpa", octets[1], octets[0])),
24 => Some(format!(
"{}.{}.{}.in-addr.arpa",
octets[2], octets[1], octets[0]
)),
_ => None,
}
}
/// Answers one question.
pub fn lookup(&self, qname: &str, query: Query) -> Answer {
if let Some(address) = reverse_address(qname) {
return match self.names.get(&address) {
Some(name) if query == Query::Ptr => {
Answer::Name(format!("{name}.{}", self.origin.as_str()))
}
Some(_) => Answer::NoData,
None => Answer::NoSuchName,
};
}
let Some(relative) = self.origin.relative(qname) else {
return Answer::NotOurs;
};
// The apex: the zone itself exists whether or not anybody is in it.
if relative.is_empty() {
return match query {
Query::Soa => Answer::Soa,
Query::Ns => Answer::NoData,
_ => Answer::NoData,
};
}
match self.hosts.get(&relative) {
Some(addresses) if query == Query::A => Answer::Addresses(addresses.clone()),
// The name is here, it just has no AAAA and never will while
// only IPv4 is served. Saying "no such name" instead would tell
// a resolver to stop asking for the A record too.
Some(_) => Answer::NoData,
None => Answer::NoSuchName,
}
}
}
/// Every zone one agent serves.
///
/// One agent has one identity and as many networks as it likes, and each of
/// them is a zone of its own named after the network. They are served
/// together because they arrive at the same socket: a question is not
/// labelled with the network it belongs to, only with a name, and the
/// suffix is what decides. Nothing is shared between them — a member of one
/// network is not a name in another.
#[derive(Debug, Clone, Default)]
pub struct Zones {
zones: Vec<Zone>,
}
impl Zones {
/// Collects zones, dropping any that repeats an origin already taken.
///
/// Two networks can be given the same name, and then only one of them
/// can own the suffix. First wins, deterministically, rather than one
/// shadowing the other depending on the order they happened to start.
pub fn new(zones: impl IntoIterator<Item = Zone>) -> Self {
let mut taken: Vec<Zone> = Vec::new();
for zone in zones {
if taken.iter().any(|other| other.origin() == zone.origin()) {
continue;
}
taken.push(zone);
}
Self { zones: taken }
}
/// The zones, in the order they will be consulted.
pub fn iter(&self) -> impl Iterator<Item = &Zone> {
self.zones.iter()
}
/// How many zones are served.
pub fn len(&self) -> usize {
self.zones.len()
}
/// Whether nothing is served.
pub fn is_empty(&self) -> bool {
self.zones.is_empty()
}
/// How many names are answered for, across every zone.
pub fn names(&self) -> usize {
self.zones.iter().map(Zone::len).sum()
}
/// Answers one question, and says which zone answered it.
///
/// The answer has to travel with its zone because the SOA that bounds a
/// denial belongs to the zone that denied it, and serving several means
/// that is no longer a foregone conclusion.
///
/// A reverse question carries no suffix to match on, so every zone is
/// asked and the one that holds the address answers; if none does, the
/// denial comes from whichever is authoritative for that reverse zone,
/// and from nowhere at all when none is.
pub fn lookup(&self, qname: &str, query: Query) -> Option<(&Zone, Answer)> {
let mut denial: Option<(&Zone, Answer)> = None;
for zone in &self.zones {
match zone.lookup(qname, query) {
// Not this zone's question. Another may still own it.
Answer::NotOurs => continue,
// A zone that owns the suffix but not the name can still be
// overruled by one that has an answer: only reverse
// questions reach more than one zone, and there the zone
// holding the address is the one that knows.
answer @ (Answer::NoSuchName | Answer::NoData) => {
denial.get_or_insert((zone, answer));
}
answer => return Some((zone, answer)),
}
}
denial
}
}
/// The address a reverse name asks about, if it is one.
fn reverse_address(qname: &str) -> Option<Ipv4Addr> {
let name = qname.trim_end_matches('.').to_ascii_lowercase();
let rest = name.strip_suffix(".in-addr.arpa")?;
let mut octets = [0u8; 4];
let mut seen = 0;
for (index, part) in rest.split('.').enumerate() {
if index >= 4 {
return None;
}
octets[3 - index] = part.parse().ok()?;
seen += 1;
}
(seen == 4).then(|| Ipv4Addr::from(octets))
}
/// A serial that changes with the contents and not otherwise.
///
/// Derived rather than counted, so two agents holding the same roster agree,
/// and a restart does not go backwards.
fn content_serial(hosts: &BTreeMap<String, Vec<Ipv4Addr>>) -> u32 {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
for (name, addresses) in hosts {
name.hash(&mut hasher);
for address in addresses {
address.octets().hash(&mut hasher);
}
}
// Never zero: a zero serial is legal but reads like "unset" in a log.
(hasher.finish() as u32).max(1)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn zone() -> Zone {
Zone::new(
ZoneName::new("lab").unwrap(),
[
("music".to_string(), Ipv4Addr::new(10, 13, 37, 237)),
("ai".to_string(), Ipv4Addr::new(10, 13, 37, 69)),
],
)
}
#[test]
fn a_zone_name_is_normalised_and_checked() {
assert_eq!(ZoneName::new("LAB.").unwrap().as_str(), "lab");
assert_eq!(
ZoneName::new(" lab.internal ").unwrap().as_str(),
"lab.internal"
);
assert_eq!(
ZoneName::new("lab.internal").unwrap().top_label(),
"internal"
);
assert_eq!(ZoneName::new("").unwrap_err(), ZoneError::Empty);
assert_eq!(ZoneName::new(".").unwrap_err(), ZoneError::Empty);
assert!(matches!(
ZoneName::new("a..b").unwrap_err(),
ZoneError::Label { .. }
));
assert!(matches!(
ZoneName::new("-lab").unwrap_err(),
ZoneError::Label { .. }
));
assert!(matches!(
ZoneName::new("la b").unwrap_err(),
ZoneError::Label { .. }
));
assert!(matches!(
ZoneName::new(&"x".repeat(64)).unwrap_err(),
ZoneError::Label { .. }
));
}
#[test]
fn a_zone_name_that_shadows_a_public_one_is_flagged_but_allowed() {
// Flagged, never refused: shadowing is the user's decision to make.
let ru = ZoneName::new("ru").unwrap();
let warning = ru.collision().expect("a real TLD is worth mentioning");
assert!(warning.contains("real top-level domain"), "{warning}");
assert!(ZoneName::new("com").unwrap().collision().is_some());
assert!(ZoneName::new("lab.com").unwrap().collision().is_some());
// Reserved for private use, so nothing to say.
for quiet in ["internal", "lab.internal", "home", "test", "invalid"] {
assert_eq!(ZoneName::new(quiet).unwrap().collision(), None, "{quiet}");
}
// `.local` is not a delegated TLD, but it is not free either.
let local = ZoneName::new("local").unwrap().collision().unwrap();
assert!(local.contains("multicast DNS"), "{local}");
// A name that collides with nothing is said nothing about. It
// might be delegated one day, and warning about that fires on
// every private name anybody picks — a warning about nothing,
// which is how people learn to ignore warnings.
for quiet in ["lab", "a.lab", "pidar", "twar", "kitchen-table"] {
assert_eq!(ZoneName::new(quiet).unwrap().collision(), None, "{quiet}");
}
}
#[test]
fn a_neighbouring_name_is_not_inside_the_zone() {
// `evillab` ends with `lab`, and a suffix comparison that forgot the
// label boundary would hand it to us.
let origin = ZoneName::new("lab").unwrap();
assert!(origin.covers("lab"));
assert!(origin.covers("music.lab."));
assert!(origin.covers("a.b.lab"));
assert!(!origin.covers("evillab"));
assert!(!origin.covers("lab.example.com"));
assert!(!origin.covers("example.com"));
}
#[test]
fn a_member_resolves_by_name() {
let zone = zone();
assert_eq!(
zone.lookup("music.lab", Query::A),
Answer::Addresses(vec![Ipv4Addr::new(10, 13, 37, 237)])
);
// Case and a trailing dot are the same question.
assert_eq!(
zone.lookup("MUSIC.LAB.", Query::A),
Answer::Addresses(vec![Ipv4Addr::new(10, 13, 37, 237)])
);
}
#[test]
fn a_name_with_no_record_of_that_type_is_not_a_missing_name() {
// AAAA for a member that exists must be NODATA, not NXDOMAIN: an
// NXDOMAIN would tell the resolver the name is absent and stop it
// asking for the A record.
let zone = zone();
assert_eq!(zone.lookup("music.lab", Query::Other), Answer::NoData);
assert_eq!(zone.lookup("nobody.lab", Query::A), Answer::NoSuchName);
}
#[test]
fn questions_outside_the_zone_are_refused_rather_than_denied() {
// Denying them would be a lie: this server knows nothing about them.
let zone = zone();
assert_eq!(zone.lookup("example.com", Query::A), Answer::NotOurs);
assert_eq!(zone.lookup("evillab", Query::A), Answer::NotOurs);
}
#[test]
fn the_apex_exists_even_with_nobody_in_the_network() {
let empty = Zone::new(ZoneName::new("lab").unwrap(), []);
assert!(empty.is_empty());
assert_eq!(empty.lookup("lab", Query::Soa), Answer::Soa);
assert_eq!(empty.lookup("lab", Query::A), Answer::NoData);
assert_eq!(empty.lookup("music.lab", Query::A), Answer::NoSuchName);
}
#[test]
fn an_address_resolves_back_to_its_name() {
let zone = zone();
assert_eq!(
zone.lookup("237.37.13.10.in-addr.arpa", Query::Ptr),
Answer::Name("music.lab".into())
);
assert_eq!(
zone.lookup("9.37.13.10.in-addr.arpa", Query::Ptr),
Answer::NoSuchName
);
// A reverse name asked for the wrong type is still a name we know.
assert_eq!(
zone.lookup("237.37.13.10.in-addr.arpa", Query::A),
Answer::NoData
);
}
#[test]
fn a_reverse_zone_is_claimed_only_when_it_is_exactly_ours() {
// Claiming a reverse zone wider than the range would shadow lookups
// for addresses that have nothing to do with this network.
assert_eq!(
Zone::reverse_origin(Ipv4Addr::new(10, 13, 37, 0), 24).as_deref(),
Some("37.13.10.in-addr.arpa")
);
assert_eq!(
Zone::reverse_origin(Ipv4Addr::new(10, 13, 0, 0), 16).as_deref(),
Some("13.10.in-addr.arpa")
);
assert_eq!(Zone::reverse_origin(Ipv4Addr::new(10, 13, 37, 0), 25), None);
assert_eq!(Zone::reverse_origin(Ipv4Addr::new(100, 64, 0, 0), 10), None);
}
#[test]
fn members_with_nothing_to_say_are_left_out() {
let zone = Zone::new(
ZoneName::new("lab").unwrap(),
[(String::new(), Ipv4Addr::new(10, 0, 0, 1))],
);
assert!(zone.is_empty());
}
#[test]
fn two_members_sharing_a_name_both_answer_and_the_order_is_stable() {
// The state layer resolves name ownership; if two records still
// reach here, answering with both beats picking one at random.
let build = |flip: bool| {
let members = if flip {
vec![
("music".to_string(), Ipv4Addr::new(10, 0, 0, 2)),
("music".to_string(), Ipv4Addr::new(10, 0, 0, 1)),
]
} else {
vec![
("music".to_string(), Ipv4Addr::new(10, 0, 0, 1)),
("music".to_string(), Ipv4Addr::new(10, 0, 0, 2)),
]
};
Zone::new(ZoneName::new("lab").unwrap(), members)
};
assert_eq!(
build(false).lookup("music.lab", Query::A),
build(true).lookup("music.lab", Query::A)
);
assert_eq!(build(false).serial(), build(true).serial());
}
#[test]
fn the_serial_follows_the_contents() {
let a = zone();
let b = zone();
assert_eq!(a.serial(), b.serial(), "the same roster, the same serial");
let changed = Zone::new(
ZoneName::new("lab").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 13, 37, 238))],
);
assert_ne!(a.serial(), changed.serial());
}
#[test]
fn each_network_answers_only_for_its_own_zone() {
// One agent, several networks, one socket. A question carries a
// name and not the network it belongs to, so the suffix is what
// decides — and a member of one network is not a name in another.
let zones = Zones::new([
Zone::new(
ZoneName::new("lab").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 13, 37, 2))],
),
Zone::new(
ZoneName::new("home").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 20, 0, 2))],
),
]);
assert_eq!(zones.len(), 2);
assert_eq!(zones.names(), 2);
let (zone, answer) = zones.lookup("music.lab", Query::A).unwrap();
assert_eq!(zone.origin().as_str(), "lab");
assert_eq!(
answer,
Answer::Addresses(vec![Ipv4Addr::new(10, 13, 37, 2)])
);
let (zone, answer) = zones.lookup("music.home", Query::A).unwrap();
assert_eq!(zone.origin().as_str(), "home");
assert_eq!(answer, Answer::Addresses(vec![Ipv4Addr::new(10, 20, 0, 2)]));
// A name under neither is nobody's business here.
assert!(zones.lookup("music.example.com", Query::A).is_none());
assert!(Zones::default().lookup("music.lab", Query::A).is_none());
}
#[test]
fn a_denial_comes_from_the_zone_that_owns_the_name() {
// The SOA that bounds a negative answer belongs to the zone that
// denied it; with several served that is no longer a given.
let zones = Zones::new([
Zone::new(ZoneName::new("lab").unwrap(), []),
Zone::new(
ZoneName::new("home").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 20, 0, 2))],
),
]);
let (zone, answer) = zones.lookup("nobody.lab", Query::A).unwrap();
assert_eq!(zone.origin().as_str(), "lab");
assert_eq!(answer, Answer::NoSuchName);
}
#[test]
fn a_reverse_question_is_answered_by_whichever_zone_holds_the_address() {
// It carries no suffix to match on, so every zone is asked.
let zones = Zones::new([
Zone::new(
ZoneName::new("lab").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 13, 37, 2))],
),
Zone::new(
ZoneName::new("home").unwrap(),
[("kitchen".to_string(), Ipv4Addr::new(10, 20, 0, 2))],
),
]);
let (_, answer) = zones.lookup("2.0.20.10.in-addr.arpa", Query::Ptr).unwrap();
assert_eq!(answer, Answer::Name("kitchen.home".to_string()));
let (_, answer) = zones.lookup("2.37.13.10.in-addr.arpa", Query::Ptr).unwrap();
assert_eq!(answer, Answer::Name("music.lab".to_string()));
}
#[test]
fn two_networks_of_one_name_do_not_shadow_each_other_at_random() {
// Exactly the two-`LAB` case. Only one can own the suffix; which
// one must not depend on the order they happened to start in.
let first = Zone::new(
ZoneName::new("lab").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 13, 37, 2))],
);
let second = Zone::new(
ZoneName::new("lab").unwrap(),
[("music".to_string(), Ipv4Addr::new(10, 99, 0, 2))],
);
let zones = Zones::new([first, second]);
assert_eq!(zones.len(), 1, "one suffix, one zone");
let (_, answer) = zones.lookup("music.lab", Query::A).unwrap();
assert_eq!(
answer,
Answer::Addresses(vec![Ipv4Addr::new(10, 13, 37, 2)])
);
}
}
+208
View File
@@ -0,0 +1,208 @@
//! Error types for the whole library.
//!
//! The library never panics on untrusted network input: every decoding and
//! validation failure is represented as a [`ProtocolError`] and surfaced as a
//! rejected message or session, never as a process abort.
use std::path::PathBuf;
use crate::identity::NetworkId;
/// Convenient result alias used across the crate.
pub type Result<T, E = Error> = std::result::Result<T, E>;
/// Top level error type of the library.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// The supplied network name does not satisfy the documented rules.
#[error("invalid network name: {0}")]
InvalidNetworkName(&'static str),
/// The supplied network secret does not satisfy the documented rules.
///
/// The secret itself is never included in the message.
#[error("invalid network secret: {0}")]
InvalidNetworkSecret(&'static str),
/// A textual identifier could not be parsed.
#[error("invalid {kind}: {reason}")]
InvalidEncoding {
/// What was being parsed, e.g. `network id`.
kind: &'static str,
/// Why parsing failed.
reason: &'static str,
},
/// The mandatory state directory is already owned by another live agent.
///
/// This is an ownership lock, not a "file exists" check.
#[error("state directory {path} is owned by another running agent instance")]
StateLocked {
/// Directory that could not be locked.
path: PathBuf,
},
/// The mandatory state store is unusable. It is never silently recreated.
#[error("mandatory state store at {path} is unusable and was NOT reset: {reason}")]
StateCorrupted {
/// Path of the unusable store.
path: PathBuf,
/// Human readable reason, free of secrets.
reason: String,
},
/// The mandatory state store has a schema this build cannot handle.
#[error(
"state store schema version {found} is not supported by this build (supported: {supported})"
)]
UnsupportedSchema {
/// Version found on disk.
found: i64,
/// Version this build writes.
supported: i64,
},
/// A storage operation failed.
#[error("storage error: {0}")]
Storage(String),
/// A filesystem operation failed.
#[error("io error at {path}: {source}")]
Io {
/// Path involved in the failure.
path: PathBuf,
/// Underlying error.
#[source]
source: std::io::Error,
},
/// Binding or driving the iroh endpoint failed.
#[error("iroh endpoint error: {0}")]
Endpoint(String),
/// A control protocol violation.
#[error(transparent)]
Protocol(#[from] ProtocolError),
/// The requested network is not currently active on this agent.
#[error("network {0} is not active")]
NetworkNotActive(NetworkId),
/// The requested network is already active on this agent.
#[error("network {0} is already active")]
NetworkAlreadyActive(NetworkId),
/// The requested network is not configured in the state store.
#[error("network {0} is not configured")]
NetworkUnknown(NetworkId),
/// No session with that peer exists in the given network.
#[error("no authenticated session with peer {peer} in network {network}")]
NoSuchPeer {
/// Network the lookup was scoped to.
network: NetworkId,
/// Peer that was looked up, short form.
peer: String,
},
/// The agent is shutting down or already stopped.
#[error("agent is stopped")]
Stopped,
/// Something that had to answer in bounded time did not.
///
/// Reported instead of waited out: an agent that has wedged must not
/// leave a command hanging with nothing on screen and no way out.
#[error("{what} did not answer in time")]
Timeout {
/// What was asked, phrased so the message stands on its own.
what: String,
},
/// The request is understood and cannot be done as things stand.
#[error("{reason}")]
Rejected {
/// Why, phrased so the message stands on its own.
reason: String,
},
/// Discovery backend failure. Never fatal for the agent.
#[error("discovery error: {0}")]
Discovery(String),
}
/// Errors produced while speaking the control protocol.
///
/// These always result in rejecting a single message or a single session. They
/// never stop other networks, other peers, or the agent itself.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum ProtocolError {
/// The peer announced an unsupported control protocol version.
#[error("unsupported control protocol version {found} (this build speaks {supported})")]
UnsupportedVersion {
/// Version announced by the peer.
found: u16,
/// Version this build speaks.
supported: u16,
},
/// A frame header announced more bytes than the configured limit allows.
///
/// Checked *before* any buffer of that size is allocated.
#[error("frame of {announced} bytes exceeds the {limit} byte limit")]
FrameTooLarge {
/// Length announced in the frame header.
announced: u64,
/// Configured limit.
limit: usize,
},
/// A frame could not be decoded.
#[error("malformed frame: {0}")]
Malformed(&'static str),
/// The stream ended before a complete frame was read.
#[error("stream closed while reading a frame")]
StreamClosed,
/// An underlying stream read or write failed.
#[error("stream error: {0}")]
Stream(String),
/// The peer failed to prove knowledge of the derived network secret.
#[error("network authentication failed")]
AuthenticationFailed,
/// The peer asked for a network this agent does not have active.
#[error("peer requested an unknown or inactive network")]
UnknownNetwork,
/// A message carried a network id different from the session's network.
#[error("message network id does not match the authenticated session network")]
NetworkMismatch,
/// A regular control message arrived before the handshake completed.
#[error("control message received before authentication completed")]
NotAuthenticated,
/// A handshake step did not complete within the configured timeout.
#[error("handshake timed out")]
HandshakeTimeout,
/// A field exceeded its configured bound.
#[error("field `{field}` exceeds its limit ({len} > {limit})")]
FieldTooLarge {
/// Name of the offending field.
field: &'static str,
/// Observed length.
len: usize,
/// Configured limit.
limit: usize,
},
/// The connection is not usable for deriving channel binding material.
#[error("connection does not provide TLS exporter material: {0}")]
NoChannelBinding(String),
}
+83
View File
@@ -0,0 +1,83 @@
//! Device identity and network space identity.
//!
//! These are two independent things and must not be confused:
//!
//! * [`DeviceIdentity`] wraps the persistent iroh [`SecretKey`]. Its public key
//! *is* the iroh [`EndpointId`]. It survives restarts and survives a change of
//! the network secret.
//! * [`NetworkId`] identifies a network space and is derived purely from the
//! network name and shared secret. It is unrelated to any device key.
mod network;
pub use network::{
DiscoveryKey, IDENTITY_SCHEME, MAX_NETWORK_NAME_LEN, MAX_NETWORK_SECRET_LEN,
MIN_NETWORK_SECRET_LEN, NetworkDescriptor, NetworkId, NetworkKeys, NetworkName, NetworkSecret,
SECRET_TEXT_PREFIX,
};
use iroh::{EndpointId, SecretKey};
/// The persistent identity of this device.
///
/// Created once and stored in the mandatory state store. Restarting the agent
/// must not produce a new peer, so the stored secret key is always reused.
/// Corruption of the stored key is reported as an error and never silently
/// replaced by a fresh key.
#[derive(Clone)]
pub struct DeviceIdentity {
secret: SecretKey,
}
impl DeviceIdentity {
/// Generates a brand new device identity.
pub fn generate() -> Self {
Self {
secret: SecretKey::generate(),
}
}
/// Reconstructs a device identity from its stored 32 secret key bytes.
pub fn from_secret_bytes(bytes: &[u8; 32]) -> Self {
Self {
secret: SecretKey::from_bytes(bytes),
}
}
/// The iroh endpoint id, i.e. the public key of this device.
pub fn endpoint_id(&self) -> EndpointId {
self.secret.public()
}
/// The raw secret key bytes, for persistence and transport certificate generation.
pub fn secret_bytes(&self) -> [u8; 32] {
self.secret.to_bytes()
}
/// Signs a message with the device's persistent secret key.
pub fn sign(&self, message: &[u8]) -> iroh::Signature {
self.secret.sign(message)
}
/// A clone of the iroh secret key, for endpoint construction only.
pub(crate) fn secret_key(&self) -> SecretKey {
self.secret.clone()
}
/// A clone of the key used to sign this device's own state records.
///
/// The same persistent identity the control plane authenticates, so a
/// record signed today is still attributable after any absence.
pub(crate) fn signing_key(&self) -> SecretKey {
self.secret.clone()
}
}
impl std::fmt::Debug for DeviceIdentity {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeviceIdentity")
.field("endpoint_id", &self.endpoint_id().fmt_short().to_string())
.field("secret", &"<redacted>")
.finish()
}
}
+427
View File
@@ -0,0 +1,427 @@
//! Deterministic network space identity.
//!
//! A *network space* is fully determined by a [`NetworkName`] and a
//! [`NetworkSecret`]. Two agents that were given the same pair derive the same
//! [`NetworkId`], [`DiscoveryKey`] and handshake authentication key, with no
//! coordination, no creator, no timestamp and no leader election.
//!
//! # Derivation scheme (`tsunagi-network-id-v1`)
//!
//! All inputs are encoded with an unambiguous length-prefixed encoding, written
//! here as `LP(x) = u32_be(x.len()) || x`. String concatenation is never used.
//!
//! ```text
//! salt = SHA-256( LP("tsunagi-network-id-v1") || LP(name_utf8) )
//! prk = HKDF-SHA256-Extract(salt, ikm = secret_bytes)
//! info(label) = LP("tsunagi-network-id-v1") || LP(label)
//! network_id = HKDF-Expand(prk, info("network-id"), 32)
//! discovery_key = HKDF-Expand(prk, info("discovery-key"), 32)
//! auth_key = HKDF-Expand(prk, info("handshake-auth"), 32)
//! ```
//!
//! HKDF's `info` parameter is what separates the three derived values
//! (RFC 5869 §3.2). Learning `discovery_key` — which is published to a
//! discovery backend and is therefore semi-public — does not reveal `auth_key`,
//! so the discovery key must never be used as a password or bearer token.
//!
//! The scheme label is versioned and frozen. Upgrading this crate or bumping
//! the control protocol version must not change an existing [`NetworkId`].
use hkdf::Hkdf;
use sha2::{Digest, Sha256};
use zeroize::{Zeroize, Zeroizing};
use crate::error::{Error, Result};
/// Frozen label identifying the network identity derivation scheme.
///
/// Changing this string creates a different, incompatible network space for the
/// same name and secret. It must never be changed casually.
pub const IDENTITY_SCHEME: &str = "tsunagi-network-id-v1";
/// Maximum length of a network name in UTF-8 bytes.
pub const MAX_NETWORK_NAME_LEN: usize = 64;
/// Minimum length of a network secret in bytes.
///
/// The proof-of-concept targets high-entropy shared secrets. See
/// [`NetworkSecret::generate`].
pub const MIN_NETWORK_SECRET_LEN: usize = 16;
/// Maximum length of a network secret in bytes.
pub const MAX_NETWORK_SECRET_LEN: usize = 1024;
/// Human-readable prefix of the canonical secret text encoding.
pub const SECRET_TEXT_PREFIX: &str = "tsn1";
/// Appends `LP(bytes) = u32_be(len) || bytes` to `out`.
///
/// Panics are impossible here: the caller-provided slices are already bounded by
/// [`MAX_NETWORK_NAME_LEN`] / [`MAX_NETWORK_SECRET_LEN`], and the cast is
/// saturating for anything larger.
fn push_lp(out: &mut Vec<u8>, bytes: &[u8]) {
let len = u32::try_from(bytes.len()).unwrap_or(u32::MAX);
out.extend_from_slice(&len.to_be_bytes());
out.extend_from_slice(bytes);
}
/// The name half of a network space.
///
/// Rules, deliberately strict and never silently applied:
///
/// * 1..=[`MAX_NETWORK_NAME_LEN`] bytes of UTF-8.
/// * No ASCII control characters.
/// * No leading or trailing ASCII whitespace — such a name is **rejected**,
/// not trimmed.
/// * Used verbatim. No case folding and no Unicode normalisation is performed,
/// so `Home` and `home` are different network spaces.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NetworkName(String);
impl NetworkName {
/// Validates and wraps a network name.
pub fn new(name: impl Into<String>) -> Result<Self> {
let name = name.into();
if name.is_empty() {
return Err(Error::InvalidNetworkName("must not be empty"));
}
if name.len() > MAX_NETWORK_NAME_LEN {
return Err(Error::InvalidNetworkName(
"must not exceed 64 bytes of UTF-8",
));
}
if name.chars().any(|c| c.is_control()) {
return Err(Error::InvalidNetworkName(
"must not contain control characters",
));
}
let trimmed = name.trim_matches(|c: char| c.is_ascii_whitespace());
if trimmed.len() != name.len() {
return Err(Error::InvalidNetworkName(
"must not have leading or trailing ASCII whitespace",
));
}
Ok(Self(name))
}
/// Returns the name as a string slice.
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for NetworkName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::str::FromStr for NetworkName {
type Err = Error;
fn from_str(s: &str) -> Result<Self> {
Self::new(s)
}
}
/// The shared secret half of a network space.
///
/// This is the single shared secret the end user configures; "password" and
/// "secret" refer to the same value. The bytes are used **verbatim**: never
/// trimmed, case-folded, normalised or truncated.
///
/// The value is zeroized on drop and redacted from [`Debug`].
#[derive(Clone, PartialEq, Eq)]
pub struct NetworkSecret(Zeroizing<Vec<u8>>);
impl NetworkSecret {
/// Wraps raw secret bytes.
///
/// Requires at least [`MIN_NETWORK_SECRET_LEN`] bytes. This crate makes no
/// security promises for short, low-entropy human passphrases: there is no
/// PAKE here, so an offline guessing attack against a weak secret is cheap
/// for anyone who can reach the handshake.
pub fn from_bytes(bytes: impl Into<Vec<u8>>) -> Result<Self> {
let bytes = Zeroizing::new(bytes.into());
if bytes.len() < MIN_NETWORK_SECRET_LEN {
return Err(Error::InvalidNetworkSecret(
"must be at least 16 bytes; use NetworkSecret::generate()",
));
}
if bytes.len() > MAX_NETWORK_SECRET_LEN {
return Err(Error::InvalidNetworkSecret("must not exceed 1024 bytes"));
}
Ok(Self(bytes))
}
/// Generates a fresh 32-byte random secret.
///
/// This is the recommended way to create a network secret.
pub fn generate() -> Self {
let mut buf = vec![0u8; 32];
rand::fill(&mut buf[..]);
Self(Zeroizing::new(buf))
}
/// Parses the canonical text form produced by [`NetworkSecret::encode`].
///
/// The format is `tsn1` followed by lowercase unpadded RFC 4648 base32.
/// Parsing is strict: no whitespace, no case mixing in the payload.
pub fn decode(text: &str) -> Result<Self> {
let payload = text
.strip_prefix(SECRET_TEXT_PREFIX)
.ok_or(Error::InvalidNetworkSecret(
"canonical secrets start with `tsn1`",
))?;
let bytes = data_encoding::BASE32_NOPAD
.decode(payload.to_ascii_uppercase().as_bytes())
.map_err(|_| Error::InvalidNetworkSecret("not valid base32"))?;
Self::from_bytes(bytes)
}
/// Encodes the secret in its canonical text form.
///
/// The returned string is zeroized on drop. Never log it.
pub fn encode(&self) -> Zeroizing<String> {
let mut encoded = data_encoding::BASE32_NOPAD.encode(&self.0);
encoded.make_ascii_lowercase();
let out = Zeroizing::new(format!("{SECRET_TEXT_PREFIX}{encoded}"));
encoded.zeroize();
out
}
/// Exposes the raw secret bytes.
///
/// Callers must not log, serialise or copy these bytes into diagnostics.
pub(crate) fn expose(&self) -> &[u8] {
&self.0
}
}
impl std::fmt::Debug for NetworkSecret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("NetworkSecret(<redacted>)")
}
}
/// Encodes 32 bytes as lowercase unpadded base32.
fn b32(bytes: &[u8; 32]) -> String {
let mut s = data_encoding::BASE32_NOPAD.encode(bytes);
s.make_ascii_lowercase();
s
}
/// Decodes lowercase unpadded base32 into 32 bytes.
fn unb32(kind: &'static str, s: &str) -> Result<[u8; 32]> {
let bytes = data_encoding::BASE32_NOPAD
.decode(s.to_ascii_uppercase().as_bytes())
.map_err(|_| Error::InvalidEncoding {
kind,
reason: "not valid base32",
})?;
<[u8; 32]>::try_from(bytes.as_slice()).map_err(|_| Error::InvalidEncoding {
kind,
reason: "expected 32 bytes",
})
}
/// Public, non-secret identifier of a network space.
///
/// Safe to log, publish and put into status output. It does not authorise
/// anything on its own: an attacker who knows a `NetworkId` still cannot pass
/// the handshake without the secret.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NetworkId([u8; 32]);
impl NetworkId {
/// Returns the raw 32 bytes.
pub fn as_bytes(&self) -> &[u8; 32] {
&self.0
}
/// Builds a network id from raw bytes.
pub fn from_bytes(bytes: [u8; 32]) -> Self {
Self(bytes)
}
/// Returns a short prefix useful for logs.
pub fn fmt_short(&self) -> String {
b32(&self.0).chars().take(10).collect()
}
}
impl std::fmt::Display for NetworkId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&b32(&self.0))
}
}
impl std::fmt::Debug for NetworkId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "NetworkId({})", self.fmt_short())
}
}
impl std::str::FromStr for NetworkId {
type Err = Error;
fn from_str(s: &str) -> Result<Self> {
unb32("network id", s).map(Self)
}
}
/// Lookup key used to find candidates for a network in a discovery backend.
///
/// Derived from the secret, so it is not published in the clear the way a
/// [`NetworkId`] is. It is nevertheless **not** a credential: a discovery
/// backend, or anyone observing it, learns nothing that helps pass the
/// handshake.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct DiscoveryKey([u8; 32]);
impl DiscoveryKey {
/// Returns the raw 32 bytes.
pub fn as_bytes(&self) -> &[u8; 32] {
&self.0
}
/// Builds a discovery key from raw bytes.
pub fn from_bytes(bytes: [u8; 32]) -> Self {
Self(bytes)
}
}
impl std::fmt::Display for DiscoveryKey {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&b32(&self.0))
}
}
impl std::fmt::Debug for DiscoveryKey {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"DiscoveryKey({}…)",
b32(&self.0).chars().take(10).collect::<String>()
)
}
}
/// The deterministic, immutable description of a network space.
///
/// There is no competing genesis: this value contains no creator identity, no
/// creation time and no owner signature, so two agents started independently
/// with the same parameters produce byte-identical descriptors. The secret is
/// never part of it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NetworkDescriptor {
/// Frozen derivation scheme label, see [`IDENTITY_SCHEME`].
pub scheme: &'static str,
/// The network name.
pub name: NetworkName,
/// The derived public network identifier.
pub network_id: NetworkId,
}
impl NetworkDescriptor {
/// Canonical, unambiguous byte encoding of the descriptor.
pub fn to_canonical_bytes(&self) -> Vec<u8> {
let mut out = Vec::new();
push_lp(&mut out, self.scheme.as_bytes());
push_lp(&mut out, self.name.as_str().as_bytes());
push_lp(&mut out, &self.network_id.0);
out
}
}
/// All key material derived from a [`NetworkName`] and [`NetworkSecret`].
///
/// The authentication key is zeroized on drop and never leaves this crate.
#[derive(Clone)]
pub struct NetworkKeys {
network_id: NetworkId,
discovery_key: DiscoveryKey,
auth_key: Zeroizing<[u8; 32]>,
// Separate from the semi-public DiscoveryKey. Existing derivations stay frozen.
dht_write_key: Zeroizing<[u8; 32]>,
name: NetworkName,
}
impl NetworkKeys {
/// Derives all network key material.
///
/// This is a pure function of `(name, secret)`. It does not depend on the
/// device key, the hostname, the wall clock or the order in which agents
/// start.
pub fn derive(name: &NetworkName, secret: &NetworkSecret) -> Self {
let mut salt_input = Vec::new();
push_lp(&mut salt_input, IDENTITY_SCHEME.as_bytes());
push_lp(&mut salt_input, name.as_str().as_bytes());
let salt = Sha256::digest(&salt_input);
let hk = Hkdf::<Sha256>::new(Some(&salt), secret.expose());
let expand = |label: &str| -> [u8; 32] {
let mut info = Vec::new();
push_lp(&mut info, IDENTITY_SCHEME.as_bytes());
push_lp(&mut info, label.as_bytes());
let mut okm = [0u8; 32];
// 32 bytes is far below HKDF-SHA256's 255*32 limit, so this cannot fail.
match hk.expand(&info, &mut okm) {
Ok(()) => okm,
Err(_) => unreachable!("HKDF-SHA256 expand of 32 bytes cannot fail"),
}
};
Self {
network_id: NetworkId(expand("network-id")),
discovery_key: DiscoveryKey(expand("discovery-key")),
auth_key: Zeroizing::new(expand("handshake-auth")),
dht_write_key: Zeroizing::new(expand("mainline-rendezvous-write-v1")),
name: name.clone(),
}
}
/// The public network identifier.
pub fn network_id(&self) -> NetworkId {
self.network_id
}
/// The discovery lookup key.
pub fn discovery_key(&self) -> DiscoveryKey {
self.discovery_key
}
/// The network name.
pub fn name(&self) -> &NetworkName {
&self.name
}
/// The immutable descriptor of this network space.
pub fn descriptor(&self) -> NetworkDescriptor {
NetworkDescriptor {
scheme: IDENTITY_SCHEME,
name: self.name.clone(),
network_id: self.network_id,
}
}
/// The handshake authentication key. Crate-internal on purpose.
pub(crate) fn auth_key(&self) -> &[u8; 32] {
&self.auth_key
}
/// Secret signing material for network rendezvous; never exported or logged.
pub(crate) fn dht_write_key(&self) -> &[u8; 32] {
&self.dht_write_key
}
}
impl std::fmt::Debug for NetworkKeys {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NetworkKeys")
.field("name", &self.name)
.field("network_id", &self.network_id)
.field("discovery_key", &self.discovery_key)
.field("auth_key", &"<redacted>")
.finish()
}
}
File diff suppressed because it is too large. Load diff
+227
View File
@@ -0,0 +1,227 @@
//! A Unix socket adapter for the local control interface.
//!
//! One of the per-platform adapters; see [`super`]. It provides only what is
//! particular to a Unix socket — binding a listener, connecting a client, and
//! the owner-only permissions — and hands every accepted connection to the
//! shared [`serve_connection`](super::serve_connection). The request framing,
//! the dispatch and the client wrappers are the same on every platform and
//! live in [`super`], re-exported here so `ipc::unix::request_status` and its
//! siblings keep resolving.
//!
//! It is reachable only by a process that can open a file inside the agent's
//! owner-only state directory, and the socket itself is created mode `0600`.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::net::{UnixListener, UnixStream};
use tokio::task::JoinHandle;
use crate::error::{Error, Result};
// The transport-agnostic surface, re-exported so this module is a complete
// view of the local control interface on its own.
pub use super::{
CONTROL_PROTOCOL, ControlSocketAccess, EXCHANGE_TIMEOUT, ReportSource, is_serving,
join_network, leave_network, request_status, set_active, set_dns, set_hostname,
};
/// Serves the local control interface on a Unix socket.
#[derive(Debug)]
pub struct ControlSocket {
path: PathBuf,
task: Option<JoinHandle<()>>,
}
impl ControlSocket {
/// Binds the socket and starts serving.
///
/// A socket file left behind by a crashed agent is replaced, but only
/// after checking that nothing is listening on it, so two live agents
/// never fight over one path.
pub async fn bind(
path: impl AsRef<Path>,
source: Arc<dyn ReportSource>,
access: ControlSocketAccess,
) -> Result<Self> {
let path = path.as_ref().to_path_buf();
if let Some(parent) = path.parent() {
create_parent(parent, &access)?;
}
if path.exists() {
if is_serving(&path).await {
return Err(Error::StateLocked { path: path.clone() });
}
// Nothing is listening, so the file is a leftover.
std::fs::remove_file(&path).map_err(|source| Error::Io {
path: path.clone(),
source,
})?;
}
let listener = UnixListener::bind(&path).map_err(|source| Error::Io {
path: path.clone(),
source,
})?;
apply_access(&path, &access)?;
let task = tokio::spawn(serve(listener, source));
Ok(Self {
path,
task: Some(task),
})
}
/// The path being served.
pub fn path(&self) -> &Path {
&self.path
}
/// Stops serving and removes the socket file.
pub async fn shutdown(mut self) {
if let Some(task) = self.task.take() {
task.abort();
let _ = task.await;
}
let _ = std::fs::remove_file(&self.path);
}
}
impl Drop for ControlSocket {
fn drop(&mut self) {
if let Some(task) = self.task.take() {
task.abort();
}
let _ = std::fs::remove_file(&self.path);
}
}
/// Creates the socket's parent directory.
///
/// A group-shared socket must sit in a directory the group can traverse, so it
/// is `0755`; a private socket keeps the owner-only state-directory default.
fn create_parent(parent: &Path, access: &ControlSocketAccess) -> Result<()> {
match access {
ControlSocketAccess::Private => crate::storage::create_dir(parent),
ControlSocketAccess::Group(_) => {
std::fs::create_dir_all(parent).map_err(|source| Error::Io {
path: parent.to_path_buf(),
source,
})?;
set_mode(parent, 0o755)
}
}
}
/// Applies the access policy to the bound socket.
///
/// Private is owner-only `0600`. Group sets the socket's group to the gid and
/// the mode to `0660`; if the group cannot be set — the agent is not a member,
/// say — it falls back to owner-only rather than leaving the group wrong, so
/// the socket is never more open than intended.
fn apply_access(path: &Path, access: &ControlSocketAccess) -> Result<()> {
match access {
ControlSocketAccess::Private => set_mode(path, 0o600),
ControlSocketAccess::Group(gid) => {
if let Err(source) = std::os::unix::fs::chown(path, None, Some(*gid)) {
tracing::warn!(
path = %path.display(), gid, %source,
"cannot set the control socket group; keeping it owner-only"
);
return set_mode(path, 0o600);
}
set_mode(path, 0o660)
}
}
}
fn set_mode(path: &Path, mode: u32) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).map_err(|source| {
Error::Io {
path: path.to_path_buf(),
source,
}
})
}
async fn serve(listener: UnixListener, source: Arc<dyn ReportSource>) {
loop {
let Ok((stream, _)) = listener.accept().await else {
continue;
};
let source = Arc::clone(&source);
tokio::spawn(async move {
if let Err(err) = super::serve_connection(stream, source).await {
tracing::debug!(%err, "local control request failed");
}
});
}
}
/// Connects a client to the socket at `path`.
pub(crate) async fn connect(path: &Path) -> Result<UnixStream> {
UnixStream::connect(path).await.map_err(|source| Error::Io {
path: path.to_path_buf(),
source,
})
}
/// Whether an agent is listening on the socket at `path`.
pub(crate) async fn probe(path: &Path) -> bool {
UnixStream::connect(path).await.is_ok()
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::time::Duration;
use super::super::{Request, Response, StatusReport, exchange};
use super::*;
use crate::BoxFuture;
#[tokio::test]
async fn a_silent_agent_is_reported_rather_than_waited_out() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("control.sock");
// A socket that accepts and then says nothing, like a wedged agent:
// the connection succeeds, the answer never comes. Unbounded, this
// call would never return.
let _listener = UnixListener::bind(&path).unwrap();
let error = exchange(&path, &Request::Status, Duration::from_millis(100))
.await
.expect_err("a silent agent cannot be reported as healthy");
match error {
Error::Timeout { what } => assert!(
what.contains("control.sock"),
"the message names what did not answer: {what}"
),
other => panic!("expected a timeout, got {other:?}"),
}
}
#[tokio::test]
async fn an_answered_request_is_not_affected_by_the_bound() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("control.sock");
let source: Arc<dyn ReportSource> = Arc::new(|| -> BoxFuture<'static, StatusReport> {
Box::pin(async { StatusReport::default() })
});
let control = ControlSocket::bind(&path, source, ControlSocketAccess::Private)
.await
.unwrap();
let answer = exchange(&path, &Request::Status, EXCHANGE_TIMEOUT)
.await
.unwrap();
assert!(matches!(answer, Response::Status(_)));
control.shutdown().await;
}
}
+453
View File
@@ -0,0 +1,453 @@
//! A named-pipe adapter for the local control interface.
//!
//! The Windows counterpart of [`super::unix`]. It provides only what is
//! particular to a named pipe — creating the server instance, opening a client
//! and turning the state directory path into a pipe name — and hands every
//! accepted connection to the shared [`serve_connection`](super::serve_connection).
//! The framing, the dispatch and the client wrappers are the same on every
//! platform and live in [`super`], re-exported here so `ipc::windows::request_status`
//! and its siblings resolve just as their Unix equivalents do.
//!
//! # From a path to a pipe
//!
//! The rest of the agent addresses the control interface by a filesystem path,
//! the same one a Unix socket would live at. A named pipe has no filesystem
//! path, so the path is hashed into a stable name under `\\.\pipe\`. The agent
//! and a client derive it the same way from the same path, so neither has to
//! be told where the other put it.
//!
//! # Who may connect
//!
//! The pipe rejects clients from other machines (`reject_remote_clients(true)`).
//! An explicit SDDL security descriptor grants Full Control to Administrators
//! and LocalSystem, while granting Read and Write (`GRGW`) to all local users
//! with a Medium Integrity SACL label (`S:(ML;;NW;;;ME)`). This allows unprivileged
//! user-space tools (such as tray applications, status monitors, or CLI commands
//! launched by regular users) to connect to the agent even when the agent was
//! started by an elevated Administrator or as a system service.
//! A permission denial is reported as such, never as an absent agent.
#![allow(unsafe_code)]
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use sha2::{Digest, Sha256};
use tokio::net::windows::named_pipe::{ClientOptions, NamedPipeClient, ServerOptions};
use tokio::task::JoinHandle;
use crate::error::{Error, Result};
pub use super::{
CONTROL_PROTOCOL, ControlSocketAccess, EXCHANGE_TIMEOUT, ReportSource, is_serving,
join_network, leave_network, request_status, set_active, set_dns, set_hostname,
};
/// `ERROR_ACCESS_DENIED`: creating a first instance finds an existing pipe,
/// or a client cannot access an existing pipe with its current privileges.
const ERROR_ACCESS_DENIED: i32 = 5;
/// `ERROR_PIPE_BUSY`: every instance is serving a client right now. A server
/// is there; the client only has to wait for a free instance.
const ERROR_PIPE_BUSY: i32 = 231;
/// RAII wrapper for Windows named pipe security attributes with permissive local ACL.
///
/// SDDL specification:
/// - `D:(A;;GRGW;;;WD)`: DACL allows Generic Read and Generic Write to Everyone (`WD`).
/// Full control (`GA`) remains reserved to Administrators and LocalSystem.
/// - `(A;;GA;;;BA)`: DACL allows Generic All to Builtin Administrators (`BA`).
/// - `(A;;GA;;;SY)`: DACL allows Generic All to LocalSystem (`SY`).
/// - `S:(ML;;NW;;;ME)`: SACL Mandatory Label with Medium Integrity (`ME`) and No Write Up (`NW`).
/// This allows unprivileged user-space processes (Medium Integrity) to write to the
/// named pipe even when the agent runs as an elevated Administrator (High Integrity).
struct PipeSecurityAttributes {
attrs: windows_sys::Win32::Security::SECURITY_ATTRIBUTES,
}
impl std::fmt::Debug for PipeSecurityAttributes {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PipeSecurityAttributes")
.field("nLength", &self.attrs.nLength)
.field("lpSecurityDescriptor", &self.attrs.lpSecurityDescriptor)
.field("bInheritHandle", &self.attrs.bInheritHandle)
.finish()
}
}
impl PipeSecurityAttributes {
fn new() -> Result<Self> {
use std::ffi::OsStr;
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Security::Authorization::{
ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1,
};
const SDDL: &str = "D:(A;;GRGW;;;WD)(A;;GA;;;BA)(A;;GA;;;SY)S:(ML;;NW;;;ME)";
let wide: Vec<u16> = OsStr::new(SDDL)
.encode_wide()
.chain(std::iter::once(0))
.collect();
let mut p_sd = std::ptr::null_mut();
// SAFETY: We pass a valid null-terminated wide string, standard revision 1,
// and a valid pointer to receive the descriptor.
let success = unsafe {
ConvertStringSecurityDescriptorToSecurityDescriptorW(
wide.as_ptr(),
SDDL_REVISION_1,
&mut p_sd,
std::ptr::null_mut(),
)
};
if success == 0 || p_sd.is_null() {
let err = std::io::Error::last_os_error();
return Err(Error::Storage(format!(
"cannot create pipe security descriptor: {err}"
)));
}
let attrs = windows_sys::Win32::Security::SECURITY_ATTRIBUTES {
nLength: std::mem::size_of::<windows_sys::Win32::Security::SECURITY_ATTRIBUTES>()
as u32,
lpSecurityDescriptor: p_sd,
bInheritHandle: 0,
};
Ok(Self { attrs })
}
fn as_raw_mut(&mut self) -> *mut std::ffi::c_void {
&mut self.attrs as *mut _ as *mut std::ffi::c_void
}
}
impl Drop for PipeSecurityAttributes {
fn drop(&mut self) {
if !self.attrs.lpSecurityDescriptor.is_null() {
// SAFETY: The descriptor was allocated by ConvertStringSecurityDescriptorToSecurityDescriptorW
// and must be freed with LocalFree.
unsafe {
windows_sys::Win32::Foundation::LocalFree(self.attrs.lpSecurityDescriptor);
}
self.attrs.lpSecurityDescriptor = std::ptr::null_mut();
}
}
}
/// Serves the local control interface on a named pipe.
#[derive(Debug)]
pub struct ControlSocket {
path: PathBuf,
task: Option<JoinHandle<()>>,
}
impl ControlSocket {
/// Binds the pipe and starts serving.
///
/// Claiming the first instance of the name is what detects a second agent:
/// if the name already exists, the create is refused and that is reported
/// as the state being locked, so two live agents never share one pipe.
///
/// `_access` is accepted for parity with the Unix socket; granting a group
/// here means a pipe security descriptor and is a separate follow-up, so for
/// now the pipe keeps its default (creator plus the built-in access the
/// `PipeSecurityAttributes` set).
pub async fn bind(
path: impl AsRef<Path>,
source: Arc<dyn ReportSource>,
_access: ControlSocketAccess,
) -> Result<Self> {
let path = path.as_ref().to_path_buf();
let name = pipe_name(&path);
let mut sec_attrs = PipeSecurityAttributes::new()?;
// SAFETY: sec_attrs points to valid initialized SECURITY_ATTRIBUTES.
let server = match unsafe {
ServerOptions::new()
.first_pipe_instance(true)
.reject_remote_clients(true)
.create_with_security_attributes_raw(&name, sec_attrs.as_raw_mut())
} {
Ok(server) => server,
Err(err) if err.raw_os_error() == Some(ERROR_ACCESS_DENIED) => {
return Err(Error::StateLocked { path });
}
Err(source) => return Err(Error::Io { path, source }),
};
let task = tokio::spawn(serve(name, server, source));
Ok(Self {
path,
task: Some(task),
})
}
/// The path the pipe name was derived from.
pub fn path(&self) -> &Path {
&self.path
}
/// Stops serving. The pipe goes with the server, so there is nothing to
/// remove.
pub async fn shutdown(mut self) {
if let Some(task) = self.task.take() {
task.abort();
let _ = task.await;
}
}
}
impl Drop for ControlSocket {
fn drop(&mut self) {
if let Some(task) = self.task.take() {
task.abort();
}
}
}
/// Accepts one client at a time, keeping a fresh instance ready for the next.
///
/// A named pipe server instance serves a single client, so a new instance is
/// created as soon as one is taken — otherwise a second `tsng status` while
/// the first is mid-flight would find nothing listening.
async fn serve(
name: String,
first: tokio::net::windows::named_pipe::NamedPipeServer,
source: Arc<dyn ReportSource>,
) {
let mut server = first;
loop {
if server.connect().await.is_err() {
match next_instance(&name) {
Some(next) => {
server = next;
continue;
}
None => return,
}
}
let connected = server;
match next_instance(&name) {
Some(next) => server = next,
None => {
// Nothing left to accept the next client on, but the one in
// hand is still answered before the loop ends.
let source = Arc::clone(&source);
tokio::spawn(async move {
let _ = super::serve_connection(connected, source).await;
});
return;
}
}
let source = Arc::clone(&source);
tokio::spawn(async move {
if let Err(err) = super::serve_connection(connected, source).await {
tracing::debug!(%err, "local control request failed");
}
});
}
}
/// Creates the next pipe instance, or `None` if the name can no longer be
/// served.
fn next_instance(name: &str) -> Option<tokio::net::windows::named_pipe::NamedPipeServer> {
let mut sec_attrs = match PipeSecurityAttributes::new() {
Ok(attrs) => attrs,
Err(err) => {
tracing::debug!(%err, "cannot build pipe security attributes for next instance");
return None;
}
};
// SAFETY: sec_attrs points to valid initialized SECURITY_ATTRIBUTES.
match unsafe {
ServerOptions::new()
.reject_remote_clients(true)
.create_with_security_attributes_raw(name, sec_attrs.as_raw_mut())
} {
Ok(server) => Some(server),
Err(err) => {
tracing::debug!(%err, "cannot create the next control pipe instance");
None
}
}
}
/// Connects a client to the pipe for `path`, waiting out a busy pipe.
///
/// The wait is bounded by the caller's exchange timeout, so a pipe that is
/// busy forever is given up on rather than spun on.
pub(crate) async fn connect(path: &Path) -> Result<NamedPipeClient> {
let name = pipe_name(path);
loop {
match ClientOptions::new().open(&name) {
Ok(client) => return Ok(client),
Err(err) if err.raw_os_error() == Some(ERROR_PIPE_BUSY) => {
tokio::time::sleep(Duration::from_millis(50)).await;
}
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
return Err(Error::Io {
path: path.to_path_buf(),
source: std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
format!(
"the agent's control pipe exists, but Windows denied access. \
Run this command as the same Windows user and with the same \
elevation as `tsng up` (use an administrator terminal if \
the agent is elevated). Windows error: {err}"
),
),
});
}
Err(source) => {
return Err(Error::Io {
path: path.to_path_buf(),
source,
});
}
}
}
}
/// Whether an agent is serving the pipe for `path`.
///
/// A busy or inaccessible pipe still means a server is there. In particular,
/// access denied must not send a mutating command down the offline path,
/// where it would only produce a misleading state-directory lock error.
pub(crate) async fn probe(path: &Path) -> bool {
let name = pipe_name(path);
match ClientOptions::new().open(&name) {
Ok(_) => true,
Err(err) => {
err.raw_os_error() == Some(ERROR_PIPE_BUSY)
|| err.kind() == std::io::ErrorKind::PermissionDenied
}
}
}
/// The pipe name for a control-socket path.
///
/// A hash rather than the path itself, because a pipe name may not contain the
/// separators and drive letters a path does, and because two agents with
/// different state directories must never collide.
fn pipe_name(path: &Path) -> String {
let digest = Sha256::digest(path.as_os_str().as_encoded_bytes());
format!(r"\\.\pipe\tsunagi-{}", hex::encode(&digest[..16]))
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::time::Duration;
use super::super::{Request, Response, StatusReport, exchange};
use super::*;
use crate::BoxFuture;
#[test]
fn one_path_maps_to_one_name_and_two_paths_do_not_collide() {
let a = pipe_name(Path::new(r"C:\a\agent.sock"));
let b = pipe_name(Path::new(r"C:\b\agent.sock"));
assert!(a.starts_with(r"\\.\pipe\tsunagi-"), "{a}");
assert_eq!(a, pipe_name(Path::new(r"C:\a\agent.sock")));
assert_ne!(a, b);
}
#[tokio::test]
async fn a_request_makes_the_round_trip_over_a_real_pipe() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("agent.sock");
let source: Arc<dyn ReportSource> = Arc::new(|| -> BoxFuture<'static, StatusReport> {
Box::pin(async { StatusReport::default() })
});
assert!(!is_serving(&path).await, "nothing serves the pipe yet");
let control = ControlSocket::bind(&path, source).await.unwrap();
assert!(is_serving(&path).await, "the agent serves it now");
let answer = exchange(&path, &Request::Status, EXCHANGE_TIMEOUT)
.await
.unwrap();
assert!(matches!(answer, Response::Status(_)));
control.shutdown().await;
}
#[tokio::test]
async fn an_inaccessible_agent_is_not_mistaken_for_an_absent_one() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("agent.sock");
// An outbound-only pipe denies the client's duplex open even when
// both processes have the same privileges. This exercises the same
// Windows error as a client unable to access an elevated agent,
// without requiring elevation or changing any ACLs.
let _server = ServerOptions::new()
.first_pipe_instance(true)
.access_inbound(false)
.create(pipe_name(&path))
.unwrap();
assert!(is_serving(&path).await, "access denied is not absence");
for error in [
request_status(&path).await.unwrap_err(),
join_network(&path, "test", "local-ipc-test-secret")
.await
.unwrap_err(),
set_dns(&path, false, None).await.unwrap_err(),
] {
let Error::Io { source, .. } = error else {
panic!("expected an access error, got {error}");
};
assert_eq!(source.kind(), std::io::ErrorKind::PermissionDenied);
let message = source.to_string();
assert!(message.contains("same Windows user"), "{message}");
assert!(message.contains("administrator"), "{message}");
assert!(!message.contains("local-ipc-test-secret"), "{message}");
}
}
#[tokio::test]
async fn a_silent_agent_is_reported_rather_than_waited_out() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("agent.sock");
// A server instance that accepts and then never answers, like a
// wedged agent. Unbounded, the exchange below would never return.
let name = pipe_name(&path);
let server = ServerOptions::new()
.first_pipe_instance(true)
.create(&name)
.unwrap();
let _accept = tokio::spawn(async move {
let _ = server.connect().await;
// Hold the connection open, answering nothing.
tokio::time::sleep(Duration::from_secs(30)).await;
});
let error = exchange(&path, &Request::Status, Duration::from_millis(200))
.await
.expect_err("a silent agent cannot be reported as healthy");
assert!(matches!(error, Error::Timeout { .. }), "{error:?}");
}
#[tokio::test]
async fn a_second_agent_on_the_same_pipe_is_refused() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("agent.sock");
let source: Arc<dyn ReportSource> = Arc::new(|| -> BoxFuture<'static, StatusReport> {
Box::pin(async { StatusReport::default() })
});
let first = ControlSocket::bind(&path, Arc::clone(&source))
.await
.unwrap();
let second = ControlSocket::bind(&path, source).await;
assert!(
matches!(second, Err(Error::StateLocked { .. })),
"{second:?}"
);
first.shutdown().await;
}
}
+120
View File
@@ -0,0 +1,120 @@
//! Tsunagi: a proof-of-concept agent for small private mesh networks.
//!
//! See `README.md` for the exact scope of this proof of concept, and
//! `docs/architecture.md` / `docs/protocol.md` for the design.
//!
//! # Shape of the library
//!
//! * [`identity`] — persistent device identity and deterministic network space
//! identity.
//! * [`storage`] — mandatory state (`state.sqlite`) and separately disposable
//! cache (`cache.sqlite`).
//! * [`discovery`] — pluggable sources of *candidate* addresses. Candidates are
//! never trusted peers.
//! * [`proto`] — the control protocol: framing, messages, handshake.
//! * [`net`] — the iroh connectivity adapter and its observability surface.
//! * [`agent`] — the runtime: agent lifecycle, per-network runtimes, reconnect.
//! * [`dataplane`] — the contract IP plugins satisfy, the packet transport,
//! and the WireGuard data plane.
//! * [`state`] — signed records that outlive a session, and the rules for
//! merging them between replicas.
//! * [`ipc`] — the local control interface a command line tool talks to. An
//! adapter over the public API; the core does not know it exists.
//!
//! # What this library deliberately does not do
//!
//! It never starts a global tokio runtime, never installs a global tracing
//! subscriber, never handles process signals, never forks and never calls
//! `process::exit`. Several independent agents can run in one process.
//!
//! Only control messages travel over iroh. User IP traffic is not tunnelled
//! through it.
#![deny(rustdoc::broken_intra_doc_links)]
/// A boxed future, used where a trait must stay object safe.
pub type BoxFuture<'a, T> = std::pin::Pin<Box<dyn std::future::Future<Output = T> + Send + 'a>>;
pub mod agent;
pub mod config;
pub mod dataplane;
pub mod discovery;
pub mod dns;
pub mod error;
pub mod identity;
pub mod ipc;
pub mod net;
pub mod overlay;
pub mod proto;
pub mod state;
pub mod storage;
pub mod task;
#[cfg(feature = "testing")]
pub mod testing;
pub use agent::{Agent, AgentStatus, Event, NetworkStatus, PeerStatus};
pub use config::{AgentConfig, Limits, ReconnectPolicy, StoragePaths, TransportPolicy};
pub use error::{Error, ProtocolError, Result};
pub use identity::{
DeviceIdentity, DiscoveryKey, NetworkDescriptor, NetworkId, NetworkName, NetworkSecret,
};
/// Re-exported iroh types that appear in this crate's public API.
pub mod iroh_types {
pub use iroh::{EndpointAddr, EndpointId, RelayUrl};
}
#[doc(hidden)]
pub mod test_support {
//! Internals exposed for this crate's own negative tests.
//!
//! **Not part of the stable API.** It exists so the integration tests can
//! hand-craft handshakes — a valid proof to replay on another connection, a
//! message sent before authentication, an oversized frame — which is the
//! only way to test those rejections against a real agent.
use iroh::endpoint::Connection;
use crate::error::ProtocolError;
use crate::identity::NetworkKeys;
use crate::proto::handshake;
/// Exposes the derived handshake authentication key.
pub fn auth_key(keys: &NetworkKeys) -> [u8; 32] {
*keys.auth_key()
}
/// Derives this connection's channel binding material.
pub fn channel_binding(
conn: &Connection,
network_id: &[u8; 32],
) -> Result<[u8; 32], ProtocolError> {
handshake::channel_binding_for_test(conn, network_id)
}
/// Computes a handshake proof for the given role.
#[allow(clippy::too_many_arguments)]
pub fn compute_proof(
auth_key: &[u8; 32],
role: &str,
version: u16,
network_id: &[u8; 32],
initiator: &[u8; 32],
responder: &[u8; 32],
channel_binding: &[u8],
nonce_initiator: &[u8; 16],
nonce_responder: &[u8; 16],
) -> [u8; 32] {
handshake::proof_for_test(
auth_key,
role,
version,
network_id,
initiator,
responder,
channel_binding,
nonce_initiator,
nonce_responder,
)
}
}
+470
View File
@@ -0,0 +1,470 @@
//! The iroh connectivity adapter and its observability surface.
//!
//! This module builds and owns the [`Endpoint`], and turns iroh's runtime
//! information into plain owned snapshots that the rest of the library and the
//! library's users can read.
//!
//! # Honest reporting
//!
//! Three different things are kept distinct and never conflated:
//!
//! * an **unverified candidate** — something discovery handed us
//! ([`crate::discovery::Candidate`]);
//! * an **observed address** — an address this endpoint believes it has
//! ([`EndpointSnapshot::observed_addrs`]);
//! * a **verified path** — a network path QUIC has actually validated and is
//! using or can use ([`PathInfo`]).
//!
//! A value that is not available is reported as `None`. It is never invented.
//!
//! An iroh address is an address for *iroh*. It must not be assumed to be usable
//! by any other protocol; a future WireGuard plugin is expected to collect its
//! own reachability data.
use std::net::SocketAddr;
use std::time::Duration;
use iroh::endpoint::{
Connection, ConnectionStats, PortmapperConfig, RecvStream, SendStream, presets,
};
use iroh::{Endpoint, EndpointAddr, EndpointId, RelayMode};
use crate::config::{AgentConfig, TransportPolicy};
use crate::error::{Error, Result};
use crate::identity::DeviceIdentity;
use crate::proto::message::{ALPN, DATA_ALPN};
/// A network path address as reported by iroh.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PathAddr {
/// A direct IP path.
Ip(SocketAddr),
/// A path through a relay server.
Relay(String),
/// A custom transport iroh reported but this crate does not model.
Other(String),
}
impl std::fmt::Display for PathAddr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PathAddr::Ip(addr) => write!(f, "{addr}"),
PathAddr::Relay(url) => write!(f, "relay {url}"),
PathAddr::Other(what) => write!(f, "{what}"),
}
}
}
/// One verified network path of a connection.
#[derive(Debug, Clone)]
pub struct PathInfo {
/// Remote address of the path.
pub remote: PathAddr,
/// Local address of the path, when the OS reports one.
pub local: Option<String>,
/// Whether QUIC currently transmits application data over this path.
pub is_selected: bool,
/// Round-trip time estimate for this path.
pub rtt: Duration,
}
impl PathInfo {
/// Whether this is a direct IP path.
pub fn is_direct(&self) -> bool {
matches!(self.remote, PathAddr::Ip(_))
}
/// Whether this path goes through a relay.
pub fn is_relay(&self) -> bool {
matches!(self.remote, PathAddr::Relay(_))
}
}
/// How a connection currently reaches its peer.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TransportKind {
/// A direct IP path is selected.
Direct,
/// A relay path is selected.
Relay,
/// iroh has not reported a selected path yet.
Unknown,
}
impl std::fmt::Display for TransportKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let word = match self {
TransportKind::Direct => "direct",
TransportKind::Relay => "relay",
TransportKind::Unknown => "unknown",
};
f.write_str(word)
}
}
/// Counters for one connection.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ConnectionCounters {
/// UDP bytes sent on this connection.
pub udp_tx_bytes: u64,
/// UDP bytes received on this connection.
pub udp_rx_bytes: u64,
/// UDP datagrams sent.
pub udp_tx_datagrams: u64,
/// UDP datagrams received.
pub udp_rx_datagrams: u64,
/// Packets declared lost.
pub lost_packets: u64,
}
impl From<ConnectionStats> for ConnectionCounters {
fn from(stats: ConnectionStats) -> Self {
Self {
udp_tx_bytes: stats.udp_tx.bytes,
udp_rx_bytes: stats.udp_rx.bytes,
udp_tx_datagrams: stats.udp_tx.datagrams,
udp_rx_datagrams: stats.udp_rx.datagrams,
lost_packets: stats.lost_packets,
}
}
}
/// An owned snapshot of one live connection.
#[derive(Debug, Clone)]
pub struct ConnectionSnapshot {
/// Authenticated endpoint id of the remote side.
pub remote_id: EndpointId,
/// Verified paths, as reported by iroh at snapshot time.
pub paths: Vec<PathInfo>,
/// How the connection currently reaches the peer.
pub transport: TransportKind,
/// RTT of the selected path, when there is one.
pub rtt: Option<Duration>,
/// Per-connection counters.
pub counters: ConnectionCounters,
}
/// Snapshot of this endpoint, independent of any particular network.
#[derive(Debug, Clone)]
pub struct EndpointSnapshot {
/// This endpoint's id.
pub endpoint_id: EndpointId,
/// Sockets actually bound locally.
pub bound_sockets: Vec<SocketAddr>,
/// Addresses iroh believes this endpoint is reachable at.
///
/// These are *observed*, not verified by any remote peer.
pub observed_addrs: Vec<PathAddr>,
/// Relay URLs this endpoint currently considers usable, if any.
pub relay_urls: Vec<String>,
}
fn path_addr(addr: &iroh::TransportAddr) -> PathAddr {
match addr {
iroh::TransportAddr::Ip(socket) => PathAddr::Ip(*socket),
iroh::TransportAddr::Relay(url) => PathAddr::Relay(url.to_string()),
other => PathAddr::Other(format!("{other:?}")),
}
}
/// Builds an owned snapshot of a live connection.
pub fn snapshot_connection(conn: &Connection) -> ConnectionSnapshot {
let mut paths = Vec::new();
let mut transport = TransportKind::Unknown;
let mut rtt = None;
for path in conn.paths().iter() {
let info = PathInfo {
remote: path_addr(path.remote_addr()),
local: local_addr_string(path.local_addr()),
is_selected: path.is_selected(),
rtt: path.rtt(),
};
if info.is_selected {
transport = if info.is_relay() {
TransportKind::Relay
} else if info.is_direct() {
TransportKind::Direct
} else {
TransportKind::Unknown
};
rtt = Some(info.rtt);
}
paths.push(info);
}
ConnectionSnapshot {
remote_id: conn.remote_id(),
paths,
transport,
rtt,
counters: conn.stats().into(),
}
}
fn local_addr_string(addr: &iroh::endpoint::LocalTransportAddr) -> Option<String> {
match addr {
iroh::endpoint::LocalTransportAddr::Ip(Some(ip)) => Some(ip.to_string()),
iroh::endpoint::LocalTransportAddr::Ip(None) => None,
iroh::endpoint::LocalTransportAddr::Relay(url) => Some(url.to_string()),
iroh::endpoint::LocalTransportAddr::Custom(Some(custom)) => Some(format!("{custom:?}")),
iroh::endpoint::LocalTransportAddr::Custom(None) => None,
_ => None,
}
}
/// Thin wrapper around the iroh endpoint.
///
/// The endpoint serves two ALPNs: the control protocol and the data plane.
/// They are separate connections with separate congestion control, so a busy
/// or broken data plane cannot disturb control traffic.
#[derive(Debug, Clone)]
pub struct EndpointAdapter {
endpoint: Endpoint,
}
impl EndpointAdapter {
/// Binds an endpoint according to `config`, reusing the persistent device key.
pub async fn bind(config: &AgentConfig, identity: &DeviceIdentity) -> Result<Self> {
let mut builder = Endpoint::builder(presets::Minimal)
.secret_key(identity.secret_key())
.alpns(vec![ALPN.to_vec(), DATA_ALPN.to_vec()]);
builder = match config.transport {
TransportPolicy::LocalOnly => builder
.relay_mode(RelayMode::Disabled)
.clear_address_lookup()
.portmapper_config(PortmapperConfig::Disabled)
.net_report_config(iroh::NetReportConfig::minimal()),
TransportPolicy::DirectOnly => builder.preset(presets::N0DisableRelay),
TransportPolicy::N0Defaults => builder.preset(presets::N0),
};
if matches!(
config.transport,
TransportPolicy::N0Defaults | TransportPolicy::DirectOnly
) {
let filter = iroh::address_lookup::AddrFilter::new(|addrs| {
let mut overlay_ips = std::collections::HashSet::new();
for iface in netdev::get_interfaces() {
let name = iface.name.to_lowercase();
let friendly = iface
.friendly_name
.as_deref()
.map(str::to_lowercase)
.unwrap_or_default();
let desc = iface
.description
.as_deref()
.map(str::to_lowercase)
.unwrap_or_default();
if name.starts_with("tsun")
|| name.contains("tsunagi")
|| name.contains("wintun")
|| friendly.starts_with("tsun")
|| friendly.contains("tsunagi")
|| friendly.contains("wintun")
|| desc.contains("wintun")
|| desc.contains("tsunagi")
{
for ip in iface.ipv4 {
overlay_ips.insert(std::net::IpAddr::V4(ip.addr()));
}
for ip in iface.ipv6 {
overlay_ips.insert(std::net::IpAddr::V6(ip.addr()));
}
}
}
if overlay_ips.is_empty() {
return std::borrow::Cow::Borrowed(addrs);
}
let filtered: Vec<iroh::TransportAddr> = addrs
.iter()
.filter(|addr| match addr {
iroh::TransportAddr::Ip(sock) => !overlay_ips.contains(&sock.ip()),
_ => true,
})
.cloned()
.collect();
std::borrow::Cow::Owned(filtered)
});
builder = builder.addr_filter(filter);
}
#[cfg(feature = "testing")]
if let Some(transport) = &config.test_quic_transport {
builder = builder.transport_config(transport.clone());
}
if !config.bind_addrs.is_empty() {
builder = builder.clear_ip_transports();
for addr in &config.bind_addrs {
builder = builder
.bind_addr(*addr)
.map_err(|err| Error::Endpoint(format!("invalid bind address: {err}")))?;
}
}
let endpoint = builder
.bind()
.await
.map_err(|err| Error::Endpoint(format!("cannot bind endpoint: {err}")))?;
tracing::debug!(
endpoint = %endpoint.id().fmt_short(),
bound_sockets = ?endpoint.bound_sockets(),
"iroh endpoint bound locally"
);
let ep_id = endpoint.id();
let mut addr_watcher = endpoint.watch_addr();
let mut relay_watcher = endpoint.home_relay_status();
let close_token = endpoint.clone();
tokio::spawn(async move {
use iroh::Watcher;
let mut last_addrs: Vec<SocketAddr> = Vec::new();
let mut last_relays: Vec<String> = Vec::new();
let warn_timer = tokio::time::sleep(Duration::from_secs(12));
tokio::pin!(warn_timer);
let mut warned = false;
loop {
tokio::select! {
biased;
_ = close_token.closed() => {
break;
}
() = &mut warn_timer, if !warned => {
warned = true;
if last_addrs.is_empty() && !last_relays.is_empty() {
tracing::warn!(
endpoint = %ep_id.fmt_short(),
relays = ?last_relays,
"No direct reflexive or UPnP addresses discovered after probe; communications will rely on relay servers"
);
} else if last_addrs.is_empty() && last_relays.is_empty() {
tracing::warn!(
endpoint = %ep_id.fmt_short(),
"No reachability discovered (neither direct reflexive/UPnP addresses nor relay servers available)"
);
}
}
res = addr_watcher.updated() => {
let addr = match res {
Ok(a) => a,
Err(_) => break,
};
let addrs: Vec<SocketAddr> = addr.ip_addrs().copied().collect();
let relays: Vec<String> = addr.relay_urls().map(|u| u.to_string()).collect();
if addrs != last_addrs || relays != last_relays {
tracing::debug!(
endpoint = %ep_id.fmt_short(),
direct_addrs = ?addrs,
relays = ?relays,
"Discovered endpoint reachability (STUN/UPnP/local/relay)"
);
last_addrs = addrs;
last_relays = relays;
}
}
res = relay_watcher.updated() => {
let relays = match res {
Ok(r) => r,
Err(_) => break,
};
for relay in &relays {
if relay.is_connected() {
tracing::debug!(
endpoint = %ep_id.fmt_short(),
relay = %relay.url(),
"Connected to home relay server"
);
} else if let Some(err) = relay.last_error() {
tracing::debug!(
endpoint = %ep_id.fmt_short(),
relay = %relay.url(),
error = %err,
"Relay connection attempt failed"
);
}
}
}
}
}
});
Ok(Self { endpoint })
}
/// The underlying iroh endpoint.
pub fn endpoint(&self) -> &Endpoint {
&self.endpoint
}
/// This endpoint's id.
pub fn endpoint_id(&self) -> EndpointId {
self.endpoint.id()
}
/// This endpoint's dialable address, as currently known.
pub fn addr(&self) -> EndpointAddr {
self.endpoint.addr()
}
/// Builds an address containing only the locally bound sockets.
///
/// Useful when address lookup and relays are disabled and peers must be
/// given literal addresses.
pub fn loopback_addr(&self) -> EndpointAddr {
self.endpoint
.bound_sockets()
.into_iter()
.fold(EndpointAddr::new(self.endpoint.id()), |addr, socket| {
addr.with_ip_addr(socket)
})
}
/// Snapshot of endpoint-level information.
pub fn snapshot(&self) -> EndpointSnapshot {
let addr = self.endpoint.addr();
let mut observed = Vec::new();
let mut relays = Vec::new();
for socket in addr.ip_addrs() {
observed.push(PathAddr::Ip(*socket));
}
for url in addr.relay_urls() {
relays.push(url.to_string());
observed.push(PathAddr::Relay(url.to_string()));
}
EndpointSnapshot {
endpoint_id: self.endpoint.id(),
bound_sockets: self.endpoint.bound_sockets(),
observed_addrs: observed,
relay_urls: relays,
}
}
/// Dials a candidate and opens the control stream.
pub async fn connect(
&self,
addr: EndpointAddr,
) -> Result<(Connection, SendStream, RecvStream), Error> {
let conn = self
.endpoint
.connect(addr, ALPN)
.await
.map_err(|err| Error::Endpoint(format!("connect failed: {err}")))?;
let (send, recv) = conn
.open_bi()
.await
.map_err(|err| Error::Endpoint(format!("cannot open control stream: {err}")))?;
Ok((conn, send, recv))
}
/// Closes the endpoint and waits for it to finish.
pub async fn close(&self) {
self.endpoint.close().await;
}
}
@@ -0,0 +1,231 @@
//! Scoped LAN discovery fanout at IP ingress. A domain is one logical network;
//! routed copies use ordinary encrypted peer links and are never reflooded.
//! Future subnet exporters must supply explicitly authorized ingress domains
//! here, not teach the encrypted transit router to inspect application bytes.
use super::router::{NetworkRoutes, Route};
use crate::NetworkId;
use iroh::EndpointId;
use std::collections::{HashMap, HashSet};
use std::net::Ipv4Addr;
use std::sync::Arc;
/// Local participation and the currently authenticated participants in a domain.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BroadcastPolicy {
/// Whether this agent originates and accepts LAN broadcasts in this network.
pub enabled: bool,
/// Live members explicitly advertising that they accept broadcasts.
pub peers: HashSet<EndpointId>,
}
impl Default for BroadcastPolicy {
fn default() -> Self {
Self {
enabled: true,
peers: HashSet::new(),
}
}
}
#[derive(Debug)]
struct Domain {
network: NetworkId,
directed: Ipv4Addr,
enabled: bool,
recipients: Arc<[Route]>,
members: HashSet<EndpointId>,
}
/// Immutable ingress-domain index rebuilt with address/policy changes.
#[derive(Debug, Default)]
pub(crate) struct BroadcastTable {
sources: HashMap<Ipv4Addr, Arc<Domain>>,
networks: HashMap<NetworkId, Arc<Domain>>,
destinations: HashSet<Ipv4Addr>,
}
impl BroadcastTable {
pub fn build(networks: &HashMap<NetworkId, NetworkRoutes>) -> Self {
let mut table = Self::default();
for (&network, routes) in networks {
let (Some(range), Some(local)) = (routes.range, routes.local) else {
continue;
};
if range.prefix_len > 30 {
continue;
}
let directed = Ipv4Addr::from(u32::from(range.base) | (u32::MAX >> range.prefix_len));
let mut recipients: Vec<_> = routes
.peers
.iter()
.filter(|(_, peer)| routes.broadcast.peers.contains(peer))
.map(|(_, peer)| Route {
network,
peer: *peer,
})
.collect();
recipients.sort_by_key(|route| route.peer);
recipients.dedup();
let domain = Arc::new(Domain {
network,
directed,
enabled: routes.broadcast.enabled,
members: recipients.iter().map(|route| route.peer).collect(),
recipients: recipients.into(),
});
table.sources.insert(local, domain.clone());
table.networks.insert(network, domain);
table.destinations.insert(directed);
}
table
}
pub fn is_destination(&self, destination: Ipv4Addr) -> bool {
destination.is_broadcast() || self.destinations.contains(&destination)
}
pub fn outgoing(&self, source: Ipv4Addr, destination: Ipv4Addr) -> Option<Arc<[Route]>> {
let domain = self.sources.get(&source)?;
(domain.enabled && (destination.is_broadcast() || destination == domain.directed))
.then(|| domain.recipients.clone())
}
pub fn accepts(&self, network: NetworkId, peer: EndpointId, destination: Ipv4Addr) -> bool {
self.networks.get(&network).is_some_and(|domain| {
domain.network == network
&& domain.enabled
&& domain.members.contains(&peer)
&& (destination.is_broadcast() || destination == domain.directed)
})
}
}
/// Validates IPv4/UDP lengths before broadcast fanout. No game-specific ports,
/// payload rewriting or checksum changes. IP fragments retain their bytes and
/// are reassembled by the destination OS; even later fragments have protocol 17.
pub(crate) fn valid_udp(packet: &[u8]) -> bool {
if packet.len() < 20 || packet[0] >> 4 != 4 || packet[9] != 17 {
return false;
}
let header = usize::from(packet[0] & 15) * 4;
let total = usize::from(u16::from_be_bytes([packet[2], packet[3]]));
if header < 20 || total <= header || total > packet.len() {
return false;
}
let fragment = u16::from_be_bytes([packet[6], packet[7]]);
if fragment & 0x1fff != 0 {
return true;
}
if total < header + 8 {
return false;
}
let udp = usize::from(u16::from_be_bytes([packet[header + 4], packet[header + 5]]));
udp >= 8
&& if fragment & 0x2000 != 0 {
udp >= total - header
} else {
udp == total - header
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::identity::{NetworkKeys, NetworkName, NetworkSecret};
fn id(name: &str) -> NetworkId {
NetworkKeys::derive(
&NetworkName::new(name).unwrap(),
&NetworkSecret::from_bytes([4; 32]).unwrap(),
)
.network_id()
}
#[test]
fn limited_and_directed_broadcasts_never_cross_domains_or_target_disabled_peers() {
let peer = iroh::SecretKey::from_bytes(&[8; 32]).public();
let a = NetworkRoutes {
exit: Default::default(),
range: Some("10.0.0.0/24".parse().unwrap()),
local: Some("10.0.0.1".parse().unwrap()),
peers: vec![("10.0.0.2".parse().unwrap(), peer)],
broadcast: BroadcastPolicy {
enabled: true,
peers: HashSet::from([peer]),
},
};
let b = NetworkRoutes {
exit: Default::default(),
range: Some("10.1.0.0/24".parse().unwrap()),
local: Some("10.1.0.1".parse().unwrap()),
peers: vec![("10.1.0.2".parse().unwrap(), peer)],
broadcast: BroadcastPolicy::default(),
};
let table = BroadcastTable::build(&HashMap::from([(id("a"), a.clone()), (id("b"), b)]));
let routes = table
.outgoing(a.local.unwrap(), Ipv4Addr::BROADCAST)
.unwrap();
assert_eq!(
&*routes,
&[Route {
network: id("a"),
peer
}]
);
assert!(
table
.outgoing(a.local.unwrap(), "10.1.0.255".parse().unwrap())
.is_none()
);
assert!(
table
.outgoing("192.168.1.2".parse().unwrap(), Ipv4Addr::BROADCAST)
.is_none()
);
assert!(
table
.outgoing("10.1.0.1".parse().unwrap(), Ipv4Addr::BROADCAST)
.unwrap()
.is_empty()
);
assert!(!table.accepts(id("a"), peer, "10.1.0.255".parse().unwrap()));
assert!(!table.accepts(id("b"), peer, Ipv4Addr::BROADCAST));
assert!(table.accepts(id("a"), peer, Ipv4Addr::BROADCAST));
let disabled = NetworkRoutes {
exit: Default::default(),
broadcast: BroadcastPolicy {
enabled: false,
..a.broadcast.clone()
},
..a.clone()
};
let table = BroadcastTable::build(&HashMap::from([(id("a"), disabled)]));
assert!(
table
.outgoing(a.local.unwrap(), Ipv4Addr::BROADCAST)
.is_none()
);
assert!(!table.accepts(id("a"), peer, Ipv4Addr::BROADCAST));
}
#[test]
fn malformed_udp_is_not_amplified_but_ipv4_fragments_are_supported() {
let mut packet = vec![0u8; 36];
packet[0] = 0x45;
packet[3] = 36;
packet[9] = 17;
packet[25] = 16;
assert!(valid_udp(&packet));
for length in 0..36 {
assert!(!valid_udp(&packet[..length]));
}
packet[9] = 6;
assert!(!valid_udp(&packet));
packet[9] = 17;
packet[0] = 0x44;
assert!(!valid_udp(&packet));
packet[0] = 0x45;
packet[25] = 8;
assert!(!valid_udp(&packet));
packet[6] = 0x20;
packet[25] = 64;
assert!(valid_udp(&packet));
packet[6] = 0;
packet[7] = 2;
assert!(valid_udp(&packet));
}
}
+141
View File
@@ -0,0 +1,141 @@
//! The desired local WireGuard configuration, and how it is rendered.
//!
//! Each agent builds its own configuration from the agreed set of
//! participants. For a full mesh of `N` members that is `N - 1` peers locally;
//! nobody hands out a configuration to anybody else.
//!
//! Nothing in here is free-form text taken from the network. Peer keys,
//! endpoints, allowed prefixes and keepalives are typed values that this
//! module re-serialises itself, so a hostile announcement cannot inject a
//! configuration directive or a command argument.
use std::net::{IpAddr, Ipv6Addr};
use crate::identity::NetworkId;
use crate::overlay::OverlayError;
/// Prefix length of a single IPv6 host address.
const HOST_PREFIX_LEN: u8 = 128;
/// Longest interface name Linux accepts, excluding the terminating NUL.
pub const MAX_INTERFACE_NAME_LEN: usize = 15;
/// Default prefix for interface names this plugin creates.
pub const DEFAULT_INTERFACE_PREFIX: &str = "tsun";
/// An address with a prefix length.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Cidr {
/// The address.
pub addr: IpAddr,
/// The prefix length in bits.
pub prefix_len: u8,
}
impl Cidr {
/// Builds a CIDR, rejecting an impossible prefix length.
pub fn new(addr: IpAddr, prefix_len: u8) -> Result<Self, OverlayError> {
let max = match addr {
IpAddr::V4(_) => 32,
IpAddr::V6(_) => 128,
};
if prefix_len > max {
return Err(OverlayError::Other(format!(
"prefix length /{prefix_len} is impossible for {addr}"
)));
}
Ok(Self { addr, prefix_len })
}
/// A single host address.
pub fn host(addr: Ipv6Addr) -> Self {
Self {
addr: IpAddr::V6(addr),
prefix_len: HOST_PREFIX_LEN,
}
}
}
impl std::fmt::Display for Cidr {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}/{}", self.addr, self.prefix_len)
}
}
/// Derives this plugin's interface name for a network.
///
/// The name is stable across restarts and short enough for the platform. Two
/// agents on the same host in the same network must be given different
/// prefixes, or they would derive the same name.
pub fn interface_name(prefix: &str, network: NetworkId) -> Result<String, OverlayError> {
if prefix.is_empty() {
return Err(OverlayError::Other(
"interface prefix must not be empty".into(),
));
}
if !prefix
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
{
return Err(OverlayError::Other(
"interface prefix must be lowercase ASCII letters and digits".into(),
));
}
if prefix.len() >= MAX_INTERFACE_NAME_LEN {
return Err(OverlayError::Other(format!(
"interface prefix must be shorter than {MAX_INTERFACE_NAME_LEN} characters"
)));
}
let mut suffix = data_encoding::BASE32_NOPAD.encode(network.as_bytes());
suffix.make_ascii_lowercase();
let room = MAX_INTERFACE_NAME_LEN - prefix.len();
suffix.truncate(room);
Ok(format!("{prefix}{suffix}"))
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::identity::{NetworkKeys, NetworkName, NetworkSecret};
fn network(name: &str) -> NetworkId {
NetworkKeys::derive(
&NetworkName::new(name).unwrap(),
&NetworkSecret::from_bytes(vec![5u8; 32]).unwrap(),
)
.network_id()
}
#[test]
fn interface_names_fit_the_platform_limit_and_are_stable() {
let id = network("naming");
let name = interface_name(DEFAULT_INTERFACE_PREFIX, id).unwrap();
assert_eq!(name.len(), MAX_INTERFACE_NAME_LEN);
assert!(name.starts_with(DEFAULT_INTERFACE_PREFIX));
assert!(name.chars().all(|c| c.is_ascii_alphanumeric()));
assert_eq!(name, interface_name(DEFAULT_INTERFACE_PREFIX, id).unwrap());
assert_ne!(
name,
interface_name(DEFAULT_INTERFACE_PREFIX, network("other")).unwrap()
);
assert_ne!(name, interface_name("wg", id).unwrap());
assert!(interface_name("", id).is_err());
assert!(interface_name("has space", id).is_err());
assert!(interface_name("UPPER", id).is_err());
assert!(interface_name(&"a".repeat(MAX_INTERFACE_NAME_LEN), id).is_err());
}
#[test]
fn a_cidr_rejects_an_impossible_prefix_length() {
assert!(Cidr::new("10.0.0.1".parse().unwrap(), 33).is_err());
assert!(Cidr::new("fd00::1".parse().unwrap(), 129).is_err());
assert_eq!(
Cidr::host("fd00::1".parse().unwrap()).to_string(),
"fd00::1/128"
);
}
}
+32
View File
@@ -0,0 +1,32 @@
//! Exit-node host rules: what makes this device an exit node for others, and
//! what sends this device's own traffic through one.
//!
//! The packet side lives in [`super::router`]; this is the side the operating
//! system sees. It is kept apart from the broadcast rules, which have a
//! different lifetime and different tags.
/// How one set of host rules went.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RuleSetReport {
/// Whether every step took.
pub ok: bool,
/// One line: what is in place, or what is missing and why.
pub detail: String,
}
/// What the host rules of one network's exit-node settings are doing.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExitRulesReport {
/// Masquerading and forwarding for members' traffic, when this agent
/// offers itself as an exit node here. `None` means none are wanted.
pub offer: Option<RuleSetReport>,
/// Whether the kernel forwards packets for the overlay interface.
///
/// `Some(false)` means the rules are in place but nothing passes until
/// forwarding is turned on, which the agent never does itself. `None`
/// when it could not be read.
pub forwarding: Option<bool>,
/// The routes that send this device's traffic through its exit node.
/// `None` means it uses none through this network.
pub client: Option<RuleSetReport>,
}
@@ -0,0 +1,231 @@
//! Host rules for exit nodes: what the operating system needs for this agent
//! to carry other members' internet traffic, and for this device's own
//! traffic to go through a member.
//!
//! Two independent halves, planned together because one interface carries
//! both:
//!
//! * **Offering** — one set of rules per network that offers this agent as an
//! exit node, chosen by that network's range: masquerade the range out of
//! every other interface and let it through the forward chain. A network
//! that does not offer has no rules, so its members' packets are dropped by
//! the firewall as well as by the agent.
//! * **Using** — policy routing that sends everything the host does not have
//! a more specific route for into the overlay interface, while the agent's
//! own sockets keep the ordinary route (see `linux_exit`).
//!
//! Nothing a peer announces becomes a host setting. Offering is the owner's
//! switch; using is the user's choice of a member. Every object is tagged,
//! reversible, and removed when it is no longer wanted, when the network
//! leaves or stops, and when the agent stops.
use std::sync::{Arc, Mutex};
use crate::BoxFuture;
use crate::state::Ipv4Range;
use super::RuleOutcome;
/// What the host should hold for one interface's exit settings.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExitHostPlan {
/// The overlay interface.
pub interface: String,
/// The ranges of every network that offers this agent as an exit node.
pub offer: Vec<Ipv4Range>,
/// Whether this device sends its internet traffic through a member.
pub client: bool,
}
impl ExitHostPlan {
/// Whether there is nothing to hold.
pub fn is_empty(&self) -> bool {
self.offer.is_empty() && !self.client
}
}
/// What applying a plan achieved.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExitHostReport {
/// One outcome per offered range: its masquerade and forward rules.
pub offer: Vec<(Ipv4Range, RuleOutcome)>,
/// Whether the kernel forwards packets for the interface. `None` when it
/// could not be read. The agent only reads this: turning forwarding on is
/// the owner's decision.
pub forwarding: Option<bool>,
/// The routes and rules that send this device's traffic through the
/// interface, when the plan asked for them.
pub client: Option<RuleOutcome>,
}
/// Installs and removes exit-node rules for an overlay interface.
///
/// Like the broadcast rules, no call fails as a whole: a step that cannot be
/// done is in the report, because the caller's only useful response is to
/// say so.
pub trait ExitHostRules: Send + Sync + std::fmt::Debug + 'static {
/// Makes the host match the plan, replacing what this agent put there
/// for the interface before.
fn apply<'a>(&'a self, plan: &'a ExitHostPlan) -> BoxFuture<'a, ExitHostReport>;
/// Removes everything this agent put there for the interface. Succeeds
/// when nothing is there: it runs on the shutdown path, and once at
/// startup to take away what a crashed run left.
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()>;
}
/// The tag every firewall object of one offered range carries.
pub fn exit_tag(interface: &str, range: &Ipv4Range) -> String {
format!("{}{range}", exit_tag_prefix(interface))
}
/// The start of every tag for an interface; what cleanup selects by.
pub fn exit_tag_prefix(interface: &str) -> String {
format!("tsunagi-exit:{interface}:")
}
/// The one-line detail for a set of offer outcomes.
pub fn offer_detail(outcome: &RuleOutcome, range: &Ipv4Range) -> String {
match outcome {
RuleOutcome::Applied => format!("masquerade + forward for {range} in place"),
RuleOutcome::Failed(reason) => reason.clone(),
}
}
/// An in-memory host for exit rules, for tests: nothing is touched.
#[derive(Debug, Clone, Default)]
pub struct MockExitRules {
state: Arc<Mutex<Option<ExitHostPlan>>>,
calls: Arc<Mutex<Vec<String>>>,
forwarding: Arc<Mutex<Option<bool>>>,
failure: Arc<Mutex<Option<String>>>,
}
impl MockExitRules {
/// An empty host with forwarding turned on.
pub fn new() -> Self {
let rules = Self::default();
*lock(&rules.forwarding) = Some(true);
rules
}
/// What the host reports for kernel forwarding from now on.
pub fn set_forwarding(&self, forwarding: Option<bool>) {
*lock(&self.forwarding) = forwarding;
}
/// Makes every step fail with this reason from now on.
pub fn fail_with(&self, reason: Option<String>) {
*lock(&self.failure) = reason;
}
/// What is installed now.
pub fn installed(&self) -> Option<ExitHostPlan> {
lock(&self.state).clone()
}
/// Every `apply:<interface>` / `clear:<interface>` call, in order.
pub fn calls(&self) -> Vec<String> {
lock(&self.calls).clone()
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
match mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
impl ExitHostRules for MockExitRules {
fn apply<'a>(&'a self, plan: &'a ExitHostPlan) -> BoxFuture<'a, ExitHostReport> {
Box::pin(async move {
lock(&self.calls).push(format!("apply:{}", plan.interface));
let outcome = match lock(&self.failure).clone() {
Some(reason) => RuleOutcome::Failed(reason),
None => {
*lock(&self.state) = Some(plan.clone());
RuleOutcome::Applied
}
};
ExitHostReport {
offer: plan
.offer
.iter()
.map(|range| (*range, outcome.clone()))
.collect(),
forwarding: *lock(&self.forwarding),
client: plan.client.then_some(outcome),
}
})
}
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
lock(&self.calls).push(format!("clear:{interface}"));
*lock(&self.state) = None;
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn range() -> Ipv4Range {
"10.13.37.0/24".parse().unwrap()
}
#[test]
fn tags_name_the_interface_and_the_range() {
assert_eq!(
exit_tag("tsun0", &range()),
"tsunagi-exit:tsun0:10.13.37.0/24"
);
assert!(exit_tag("tsun0", &range()).starts_with(&exit_tag_prefix("tsun0")));
// Another interface's tag never matches this one's prefix.
assert!(!exit_tag("tsun01", &range()).starts_with(&exit_tag_prefix("tsun0")));
}
#[test]
fn an_empty_plan_is_nothing_to_hold() {
let mut plan = ExitHostPlan::default();
assert!(plan.is_empty());
plan.client = true;
assert!(!plan.is_empty());
}
#[tokio::test]
async fn the_mock_applies_reports_and_clears() {
let rules = MockExitRules::new();
let plan = ExitHostPlan {
interface: "tsun0".into(),
offer: vec![range()],
client: true,
};
let report = rules.apply(&plan).await;
assert_eq!(report.offer[0].1, RuleOutcome::Applied);
assert_eq!(report.forwarding, Some(true));
assert_eq!(report.client, Some(RuleOutcome::Applied));
assert_eq!(rules.installed(), Some(plan));
rules.set_forwarding(Some(false));
rules.fail_with(Some("denied".into()));
let report = rules
.apply(&ExitHostPlan {
interface: "tsun0".into(),
offer: vec![range()],
client: false,
})
.await;
assert_eq!(report.forwarding, Some(false));
assert!(matches!(report.offer[0].1, RuleOutcome::Failed(_)));
assert_eq!(report.client, None);
rules.clear("tsun0").await;
assert_eq!(rules.installed(), None);
assert_eq!(rules.calls(), ["apply:tsun0", "apply:tsun0", "clear:tsun0"]);
}
}
@@ -0,0 +1,570 @@
//! Broadcast host rules on Linux: a netlink route and an `iptables` rule.
//!
//! The route is added by the netlink thread the interface provisioner already
//! owns (see [`super::super::provision`]), because that thread is the only one
//! that ever holds `CAP_NET_ADMIN`. It belongs to the interface and goes away
//! with it.
//!
//! The firewall allowance is a rule in the `INPUT` chain, inserted at the top
//! so a default-deny chain further down cannot drop discovery replies first:
//!
//! ```text
//! iptables -I INPUT 1 -i <if> -p udp -s <range> -m comment --comment tsunagi:<if> -j ACCEPT
//! ```
//!
//! `iptables` is a program, not a netlink call, so it is looked up in fixed
//! system directories rather than on `PATH`, and its arguments are built from
//! values this agent computed. It works with both the legacy backend and
//! `iptables-nft`. A host with only `nft` or `firewalld` has no `iptables`;
//! that is reported on the firewall half and the route is still installed.
//!
//! The rule carries a comment tag. That is how a rule left behind by a crashed
//! run is found and replaced, and why nothing else in the chain is ever
//! deleted.
use std::net::Ipv4Addr;
use crate::BoxFuture;
use crate::overlay::provision::RouteHandle;
use super::{BroadcastHostRules, BroadcastRulesPlan, BroadcastRulesReport, RuleOutcome, tag_for};
const CHAIN: &str = "INPUT";
/// Fixed places `iptables` is installed. Never `PATH`: the agent is
/// privileged and a directory a user controls must not decide what it runs.
const IPTABLES_DIRECTORIES: [&str; 5] = [
"/usr/sbin/iptables",
"/sbin/iptables",
"/usr/bin/iptables",
"/bin/iptables",
"/usr/local/sbin/iptables",
];
/// The first candidate that exists.
fn pick_iptables(exists: impl Fn(&str) -> bool) -> Option<&'static str> {
IPTABLES_DIRECTORIES
.into_iter()
.find(|candidate| exists(candidate))
}
/// What running `iptables` produced.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum Ran {
/// There is no `iptables` to run.
Missing,
/// It ran, and exited like this.
Exited {
success: bool,
stdout: String,
stderr: String,
},
}
/// Runs `iptables` with these arguments. A function value so the flow below
/// can be tested without touching the host.
pub(super) type Run<'a> = &'a dyn Fn(&[String]) -> Ran;
/// The match and target of the rule, without the chain or position.
fn rule_spec(plan: &BroadcastRulesPlan) -> Vec<String> {
[
"-i",
plan.interface.as_str(),
"-p",
"udp",
"-s",
&plan.range.to_string(),
"-m",
"comment",
"--comment",
&plan.tag(),
"-j",
"ACCEPT",
]
.into_iter()
.map(str::to_string)
.collect()
}
/// Waits briefly for the xtables lock instead of failing when another tool
/// holds it.
pub(super) fn locked(args: impl IntoIterator<Item = String>) -> Vec<String> {
["-w", "5"]
.into_iter()
.map(str::to_string)
.chain(args)
.collect()
}
fn check_args(plan: &BroadcastRulesPlan) -> Vec<String> {
locked(
["-C".to_string(), CHAIN.to_string()]
.into_iter()
.chain(rule_spec(plan)),
)
}
fn insert_args(plan: &BroadcastRulesPlan) -> Vec<String> {
locked(
["-I".to_string(), CHAIN.to_string(), "1".to_string()]
.into_iter()
.chain(rule_spec(plan)),
)
}
fn list_args() -> Vec<String> {
locked(["-S".to_string(), CHAIN.to_string()])
}
/// Splits one `iptables -S` line into words, honouring double quotes, which
/// `iptables` puts around comments that need them.
pub(super) fn split_rule_line(line: &str) -> Vec<String> {
let mut words = Vec::new();
let mut current = String::new();
let mut quoted = false;
let mut started = false;
for c in line.chars() {
match c {
'"' => {
quoted = !quoted;
started = true;
}
c if c.is_whitespace() && !quoted => {
if started {
words.push(std::mem::take(&mut current));
started = false;
}
}
c => {
current.push(c);
started = true;
}
}
}
if started {
words.push(current);
}
words
}
/// The delete commands for every rule in an `iptables -S INPUT` listing that
/// carries exactly this tag. Anything else in the chain is left alone.
fn tagged_deletions(listing: &str, tag: &str) -> Vec<Vec<String>> {
listing
.lines()
.filter_map(|line| {
let words = split_rule_line(line);
let [append, chain, rest @ ..] = words.as_slice() else {
return None;
};
if append != "-A" || chain != CHAIN {
return None;
}
let tagged = rest
.windows(2)
.any(|pair| pair[0] == "--comment" && pair[1] == tag);
tagged.then(|| {
locked(
["-D".to_string(), CHAIN.to_string()]
.into_iter()
.chain(rest.iter().cloned()),
)
})
})
.collect()
}
/// A failure message with the likely fix for the likely cause.
pub(super) fn failure(stderr: &str) -> String {
let text = stderr.trim();
let lower = text.to_ascii_lowercase();
if lower.contains("permission denied") || lower.contains("must be root") {
format!("iptables needs root: {text}")
} else if text.is_empty() {
"iptables failed without saying why".to_string()
} else {
format!("iptables failed: {text}")
}
}
fn missing(plan: &BroadcastRulesPlan) -> String {
format!(
"iptables not found; install it (the iptables-nft package is enough) or allow inbound \
UDP from {} on `{}` in your firewall",
plan.range, plan.interface
)
}
/// Makes the firewall hold exactly one rule for this interface, the planned one.
fn apply_firewall(run: Run<'_>, plan: &BroadcastRulesPlan) -> RuleOutcome {
let present = match run(&check_args(plan)) {
Ran::Missing => return RuleOutcome::Failed(missing(plan)),
Ran::Exited { success, .. } => success,
};
let listing = match run(&list_args()) {
Ran::Missing => return RuleOutcome::Failed(missing(plan)),
Ran::Exited {
success: true,
stdout,
..
} => stdout,
Ran::Exited { stderr, .. } => return RuleOutcome::Failed(failure(&stderr)),
};
let stale = tagged_deletions(&listing, &plan.tag());
// `-C` matched an identical rule and nothing else carries our tag, so
// there is nothing to do. Otherwise start from a clean slate for this
// interface: a rule from an earlier run may name another range.
if present && stale.len() == 1 {
return RuleOutcome::Applied;
}
for delete in &stale {
if let Ran::Exited {
success: false,
stderr,
..
} = run(delete)
{
return RuleOutcome::Failed(failure(&stderr));
}
}
match run(&insert_args(plan)) {
Ran::Missing => RuleOutcome::Failed(missing(plan)),
Ran::Exited { success: true, .. } => RuleOutcome::Applied,
Ran::Exited { stderr, .. } => RuleOutcome::Failed(failure(&stderr)),
}
}
/// Removes every rule tagged for this interface. Best effort.
fn clear_firewall(run: Run<'_>, interface: &str) {
let Ran::Exited {
success: true,
stdout,
..
} = run(&list_args())
else {
return;
};
for delete in tagged_deletions(&stdout, &tag_for(interface)) {
if let Ran::Exited {
success: false,
stderr,
..
} = run(&delete)
{
tracing::warn!(
interface,
"cannot remove the broadcast firewall rule: {}",
failure(&stderr)
);
}
}
}
/// Runs the real `iptables`.
pub(super) fn run_iptables(args: &[String]) -> Ran {
let Some(program) = pick_iptables(|path| std::path::Path::new(path).is_file()) else {
return Ran::Missing;
};
match std::process::Command::new(program).args(args).output() {
Ok(output) => Ran::Exited {
success: output.status.success(),
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
},
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ran::Missing,
Err(err) => Ran::Exited {
success: false,
stdout: String::new(),
stderr: err.to_string(),
},
}
}
/// The Linux implementation: netlink for the route, `iptables` for the rule.
#[derive(Debug)]
pub struct LinuxHostRules {
pub(super) route: RouteHandle,
}
impl LinuxHostRules {
pub(crate) fn new(route: RouteHandle) -> Self {
Self { route }
}
async fn set_route(&self, interface: String, source: Ipv4Addr) -> RuleOutcome {
let route = self.route.clone();
match tokio::task::spawn_blocking(move || route.set(&interface, source)).await {
Ok(Ok(())) => RuleOutcome::Applied,
Ok(Err(err)) => RuleOutcome::Failed(err.to_string()),
Err(err) => RuleOutcome::Failed(format!("the route task failed: {err}")),
}
}
}
impl BroadcastHostRules for LinuxHostRules {
fn name(&self) -> &str {
"netlink+iptables"
}
fn exit_rules(&self) -> Option<&dyn super::ExitHostRules> {
Some(self)
}
fn apply<'a>(&'a self, plan: &'a BroadcastRulesPlan) -> BoxFuture<'a, BroadcastRulesReport> {
Box::pin(async move {
let route = self.set_route(plan.interface.clone(), plan.source).await;
let owned = plan.clone();
let firewall =
tokio::task::spawn_blocking(move || apply_firewall(&run_iptables, &owned))
.await
.unwrap_or_else(|err| {
RuleOutcome::Failed(format!("the firewall task failed: {err}"))
});
BroadcastRulesReport {
plan: plan.clone(),
route,
firewall,
}
})
}
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
let route = self.route.clone();
let name = interface.to_string();
let route_name = name.clone();
if let Ok(Err(err)) =
tokio::task::spawn_blocking(move || route.clear(&route_name)).await
{
tracing::warn!(interface = %name, %err, "cannot remove the broadcast route");
}
let name = interface.to_string();
let _ = tokio::task::spawn_blocking(move || clear_firewall(&run_iptables, &name)).await;
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::cell::RefCell;
fn plan() -> BroadcastRulesPlan {
BroadcastRulesPlan {
interface: "tsun0".into(),
source: Ipv4Addr::new(10, 13, 37, 142),
range: "10.13.37.0/24".parse().unwrap(),
}
}
fn words(args: &[String]) -> String {
args.join(" ")
}
#[test]
fn the_rule_is_udp_only_from_the_overlay_range_on_this_interface() {
let spec = words(&rule_spec(&plan()));
assert_eq!(
spec,
"-i tsun0 -p udp -s 10.13.37.0/24 -m comment --comment tsunagi:tsun0 -j ACCEPT"
);
// Nothing here may widen to other protocols or every source.
assert!(spec.contains("-p udp"));
assert!(!spec.contains("0.0.0.0/0"));
assert!(words(&insert_args(&plan())).starts_with("-w 5 -I INPUT 1 -i tsun0"));
assert!(words(&check_args(&plan())).starts_with("-w 5 -C INPUT -i tsun0"));
}
#[test]
fn iptables_is_found_in_fixed_directories_never_on_path() {
assert_eq!(pick_iptables(|_| false), None);
assert_eq!(
pick_iptables(|p| p == "/sbin/iptables"),
Some("/sbin/iptables")
);
assert_eq!(pick_iptables(|_| true), Some("/usr/sbin/iptables"));
}
#[test]
fn only_rules_with_our_exact_tag_are_selected_for_deletion() {
let listing = "\
-P INPUT DROP
-A INPUT -s 10.13.37.0/24 -i tsun0 -p udp -m comment --comment \"tsunagi:tsun0\" -j ACCEPT
-A INPUT -i tsun0 -m comment --comment tsunagi:tsun0 -j ACCEPT
-A INPUT -i tsun1 -p udp -m comment --comment tsunagi:tsun1 -j ACCEPT
-A INPUT -p tcp --dport 22 -m comment --comment \"allow ssh tsunagi:tsun0\" -j ACCEPT
-A FORWARD -i tsun0 -m comment --comment tsunagi:tsun0 -j ACCEPT
-A INPUT -p tcp --dport 22 -j ACCEPT
";
let deletions = tagged_deletions(listing, "tsunagi:tsun0");
assert_eq!(deletions.len(), 2, "{deletions:?}");
assert_eq!(
words(&deletions[0]),
"-w 5 -D INPUT -s 10.13.37.0/24 -i tsun0 -p udp -m comment --comment tsunagi:tsun0 -j ACCEPT"
);
assert!(words(&deletions[1]).contains("-D INPUT -i tsun0 -m comment"));
assert!(tagged_deletions("", "tsunagi:tsun0").is_empty());
}
/// A scripted `iptables`: answers by the leading verb, records every call.
struct Fake {
calls: RefCell<Vec<String>>,
check_ok: bool,
listing: String,
insert_stderr: Option<&'static str>,
missing: bool,
}
impl Fake {
fn new(check_ok: bool, listing: &str) -> Self {
Self {
calls: RefCell::new(Vec::new()),
check_ok,
listing: listing.to_string(),
insert_stderr: None,
missing: false,
}
}
fn run(&self, args: &[String]) -> Ran {
self.calls.borrow_mut().push(words(args));
if self.missing {
return Ran::Missing;
}
let ok = |stdout: &str| Ran::Exited {
success: true,
stdout: stdout.to_string(),
stderr: String::new(),
};
match args.get(2).map(String::as_str) {
Some("-C") if self.check_ok => ok(""),
Some("-C") => Ran::Exited {
success: false,
stdout: String::new(),
stderr: "Bad rule".into(),
},
Some("-S") => ok(&self.listing),
Some("-I") => match self.insert_stderr {
Some(stderr) => Ran::Exited {
success: false,
stdout: String::new(),
stderr: stderr.into(),
},
None => ok(""),
},
_ => ok(""),
}
}
fn verbs(&self) -> Vec<String> {
self.calls
.borrow()
.iter()
.map(|call| call.split(' ').nth(2).unwrap().to_string())
.collect()
}
}
const ONE_OURS: &str =
"-A INPUT -s 10.13.37.0/24 -i tsun0 -p udp -m comment --comment tsunagi:tsun0 -j ACCEPT\n";
#[test]
fn an_identical_rule_already_in_place_is_left_alone() {
let fake = Fake::new(true, ONE_OURS);
assert_eq!(
apply_firewall(&|a| fake.run(a), &plan()),
RuleOutcome::Applied
);
assert_eq!(fake.verbs(), ["-C", "-S"], "no insert, no delete");
}
#[test]
fn a_missing_rule_is_inserted_once() {
let fake = Fake::new(false, "-P INPUT ACCEPT\n");
assert_eq!(
apply_firewall(&|a| fake.run(a), &plan()),
RuleOutcome::Applied
);
assert_eq!(fake.verbs(), ["-C", "-S", "-I"]);
}
#[test]
fn a_stale_rule_for_another_range_is_replaced_not_stacked() {
// A crashed run left a rule for an old range under our tag.
let stale = "-A INPUT -s 10.99.0.0/16 -i tsun0 -p udp -m comment --comment tsunagi:tsun0 -j ACCEPT\n";
let fake = Fake::new(false, stale);
assert_eq!(
apply_firewall(&|a| fake.run(a), &plan()),
RuleOutcome::Applied
);
assert_eq!(fake.verbs(), ["-C", "-S", "-D", "-I"]);
}
#[test]
fn duplicates_of_the_rule_collapse_to_one() {
let doubled = format!("{ONE_OURS}{ONE_OURS}");
let fake = Fake::new(true, &doubled);
assert_eq!(
apply_firewall(&|a| fake.run(a), &plan()),
RuleOutcome::Applied
);
assert_eq!(fake.verbs(), ["-C", "-S", "-D", "-D", "-I"]);
}
#[test]
fn a_missing_iptables_says_what_to_install_and_what_to_allow() {
let mut fake = Fake::new(false, "");
fake.missing = true;
let RuleOutcome::Failed(reason) = apply_firewall(&|a| fake.run(a), &plan()) else {
panic!("expected a failure");
};
assert!(reason.contains("iptables not found"), "{reason}");
assert!(reason.contains("10.13.37.0/24"), "{reason}");
assert!(reason.contains("tsun0"), "{reason}");
}
#[test]
fn a_refused_insert_is_reported_with_the_cause() {
let mut fake = Fake::new(false, "");
fake.insert_stderr = Some(
"iptables v1.8.10: can't initialize iptables table `filter': Permission denied (you must be root)",
);
let RuleOutcome::Failed(reason) = apply_firewall(&|a| fake.run(a), &plan()) else {
panic!("expected a failure");
};
assert!(reason.starts_with("iptables needs root"), "{reason}");
}
#[test]
fn clearing_removes_only_tagged_rules_and_ignores_a_missing_iptables() {
let listing = format!(
"-A INPUT -p tcp --dport 22 -j ACCEPT\n{ONE_OURS}-A INPUT -i tsun1 -m comment --comment tsunagi:tsun1 -j ACCEPT\n"
);
let fake = Fake::new(true, &listing);
clear_firewall(&|a| fake.run(a), "tsun0");
assert_eq!(fake.verbs(), ["-S", "-D"]);
let mut gone = Fake::new(true, "");
gone.missing = true;
clear_firewall(&|a| gone.run(a), "tsun0");
assert_eq!(gone.verbs(), ["-S"]);
}
#[test]
fn rule_listing_words_honour_quotes() {
assert_eq!(
split_rule_line("-A INPUT -m comment --comment \"a b\" -j ACCEPT"),
[
"-A",
"INPUT",
"-m",
"comment",
"--comment",
"a b",
"-j",
"ACCEPT"
]
);
assert!(split_rule_line(" ").is_empty());
}
}
@@ -0,0 +1,728 @@
//! Exit-node host rules on Linux: `iptables` for the offering side, policy
//! routing over netlink for the using side.
//!
//! # Offering
//!
//! For every network that offers this agent as an exit node, three rules,
//! selected by that network's range and carrying the tag
//! `tsunagi-exit:<if>:<range>`:
//!
//! ```text
//! -t nat POSTROUTING -s <range> ! -o <if> -j MASQUERADE
//! -t filter FORWARD -i <if> -s <range> ! -o <if> -j ACCEPT
//! -t filter FORWARD -o <if> -d <range> -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
//! ```
//!
//! `! -o <if>` is the egress without naming it: whatever interface the
//! default route uses, now or after it changes. Kernel forwarding is only
//! *read*. Turning it on changes how the whole host behaves, which is the
//! owner's decision, so when it is off the rules are installed anyway and the
//! report says so.
//!
//! # Using
//!
//! Policy routing, with no `/1` routes and nothing in the main table:
//!
//! ```text
//! table 0x7473: default dev <if>
//! 5280: from all uidrange <euid>-<euid> lookup main
//! 5290: from all lookup main suppress_prefixlength 0
//! 5300: from all lookup 0x7473
//! ```
//!
//! The first keeps the agent's *own* sockets off the tunnel: its QUIC, relay,
//! DHT and DNS traffic would otherwise be routed into the interface it is
//! carrying, a loop. The second lets every specific route — the LAN, another
//! VPN, a tunnel to somewhere else — keep working, because only the default
//! route is suppressed. The priorities sit after Tailscale's 5270.
//!
//! The first rule exempts a *user id*, so it is only right when the agent has
//! one of its own, as the packaged service does. An agent started from a login
//! session would take that user's own applications out of the tunnel with it,
//! and the exit node would silently do nothing for them. That is refused, not
//! guessed at: see [`client_refusal`].
use crate::BoxFuture;
use crate::state::Ipv4Range;
use super::RuleOutcome;
use super::exit::{ExitHostPlan, ExitHostReport, ExitHostRules, exit_tag, exit_tag_prefix};
use super::linux::{LinuxHostRules, Ran, Run, failure, locked, run_iptables, split_rule_line};
/// The chains the offering rules live in: `(table, chain)`.
const CHAINS: [(&str, &str); 2] = [("nat", "POSTROUTING"), ("filter", "FORWARD")];
/// One firewall rule: where it goes, and its match and target.
#[derive(Debug, Clone, PartialEq, Eq)]
struct Rule {
table: &'static str,
chain: &'static str,
spec: Vec<String>,
}
fn owned(words: &[&str]) -> Vec<String> {
words.iter().map(|word| word.to_string()).collect()
}
/// The three rules of one offered range.
fn offer_rules(interface: &str, range: &Ipv4Range) -> Vec<Rule> {
let range_text = range.to_string();
let tag = exit_tag(interface, range);
let comment = ["-m", "comment", "--comment", tag.as_str()];
let spec = |head: &[&str], tail: &[&str]| -> Vec<String> {
owned(head)
.into_iter()
.chain(owned(&comment))
.chain(owned(tail))
.collect()
};
vec![
Rule {
table: "nat",
chain: "POSTROUTING",
spec: spec(
&["-s", &range_text, "!", "-o", interface],
&["-j", "MASQUERADE"],
),
},
Rule {
table: "filter",
chain: "FORWARD",
spec: spec(
&["-i", interface, "-s", &range_text, "!", "-o", interface],
&["-j", "ACCEPT"],
),
},
Rule {
table: "filter",
chain: "FORWARD",
spec: spec(
&[
"-o",
interface,
"-d",
&range_text,
"-m",
"conntrack",
"--ctstate",
"RELATED,ESTABLISHED",
],
&["-j", "ACCEPT"],
),
},
]
}
fn check_args(rule: &Rule) -> Vec<String> {
locked(
owned(&["-t", rule.table, "-C", rule.chain])
.into_iter()
.chain(rule.spec.iter().cloned()),
)
}
fn insert_args(rule: &Rule) -> Vec<String> {
locked(
owned(&["-t", rule.table, "-I", rule.chain, "1"])
.into_iter()
.chain(rule.spec.iter().cloned()),
)
}
fn list_args(table: &str, chain: &str) -> Vec<String> {
locked(owned(&["-t", table, "-S", chain]))
}
/// A rule found in a listing whose comment starts with our prefix.
#[derive(Debug, Clone, PartialEq, Eq)]
struct Found {
/// The whole tag.
tag: String,
/// The words after `-A <chain>`, which is what `-D` needs.
rest: Vec<String>,
}
/// Every rule in an `iptables -S <chain>` listing carrying a tag that starts
/// with `prefix`. Anything else in the chain is left alone.
fn tagged_rules(listing: &str, chain: &str, prefix: &str) -> Vec<Found> {
listing
.lines()
.filter_map(|line| {
let words = split_rule_line(line);
let [append, found_chain, rest @ ..] = words.as_slice() else {
return None;
};
if append != "-A" || found_chain != chain {
return None;
}
let tag = rest
.windows(2)
.find(|pair| pair[0] == "--comment" && pair[1].starts_with(prefix))
.map(|pair| pair[1].clone())?;
Some(Found {
tag,
rest: rest.to_vec(),
})
})
.collect()
}
fn delete_args(table: &str, chain: &str, found: &Found) -> Vec<String> {
locked(
owned(&["-t", table, "-D", chain])
.into_iter()
.chain(found.rest.iter().cloned()),
)
}
fn missing_iptables(range: &Ipv4Range, interface: &str) -> String {
format!(
"iptables not found; install it (the iptables-nft package is enough), or masquerade \
{range} out of every interface but `{interface}` and forward it yourself"
)
}
/// Reads one chain, or says why not.
fn read_chain(run: Run<'_>, table: &str, chain: &str) -> Result<String, Failure> {
match run(&list_args(table, chain)) {
Ran::Missing => Err(Failure::Missing),
Ran::Exited {
success: true,
stdout,
..
} => Ok(stdout),
Ran::Exited { stderr, .. } => Err(Failure::Failed(failure(&stderr))),
}
}
enum Failure {
Missing,
Failed(String),
}
/// Deletes found rules; the first failure ends it.
fn delete_all(
run: Run<'_>,
table: &str,
chain: &str,
found: &[Found],
keep: impl Fn(&Found) -> bool,
) -> Result<(), String> {
for rule in found.iter().filter(|rule| !keep(rule)) {
if let Ran::Exited {
success: false,
stderr,
..
} = run(&delete_args(table, chain, rule))
{
return Err(failure(&stderr));
}
}
Ok(())
}
/// Makes the firewall hold exactly the offering rules of these ranges for the
/// interface, and no others of ours.
fn apply_offer(
run: Run<'_>,
interface: &str,
ranges: &[Ipv4Range],
) -> Vec<(Ipv4Range, RuleOutcome)> {
let fail_all = |reason: String| -> Vec<(Ipv4Range, RuleOutcome)> {
ranges
.iter()
.map(|range| (*range, RuleOutcome::Failed(reason.clone())))
.collect()
};
let prefix = exit_tag_prefix(interface);
let wanted: Vec<String> = ranges.iter().map(|r| exit_tag(interface, r)).collect();
// What is there now, per chain.
let mut present: Vec<(&str, &str, Vec<Found>)> = Vec::new();
for (table, chain) in CHAINS {
match read_chain(run, table, chain) {
Ok(listing) => present.push((table, chain, tagged_rules(&listing, chain, &prefix))),
Err(Failure::Missing) => {
return match ranges.first() {
Some(range) => fail_all(missing_iptables(range, interface)),
None => Vec::new(),
};
}
Err(Failure::Failed(reason)) => return fail_all(reason),
}
}
// Rules for a range no longer offered, or left by an earlier run.
for (table, chain, found) in &present {
if let Err(reason) = delete_all(run, table, chain, found, |rule| wanted.contains(&rule.tag))
{
return fail_all(reason);
}
}
ranges
.iter()
.map(|range| {
let rules = offer_rules(interface, range);
let tag = exit_tag(interface, range);
let in_place = rules
.iter()
.all(|rule| matches!(run(&check_args(rule)), Ran::Exited { success: true, .. }))
&& present.iter().all(|(_, chain, found)| {
let have = found.iter().filter(|rule| rule.tag == tag).count();
have == rules.iter().filter(|rule| rule.chain == *chain).count()
});
if in_place {
return (*range, RuleOutcome::Applied);
}
// Start this range from a clean slate: a half-installed set, or
// one that differs from the plan, is replaced as a whole.
for (table, chain, found) in &present {
if let Err(reason) = delete_all(run, table, chain, found, |rule| rule.tag != tag) {
return (*range, RuleOutcome::Failed(reason));
}
}
for rule in &rules {
match run(&insert_args(rule)) {
Ran::Missing => {
return (
*range,
RuleOutcome::Failed(missing_iptables(range, interface)),
);
}
Ran::Exited { success: true, .. } => {}
Ran::Exited { stderr, .. } => {
return (*range, RuleOutcome::Failed(failure(&stderr)));
}
}
}
(*range, RuleOutcome::Applied)
})
.collect()
}
/// Removes every offering rule of ours for the interface. Best effort.
fn clear_offer(run: Run<'_>, interface: &str) {
let prefix = exit_tag_prefix(interface);
for (table, chain) in CHAINS {
let Ok(listing) = read_chain(run, table, chain) else {
continue;
};
let found = tagged_rules(&listing, chain, &prefix);
if let Err(reason) = delete_all(run, table, chain, &found, |_| false) {
tracing::warn!(
interface,
"cannot remove an exit node firewall rule: {reason}"
);
}
}
}
/// The kernel's forwarding switch for packets arriving on the interface,
/// read from the text of the sysctl files. Never written.
fn forwarding_state(read: impl Fn(&str) -> Option<String>, interface: &str) -> Option<bool> {
let flag = |text: String| match text.trim() {
"1" => Some(true),
"0" => Some(false),
_ => None,
};
// The name is derived here, but it ends up in a path.
if !interface.is_empty() && !interface.contains(['/', '\0']) && interface != ".." {
let per_interface = format!("/proc/sys/net/ipv4/conf/{interface}/forwarding");
if let Some(state) = read(&per_interface).and_then(flag) {
return Some(state);
}
}
read("/proc/sys/net/ipv4/ip_forward").and_then(flag)
}
fn read_sysctl(path: &str) -> Option<String> {
std::fs::read_to_string(path).ok()
}
/// The effective user id from the text of `/proc/self/status`.
fn parse_euid(status: &str) -> Option<u32> {
status
.lines()
.find_map(|line| line.strip_prefix("Uid:"))
.and_then(|rest| rest.split_whitespace().nth(1))
.and_then(|effective| effective.parse().ok())
}
/// Why the using side must not be installed for this process, if it must not.
///
/// The bypass rule exempts this process's *user id* from the tunnel. That is
/// right for a service with an id of its own and wrong for a process started
/// from somebody's login session, which shares its id with every application
/// that person runs. A login session is recognised by `XDG_RUNTIME_DIR`,
/// which `systemd --user` and the login managers set and a system service
/// does not. Crude on purpose: refusing a service that was started oddly is
/// a message, and exempting a user's applications is a silent failure.
fn client_refusal(xdg_runtime_dir: Option<&str>) -> Option<String> {
xdg_runtime_dir.filter(|dir| !dir.is_empty()).map(|_| {
"the exit node needs the agent to run as a system service with a user of its own: \
this agent was started from a login session, so routing everything through the \
overlay would take that user's own applications out of the tunnel with it"
.to_string()
})
}
impl LinuxHostRules {
async fn blocking<T: Send + 'static>(
work: impl FnOnce() -> T + Send + 'static,
) -> Result<T, String> {
tokio::task::spawn_blocking(work)
.await
.map_err(|err| format!("the task failed: {err}"))
}
async fn apply_client(&self, interface: &str) -> RuleOutcome {
let xdg = std::env::var("XDG_RUNTIME_DIR").ok();
if let Some(reason) = client_refusal(xdg.as_deref()) {
// Whatever an earlier run left must not stay in force.
self.clear_client().await;
return RuleOutcome::Failed(reason);
}
let Some(euid) = std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|status| parse_euid(&status))
else {
return RuleOutcome::Failed("cannot tell which user this agent runs as".to_string());
};
let route = self.route.clone();
let interface = interface.to_string();
match Self::blocking(move || route.set_exit_client(&interface, euid)).await {
Ok(Ok(())) => RuleOutcome::Applied,
Ok(Err(err)) => RuleOutcome::Failed(err.to_string()),
Err(reason) => RuleOutcome::Failed(reason),
}
}
async fn clear_client(&self) {
let route = self.route.clone();
if let Ok(Err(err)) = Self::blocking(move || route.clear_exit_client()).await {
tracing::warn!(%err, "cannot remove the exit node routing rules");
}
}
}
impl ExitHostRules for LinuxHostRules {
fn apply<'a>(&'a self, plan: &'a ExitHostPlan) -> BoxFuture<'a, ExitHostReport> {
Box::pin(async move {
let interface = plan.interface.clone();
let ranges = plan.offer.clone();
let offer = {
let (job_interface, job_ranges) = (interface.clone(), ranges.clone());
Self::blocking(move || apply_offer(&run_iptables, &job_interface, &job_ranges))
.await
.unwrap_or_else(|reason| {
ranges
.iter()
.map(|range| (*range, RuleOutcome::Failed(reason.clone())))
.collect()
})
};
let forwarding = if ranges.is_empty() {
None
} else {
forwarding_state(read_sysctl, &interface)
};
let client = if plan.client {
Some(self.apply_client(&interface).await)
} else {
self.clear_client().await;
None
};
ExitHostReport {
offer,
forwarding,
client,
}
})
}
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
let name = interface.to_string();
let _ = Self::blocking(move || clear_offer(&run_iptables, &name)).await;
self.clear_client().await;
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::cell::RefCell;
use super::*;
fn range() -> Ipv4Range {
"10.13.37.0/24".parse().unwrap()
}
fn words(args: &[String]) -> String {
args.join(" ")
}
#[test]
fn the_rules_masquerade_the_range_out_of_every_other_interface_and_forward_it() {
let rules = offer_rules("tsun0", &range());
assert_eq!(rules.len(), 3);
assert_eq!(
words(&rules[0].spec),
"-s 10.13.37.0/24 ! -o tsun0 -m comment --comment tsunagi-exit:tsun0:10.13.37.0/24 -j MASQUERADE"
);
assert_eq!((rules[0].table, rules[0].chain), ("nat", "POSTROUTING"));
assert_eq!(
words(&rules[1].spec),
"-i tsun0 -s 10.13.37.0/24 ! -o tsun0 -m comment --comment tsunagi-exit:tsun0:10.13.37.0/24 -j ACCEPT"
);
assert_eq!((rules[1].table, rules[1].chain), ("filter", "FORWARD"));
assert_eq!(
words(&rules[2].spec),
"-o tsun0 -d 10.13.37.0/24 -m conntrack --ctstate RELATED,ESTABLISHED -m comment --comment tsunagi-exit:tsun0:10.13.37.0/24 -j ACCEPT"
);
// Nothing may widen to every source or to the other direction.
for rule in &rules {
assert!(!words(&rule.spec).contains("0.0.0.0/0"));
}
assert!(words(&insert_args(&rules[1])).starts_with("-w 5 -t filter -I FORWARD 1 -i tsun0"));
assert!(words(&check_args(&rules[0])).starts_with("-w 5 -t nat -C POSTROUTING -s"));
}
#[test]
fn only_rules_with_our_tag_for_this_interface_are_selected() {
let listing = "\
-P FORWARD DROP
-A FORWARD -i tsun0 -s 10.13.37.0/24 ! -o tsun0 -m comment --comment tsunagi-exit:tsun0:10.13.37.0/24 -j ACCEPT
-A FORWARD -i tsun1 -m comment --comment tsunagi-exit:tsun1:10.99.0.0/24 -j ACCEPT
-A FORWARD -i tsun01 -m comment --comment tsunagi-exit:tsun01:10.5.0.0/24 -j ACCEPT
-A FORWARD -m comment --comment \"allow tsunagi-exit:tsun0:x\" -j ACCEPT
-A FORWARD -m comment --comment tsunagi:tsun0 -j ACCEPT
-A INPUT -m comment --comment tsunagi-exit:tsun0:10.13.37.0/24 -j ACCEPT
-A FORWARD -j ACCEPT
";
let found = tagged_rules(listing, "FORWARD", &exit_tag_prefix("tsun0"));
assert_eq!(found.len(), 1, "{found:?}");
assert_eq!(found[0].tag, "tsunagi-exit:tsun0:10.13.37.0/24");
assert_eq!(
words(&delete_args("filter", "FORWARD", &found[0])),
"-w 5 -t filter -D FORWARD -i tsun0 -s 10.13.37.0/24 ! -o tsun0 -m comment --comment tsunagi-exit:tsun0:10.13.37.0/24 -j ACCEPT"
);
assert!(tagged_rules("", "FORWARD", "tsunagi-exit:tsun0:").is_empty());
}
/// A scripted `iptables` that remembers which rules it was given.
struct Fake {
calls: RefCell<Vec<String>>,
/// Rules present, as `-S` lines per `(table, chain)`.
listing: RefCell<Vec<(String, String)>>,
missing: bool,
check_ok: bool,
}
impl Fake {
fn new(check_ok: bool) -> Self {
Self {
calls: RefCell::new(Vec::new()),
listing: RefCell::new(Vec::new()),
missing: false,
check_ok,
}
}
fn with_rule(self, table: &str, line: &str) -> Self {
self.listing
.borrow_mut()
.push((table.to_string(), line.to_string()));
self
}
fn run(&self, args: &[String]) -> Ran {
if self.missing {
return Ran::Missing;
}
let line = args.join(" ");
self.calls.borrow_mut().push(line.clone());
let table = args
.iter()
.position(|word| word == "-t")
.map(|at| args[at + 1].clone())
.unwrap_or_default();
let ok = |stdout: String| Ran::Exited {
success: true,
stdout,
stderr: String::new(),
};
if line.contains(" -S ") {
let listing: Vec<String> = self
.listing
.borrow()
.iter()
.filter(|(t, _)| *t == table)
.map(|(_, l)| l.clone())
.collect();
return ok(listing.join("\n"));
}
if line.contains(" -C ") {
return Ran::Exited {
success: self.check_ok,
stdout: String::new(),
stderr: String::new(),
};
}
ok(String::new())
}
fn verbs(&self, verb: &str) -> Vec<String> {
self.calls
.borrow()
.iter()
.filter(|call| call.contains(&format!(" {verb} ")))
.cloned()
.collect()
}
}
#[test]
fn a_fresh_host_gets_the_three_rules() {
let fake = Fake::new(false);
let outcome = apply_offer(&|args| fake.run(args), "tsun0", &[range()]);
assert_eq!(outcome, vec![(range(), RuleOutcome::Applied)]);
let inserted = fake.verbs("-I");
assert_eq!(inserted.len(), 3, "{inserted:?}");
assert!(inserted[0].contains("-t nat -I POSTROUTING 1"));
assert!(fake.verbs("-D").is_empty());
}
#[test]
fn rules_already_in_place_are_left_alone() {
let tag = exit_tag("tsun0", &range());
let fake = Fake::new(true)
.with_rule(
"nat",
&format!("-A POSTROUTING -s 10.13.37.0/24 ! -o tsun0 -m comment --comment {tag} -j MASQUERADE"),
)
.with_rule("filter", &format!("-A FORWARD -i tsun0 -m comment --comment {tag} -j ACCEPT"))
.with_rule("filter", &format!("-A FORWARD -o tsun0 -m comment --comment {tag} -j ACCEPT"));
let outcome = apply_offer(&|args| fake.run(args), "tsun0", &[range()]);
assert_eq!(outcome, vec![(range(), RuleOutcome::Applied)]);
assert!(fake.verbs("-I").is_empty());
assert!(fake.verbs("-D").is_empty());
}
#[test]
fn a_stale_range_is_removed_and_a_half_installed_set_is_replaced() {
let old: Ipv4Range = "10.99.0.0/24".parse().unwrap();
let old_tag = exit_tag("tsun0", &old);
let tag = exit_tag("tsun0", &range());
let fake = Fake::new(true)
.with_rule(
"nat",
&format!(
"-A POSTROUTING -s 10.99.0.0/24 -m comment --comment {old_tag} -j MASQUERADE"
),
)
// Only one of the three for the current range.
.with_rule(
"filter",
&format!("-A FORWARD -i tsun0 -m comment --comment {tag} -j ACCEPT"),
)
// Somebody else's rule, which must survive.
.with_rule("filter", "-A FORWARD -i eth0 -j ACCEPT");
let outcome = apply_offer(&|args| fake.run(args), "tsun0", &[range()]);
assert_eq!(outcome, vec![(range(), RuleOutcome::Applied)]);
let deleted = fake.verbs("-D");
assert!(
deleted.iter().any(|call| call.contains(&old_tag)),
"{deleted:?}"
);
assert!(
deleted.iter().any(|call| call.contains(&tag)),
"{deleted:?}"
);
assert!(
deleted.iter().all(|call| !call.contains("eth0")),
"{deleted:?}"
);
assert_eq!(fake.verbs("-I").len(), 3);
}
#[test]
fn clearing_removes_only_our_rules_for_the_interface() {
let tag = exit_tag("tsun0", &range());
let other = exit_tag("tsun1", &range());
let fake = Fake::new(true)
.with_rule(
"nat",
&format!("-A POSTROUTING -m comment --comment {tag} -j MASQUERADE"),
)
.with_rule(
"nat",
&format!("-A POSTROUTING -m comment --comment {other} -j MASQUERADE"),
)
.with_rule("filter", "-A FORWARD -j ACCEPT");
clear_offer(&|args| fake.run(args), "tsun0");
let deleted = fake.verbs("-D");
assert_eq!(deleted.len(), 1, "{deleted:?}");
assert!(deleted[0].contains(&tag));
}
#[test]
fn a_missing_iptables_is_a_failure_that_says_what_to_do() {
let mut fake = Fake::new(false);
fake.missing = true;
let outcome = apply_offer(&|args| fake.run(args), "tsun0", &[range()]);
let RuleOutcome::Failed(reason) = &outcome[0].1 else {
panic!("{outcome:?}");
};
assert!(reason.contains("iptables not found"), "{reason}");
assert!(reason.contains("10.13.37.0/24"), "{reason}");
// Nothing wanted and nothing to run is not a failure of anything.
assert!(apply_offer(&|args| fake.run(args), "tsun0", &[]).is_empty());
}
#[test]
fn forwarding_is_read_per_interface_then_globally_and_never_written() {
let files = |per: Option<&'static str>, global: Option<&'static str>| {
move |path: &str| match path {
"/proc/sys/net/ipv4/conf/tsun0/forwarding" => per.map(str::to_string),
"/proc/sys/net/ipv4/ip_forward" => global.map(str::to_string),
_ => None,
}
};
assert_eq!(
forwarding_state(files(Some("1\n"), Some("0\n")), "tsun0"),
Some(true)
);
assert_eq!(
forwarding_state(files(Some("0\n"), Some("1\n")), "tsun0"),
Some(false)
);
assert_eq!(
forwarding_state(files(None, Some("1\n")), "tsun0"),
Some(true)
);
assert_eq!(forwarding_state(files(None, None), "tsun0"), None);
assert_eq!(forwarding_state(files(Some("junk"), None), "tsun0"), None);
// A name that would leave the directory is not looked up.
assert_eq!(forwarding_state(files(Some("1"), None), "../../x"), None);
}
#[test]
fn the_effective_uid_is_the_second_field() {
let status = "Name:\ttsunagi\nUid:\t1001\t1002\t1001\t1001\nGid:\t5\t5\t5\t5\n";
assert_eq!(parse_euid(status), Some(1002));
assert_eq!(parse_euid("Name:\tx\n"), None);
}
#[test]
fn a_login_session_is_refused_and_a_service_is_not() {
let reason = client_refusal(Some("/run/user/1000")).unwrap();
assert!(reason.contains("system service"), "{reason}");
assert!(client_refusal(None).is_none());
assert!(client_refusal(Some("")).is_none());
}
}
@@ -0,0 +1,189 @@
//! Broadcast host rules on macOS: a `route` entry; the firewall is deferred.
//!
//! The limited-broadcast route is installed with `/sbin/route`, so a game's
//! LAN discovery packet leaves through the overlay interface. It belongs to
//! the interface and is removed when broadcast is turned off, the network
//! leaves, or the agent stops — and, because a utun is torn down with the
//! process, it also goes away on its own if the agent is killed.
//!
//! The inbound-UDP firewall allowance is **not** implemented here. On macOS
//! that means a `pf` anchor, which is a good deal more machinery; until it is
//! written the firewall half reports itself as missing with what to allow by
//! hand, and the route is still installed. Failure is contained: neither half
//! can stop the agent, and `status` shows exactly which is in place.
//!
//! `route` is a program, not a system call, so it is run by absolute path from
//! `/sbin` rather than off `PATH`, and its arguments are built from values
//! this agent computed — the interface name the kernel assigned, nothing a
//! peer said.
use crate::BoxFuture;
use super::{BroadcastHostRules, BroadcastRulesPlan, BroadcastRulesReport, RuleOutcome};
/// The limited-broadcast destination routed through the overlay interface.
const LIMITED_BROADCAST: &str = "255.255.255.255";
/// The `route` arguments that direct the limited broadcast out an interface.
fn route_add_args(interface: &str) -> Vec<String> {
vec![
"-q".into(),
"-n".into(),
"add".into(),
"-host".into(),
LIMITED_BROADCAST.into(),
"-interface".into(),
interface.into(),
]
}
/// The `route` arguments that remove it again.
fn route_del_args(interface: &str) -> Vec<String> {
vec![
"-q".into(),
"-n".into(),
"delete".into(),
"-host".into(),
LIMITED_BROADCAST.into(),
"-interface".into(),
interface.into(),
]
}
/// What to do by hand while the `pf` allowance is unimplemented.
fn firewall_todo(plan: &BroadcastRulesPlan) -> String {
format!(
"the inbound-UDP firewall allowance is not implemented on macOS yet; if the host \
firewall drops discovery replies, allow inbound UDP from {} on `{}` with a pf rule \
yourself",
plan.range, plan.interface
)
}
/// The macOS implementation: `route` for the route, no firewall yet.
#[derive(Debug, Default)]
pub struct MacosHostRules;
impl MacosHostRules {
/// Creates the host rules.
pub fn new() -> Self {
Self
}
/// Installs the route, replacing one this agent added earlier so the call
/// can be repeated. The delete is best effort: a route that is not there
/// is the state the add then reaches anyway.
async fn set_route(&self, interface: String) -> RuleOutcome {
let _ = run(route_del_args(&interface)).await;
match run(route_add_args(&interface)).await {
Ok(()) => RuleOutcome::Applied,
Err(reason) => RuleOutcome::Failed(reason),
}
}
}
impl BroadcastHostRules for MacosHostRules {
fn name(&self) -> &str {
"route"
}
fn apply<'a>(&'a self, plan: &'a BroadcastRulesPlan) -> BoxFuture<'a, BroadcastRulesReport> {
Box::pin(async move {
let route = self.set_route(plan.interface.clone()).await;
BroadcastRulesReport {
plan: plan.clone(),
route,
firewall: RuleOutcome::Failed(firewall_todo(plan)),
}
})
}
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
if let Err(reason) = run(route_del_args(interface)).await {
tracing::debug!(interface = %interface, %reason, "cannot remove the broadcast route");
}
})
}
}
/// Runs `/sbin/route`, turning a non-zero exit into the message it printed.
async fn run(args: Vec<String>) -> Result<(), String> {
let display = format!("route {}", args.join(" "));
let output = tokio::task::spawn_blocking(move || {
std::process::Command::new("/sbin/route")
.args(&args)
.output()
})
.await
.map_err(|err| format!("could not run {display}: {err}"))?
.map_err(|err| format!("could not run {display}: {err}"))?;
if output.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&output.stderr);
let text = stderr.trim();
if text.is_empty() {
Err(format!("{display} failed with {}", output.status))
} else {
Err(format!("{display} failed: {text}"))
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::net::Ipv4Addr;
use super::*;
fn plan() -> BroadcastRulesPlan {
BroadcastRulesPlan {
interface: "utun5".into(),
source: Ipv4Addr::new(10, 13, 37, 142),
range: "10.13.37.0/24".parse().unwrap(),
}
}
#[test]
fn the_route_is_the_limited_broadcast_out_the_interface() {
assert_eq!(
route_add_args("utun5"),
vec![
"-q",
"-n",
"add",
"-host",
"255.255.255.255",
"-interface",
"utun5"
]
);
assert_eq!(
route_del_args("utun5"),
vec![
"-q",
"-n",
"delete",
"-host",
"255.255.255.255",
"-interface",
"utun5"
]
);
}
#[test]
fn the_firewall_todo_names_the_range_and_interface() {
let todo = firewall_todo(&plan());
assert!(todo.contains("10.13.37.0/24"), "{todo}");
assert!(todo.contains("utun5"), "{todo}");
assert!(todo.contains("pf"), "{todo}");
// Deferred, so the firewall half is always reported as not in place;
// the route runs for real and so is left to an on-host check. Running
// `route` here would need root and would change the host routing
// table, which the default test suite must never do.
}
}
@@ -0,0 +1,360 @@
//! Host rules that let a game's LAN discovery reach the overlay interface.
//!
//! Broadcast fanout happens at local TUN ingress (see [`super::broadcast`]),
//! so a limited broadcast only reaches Tsunagi if the operating system routes
//! it through the overlay interface, and a discovery *reply* only reaches the
//! game if the host firewall lets it in. Left to the user that is two manual
//! steps per host that nobody remembers, so when a network's broadcast is on
//! the agent installs exactly these two things for the interface it owns:
//!
//! * a `255.255.255.255/32` route through the overlay interface, with the
//! overlay address as its preferred source, so the game's discovery packet
//! leaves with a source address the router recognises;
//! * an inbound firewall allowance for **UDP only**, from the overlay range
//! only, on that interface only. Replies to a broadcast come from a
//! different address than the one the request was sent to, so a stateful
//! firewall never matches them to the request. Opening every protocol
//! would expose services such as SSH to every member of the network, which
//! is not what a game needs.
//!
//! # Why this is allowed here
//!
//! The architecture says plugins never touch routing or firewall settings,
//! and nothing a peer announces becomes an OS setting. This is neither: it is
//! the system level, which owns the interface, configuring that interface from
//! values it computed itself (the interface name is derived locally, the
//! address and range come from signed state it already verified). Every object
//! is tagged `tsunagi:<interface>`, reversible, and removed when broadcast is
//! turned off, the network leaves, or the agent stops. A crash can leave a
//! firewall rule behind; the next apply for the same interface removes it.
//!
//! # Failure is contained
//!
//! Neither step can stop the agent. Each reports its own outcome, so a missing
//! firewall tool still leaves the route in place and says what is missing.
//!
//! # Why a trait
//!
//! Linux, Windows and the tests do this very differently, while *what* to
//! install is the same everywhere and lives in [`BroadcastRulesPlan`] and
//! [`rules_summary`], tested on every platform.
use std::net::Ipv4Addr;
use std::sync::{Arc, Mutex};
use crate::BoxFuture;
use crate::state::Ipv4Range;
mod exit;
pub use exit::{
ExitHostPlan, ExitHostReport, ExitHostRules, MockExitRules, exit_tag, exit_tag_prefix,
offer_detail,
};
#[cfg(all(feature = "tun-device", target_os = "linux"))]
mod linux;
#[cfg(all(feature = "tun-device", target_os = "linux"))]
mod linux_exit;
#[cfg(all(feature = "tun-device", target_os = "linux"))]
pub use linux::LinuxHostRules;
// Compiled everywhere so the scripts it builds are tested everywhere; only
// the type that runs them is exported on Windows.
mod windows;
#[cfg(all(feature = "tun-device", target_os = "windows"))]
pub use windows::WindowsHostRules;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
mod macos;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
pub use macos::MacosHostRules;
/// What the host should be told for one interface.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BroadcastRulesPlan {
/// The overlay interface. Derived locally from the network id, never from
/// anything a peer said.
pub interface: String,
/// This agent's overlay address, the route's preferred source.
pub source: Ipv4Addr,
/// The overlay range inbound UDP is admitted from.
pub range: Ipv4Range,
}
impl BroadcastRulesPlan {
/// The tag every object carries, so it can be found again and nothing
/// else is ever touched.
pub fn tag(&self) -> String {
tag_for(&self.interface)
}
}
/// The tag for an interface's objects.
pub fn tag_for(interface: &str) -> String {
format!("tsunagi:{interface}")
}
/// What happened to one of the two host objects.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RuleOutcome {
/// It is in place.
Applied,
/// It is not, with a reason a person can act on.
Failed(String),
}
impl RuleOutcome {
/// Whether the object is in place.
pub fn is_applied(&self) -> bool {
matches!(self, Self::Applied)
}
}
/// What applying a plan achieved.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BroadcastRulesReport {
/// What was asked for.
pub plan: BroadcastRulesPlan,
/// The `255.255.255.255` route.
pub route: RuleOutcome,
/// The inbound firewall allowance.
pub firewall: RuleOutcome,
}
impl BroadcastRulesReport {
/// Whether both objects are in place.
pub fn is_ok(&self) -> bool {
self.route.is_applied() && self.firewall.is_applied()
}
/// One line for `status`.
pub fn summary(&self) -> String {
rules_summary(&self.route, &self.firewall)
}
}
/// The status line for a pair of outcomes.
pub fn rules_summary(route: &RuleOutcome, firewall: &RuleOutcome) -> String {
match (route, firewall) {
(RuleOutcome::Applied, RuleOutcome::Applied) => "ok (route + firewall)".to_string(),
(route, firewall) => {
let part = |what: &str, outcome: &RuleOutcome| match outcome {
RuleOutcome::Applied => format!("{what} ok"),
RuleOutcome::Failed(reason) => format!("{what} failed: {reason}"),
};
format!(
"incomplete; {}; {}",
part("route", route),
part("firewall", firewall)
)
}
}
}
/// Installs and removes the host rules for an overlay interface.
///
/// Both calls are idempotent. Neither can fail as a whole: a step that cannot
/// be done is reported in the [`BroadcastRulesReport`], because the caller's
/// only useful response is to say so.
pub trait BroadcastHostRules: Send + Sync + std::fmt::Debug + 'static {
/// A short name used in diagnostics.
fn name(&self) -> &str;
/// Makes the host match the plan, replacing whatever this agent put there
/// for the same interface before.
fn apply<'a>(&'a self, plan: &'a BroadcastRulesPlan) -> BoxFuture<'a, BroadcastRulesReport>;
/// Removes what this agent put there for the interface. Succeeds when
/// nothing is there: it runs on the shutdown path.
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()>;
/// The exit-node rules that go with these, when the platform has them.
///
/// One object per host, because both halves need the same privileged
/// helpers; the default is that this platform has none, which is said
/// where it matters rather than pretended.
fn exit_rules(&self) -> Option<&dyn ExitHostRules> {
None
}
}
/// The platform has no implementation, which is said rather than pretended.
#[derive(Debug, Default)]
pub struct UnsupportedHostRules;
impl BroadcastHostRules for UnsupportedHostRules {
fn name(&self) -> &str {
"unsupported"
}
fn apply<'a>(&'a self, plan: &'a BroadcastRulesPlan) -> BoxFuture<'a, BroadcastRulesReport> {
Box::pin(async move {
let reason = "host rules are not implemented on this platform; route \
255.255.255.255 through the overlay interface and allow inbound \
UDP from the overlay range yourself"
.to_string();
BroadcastRulesReport {
plan: plan.clone(),
route: RuleOutcome::Failed(reason.clone()),
firewall: RuleOutcome::Failed(reason),
}
})
}
fn clear<'a>(&'a self, _interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {})
}
}
/// What a [`MockHostRules`] currently holds, per interface.
pub type MockRulesState = std::collections::BTreeMap<String, BroadcastRulesPlan>;
/// An in-memory host, for tests: nothing is touched.
#[derive(Debug, Clone, Default)]
pub struct MockHostRules {
exit: MockExitRules,
state: Arc<Mutex<MockRulesState>>,
calls: Arc<Mutex<Vec<String>>>,
firewall_failure: Arc<Mutex<Option<String>>>,
}
impl MockHostRules {
/// An empty host.
pub fn new() -> Self {
Self {
exit: MockExitRules::new(),
..Self::default()
}
}
/// The exit-node half of this host, for inspecting and steering tests.
pub fn exit(&self) -> &MockExitRules {
&self.exit
}
/// Makes the firewall step fail with this reason from now on.
pub fn fail_firewall(&self, reason: impl Into<String>) {
*lock(&self.firewall_failure) = Some(reason.into());
}
/// What is installed now.
pub fn installed(&self) -> MockRulesState {
lock(&self.state).clone()
}
/// Every `apply:<interface>` / `clear:<interface>` call, in order.
pub fn calls(&self) -> Vec<String> {
lock(&self.calls).clone()
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
match mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
impl BroadcastHostRules for MockHostRules {
fn name(&self) -> &str {
"mock"
}
fn exit_rules(&self) -> Option<&dyn ExitHostRules> {
Some(&self.exit)
}
fn apply<'a>(&'a self, plan: &'a BroadcastRulesPlan) -> BoxFuture<'a, BroadcastRulesReport> {
Box::pin(async move {
lock(&self.calls).push(format!("apply:{}", plan.interface));
lock(&self.state).insert(plan.interface.clone(), plan.clone());
let firewall = match lock(&self.firewall_failure).clone() {
Some(reason) => RuleOutcome::Failed(reason),
None => RuleOutcome::Applied,
};
BroadcastRulesReport {
plan: plan.clone(),
route: RuleOutcome::Applied,
firewall,
}
})
}
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
lock(&self.calls).push(format!("clear:{interface}"));
lock(&self.state).remove(interface);
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn plan() -> BroadcastRulesPlan {
BroadcastRulesPlan {
interface: "tsun0".into(),
source: Ipv4Addr::new(10, 13, 37, 142),
range: "10.13.37.0/24".parse().unwrap(),
}
}
#[test]
fn every_object_is_tagged_with_its_interface() {
assert_eq!(plan().tag(), "tsunagi:tsun0");
assert_eq!(tag_for("tsun7"), "tsunagi:tsun7");
}
#[test]
fn the_summary_says_which_half_is_missing() {
assert_eq!(
rules_summary(&RuleOutcome::Applied, &RuleOutcome::Applied),
"ok (route + firewall)"
);
let line = rules_summary(
&RuleOutcome::Applied,
&RuleOutcome::Failed("iptables not found".into()),
);
assert!(line.contains("route ok"), "{line}");
assert!(
line.contains("firewall failed: iptables not found"),
"{line}"
);
let report = BroadcastRulesReport {
plan: plan(),
route: RuleOutcome::Failed("denied".into()),
firewall: RuleOutcome::Applied,
};
assert!(!report.is_ok());
assert!(report.summary().contains("route failed: denied"));
}
#[tokio::test]
async fn the_mock_applies_replaces_and_clears_idempotently() {
let rules = MockHostRules::new();
let first = rules.apply(&plan()).await;
assert!(first.is_ok());
assert_eq!(rules.installed().len(), 1);
let mut moved = plan();
moved.source = Ipv4Addr::new(10, 13, 37, 7);
rules.apply(&moved).await;
assert_eq!(rules.installed()["tsun0"].source, moved.source);
rules.clear("tsun0").await;
rules.clear("tsun0").await;
assert!(rules.installed().is_empty());
assert_eq!(rules.calls().len(), 4);
}
#[tokio::test]
async fn an_unsupported_platform_reports_both_steps_failed() {
let report = UnsupportedHostRules.apply(&plan()).await;
assert!(!report.route.is_applied());
assert!(!report.firewall.is_applied());
UnsupportedHostRules.clear("tsun0").await;
}
}
@@ -0,0 +1,272 @@
//! Broadcast host rules on Windows: a route and a Defender Firewall rule.
//!
//! Both are written with the `NetTCPIP` and `NetSecurity` PowerShell modules,
//! run by absolute path from `%SystemRoot%` like the DNS publisher does: the
//! in-process alternatives are `unsafe` IP Helper calls this crate forbids.
//!
//! * The route is `255.255.255.255/32` on the overlay interface, on-link, with
//! metric 1, written to the active store only so it cannot outlive a reboot.
//! Windows picks the interface's own address as the source of a packet sent
//! along it; with several networks on one interface that is whichever
//! address Windows treats as primary, which this cannot choose.
//! * The firewall rule allows inbound UDP on that interface alias from the
//! overlay range only, in every profile. Wintun adapters are usually
//! classified as a public network, where inbound UDP is otherwise dropped.
//! Its display name is the tag `tsunagi:<interface>`, which is how it is
//! found again, replaced and removed.
//!
//! Every value in a script is one this agent produced: the interface name is
//! derived from the network id and the range comes from signed state. They are
//! still single-quoted and escaped, so nothing could be read as PowerShell
//! rather than as data. Changing routes and firewall rules needs an elevated
//! process, which is reported as its own cause.
//!
//! This module is compiled everywhere so its script building is tested
//! everywhere. It could not be run against a real Windows host while it was
//! written.
#![cfg_attr(not(target_os = "windows"), allow(dead_code))]
use crate::BoxFuture;
use super::{BroadcastHostRules, BroadcastRulesPlan, BroadcastRulesReport, RuleOutcome, tag_for};
/// A PowerShell single-quoted literal, with any embedded quote doubled.
fn ps_literal(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
/// The step that makes the route match the plan.
///
/// A leftover route (from a crashed run, or for an old interface index) is
/// removed first so the result is exactly one route.
fn route_script(plan: &BroadcastRulesPlan) -> String {
let alias = ps_literal(&plan.interface);
format!(
"$ErrorActionPreference = 'Stop'\n\
$alias = {alias}\n\
Get-NetRoute -DestinationPrefix '255.255.255.255/32' -InterfaceAlias $alias \
-PolicyStore ActiveStore -ErrorAction SilentlyContinue | \
Remove-NetRoute -Confirm:$false -ErrorAction SilentlyContinue\n\
New-NetRoute -DestinationPrefix '255.255.255.255/32' -InterfaceAlias $alias \
-NextHop '0.0.0.0' -RouteMetric 1 -PolicyStore ActiveStore | Out-Null\n"
)
}
/// The step that makes the firewall hold exactly one rule for the interface.
fn firewall_script(plan: &BroadcastRulesPlan) -> String {
let alias = ps_literal(&plan.interface);
let tag = ps_literal(&plan.tag());
let range = ps_literal(&plan.range.to_string());
format!(
"$ErrorActionPreference = 'Stop'\n\
$alias = {alias}\n\
$tag = {tag}\n\
Get-NetFirewallRule -DisplayName $tag -ErrorAction SilentlyContinue | \
Remove-NetFirewallRule -ErrorAction SilentlyContinue\n\
New-NetFirewallRule -DisplayName $tag -Direction Inbound -Action Allow -Protocol UDP \
-InterfaceAlias $alias -RemoteAddress {range} -Profile Any | Out-Null\n"
)
}
/// The script that removes both, ignoring what is not there.
fn clear_script(interface: &str) -> String {
let alias = ps_literal(interface);
let tag = ps_literal(&tag_for(interface));
format!(
"$ErrorActionPreference = 'SilentlyContinue'\n\
Get-NetRoute -DestinationPrefix '255.255.255.255/32' -InterfaceAlias {alias} \
-PolicyStore ActiveStore | Remove-NetRoute -Confirm:$false\n\
Get-NetFirewallRule -DisplayName {tag} | Remove-NetFirewallRule\n\
exit 0\n"
)
}
/// Turns what PowerShell said into a cause a person can act on.
fn classify(what: &str, text: &str) -> String {
let lower = text.to_ascii_lowercase();
if lower.contains("access is denied")
|| lower.contains("requires elevation")
|| lower.contains("run as administrator")
|| lower.contains("permissiondenied")
{
format!("{what}: Windows refused; run tsng from an elevated (administrator) process")
} else if lower.contains("is not recognized") || lower.contains("commandnotfound") {
format!("{what}: the NetTCPIP/NetSecurity PowerShell modules are not available: {text}")
} else if text.is_empty() {
format!("{what}: failed without saying why")
} else {
format!("{what}: {text}")
}
}
/// The absolute path to Windows PowerShell, so nothing on `PATH` can stand in
/// for it.
fn powershell_path() -> std::path::PathBuf {
let root = std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into());
std::path::Path::new(&root)
.join("System32")
.join("WindowsPowerShell")
.join("v1.0")
.join("powershell.exe")
}
/// Runs a script and returns what it said on failure.
fn run_powershell(script: &str) -> Result<(), String> {
let output = std::process::Command::new(powershell_path())
.args([
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-Command",
script,
])
.output()
.map_err(|err| format!("could not run PowerShell: {err}"))?;
if output.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&output.stderr);
let text = if stderr.trim().is_empty() {
String::from_utf8_lossy(&output.stdout).trim().to_string()
} else {
stderr.trim().to_string()
};
Err(text)
}
async fn step(what: &'static str, script: String) -> RuleOutcome {
match tokio::task::spawn_blocking(move || run_powershell(&script)).await {
Ok(Ok(())) => RuleOutcome::Applied,
Ok(Err(text)) => RuleOutcome::Failed(classify(what, &text)),
Err(err) => RuleOutcome::Failed(format!("{what}: the task failed: {err}")),
}
}
/// The Windows implementation, through PowerShell.
#[derive(Debug, Default)]
pub struct WindowsHostRules;
impl WindowsHostRules {
/// Creates it. Nothing is run until [`BroadcastHostRules::apply`].
pub fn new() -> Self {
Self
}
}
impl BroadcastHostRules for WindowsHostRules {
fn name(&self) -> &str {
"windows-netsecurity"
}
fn apply<'a>(&'a self, plan: &'a BroadcastRulesPlan) -> BoxFuture<'a, BroadcastRulesReport> {
Box::pin(async move {
let route = step("route", route_script(plan)).await;
let firewall = step("firewall", firewall_script(plan)).await;
BroadcastRulesReport {
plan: plan.clone(),
route,
firewall,
}
})
}
fn clear<'a>(&'a self, interface: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
if let RuleOutcome::Failed(reason) = step("clear", clear_script(interface)).await {
tracing::warn!(
interface,
"cannot remove the broadcast host rules: {reason}"
);
}
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::net::Ipv4Addr;
fn plan() -> BroadcastRulesPlan {
BroadcastRulesPlan {
interface: "tsun0".into(),
source: Ipv4Addr::new(10, 13, 37, 142),
range: "10.13.37.0/24".parse().unwrap(),
}
}
#[test]
fn a_quote_in_a_value_cannot_escape_the_literal() {
assert_eq!(ps_literal("a'b"), "'a''b'");
let mut hostile = plan();
hostile.interface = "x'; Remove-Item C:\\ #".into();
let script = route_script(&hostile);
assert!(
script.contains("$alias = 'x''; Remove-Item C:\\ #'"),
"{script}"
);
assert!(firewall_script(&hostile).contains("'x''; Remove-Item C:\\ #'"));
}
#[test]
fn the_route_is_on_link_metric_one_in_the_active_store_and_replaces_its_own() {
let script = route_script(&plan());
assert!(script.contains("'255.255.255.255/32'"));
assert!(script.contains("-NextHop '0.0.0.0'"));
assert!(script.contains("-RouteMetric 1"));
assert!(script.contains("-PolicyStore ActiveStore"));
assert!(
script.find("Remove-NetRoute").unwrap() < script.find("New-NetRoute").unwrap(),
"a stale route goes before the new one:\n{script}"
);
}
#[test]
fn the_firewall_rule_is_udp_only_from_the_overlay_range_on_this_interface() {
let script = firewall_script(&plan());
assert!(script.contains("-Protocol UDP"));
assert!(script.contains("-Direction Inbound -Action Allow"));
assert!(script.contains("-InterfaceAlias $alias"));
assert!(script.contains("-RemoteAddress '10.13.37.0/24'"));
assert!(script.contains("$tag = 'tsunagi:tsun0'"));
assert!(!script.contains("-Protocol Any") && !script.contains("-Protocol TCP"));
assert!(
script.find("Remove-NetFirewallRule").unwrap()
< script.find("New-NetFirewallRule").unwrap(),
"{script}"
);
}
#[test]
fn clearing_touches_only_our_route_and_our_tagged_rule() {
let script = clear_script("tsun0");
assert!(script.contains("-InterfaceAlias 'tsun0'"));
assert!(script.contains("Get-NetFirewallRule -DisplayName 'tsunagi:tsun0'"));
assert!(script.contains("exit 0"));
}
#[test]
fn a_refusal_names_elevation_and_a_missing_module_names_the_module() {
assert!(classify("route", "Access is denied").contains("elevated"));
assert!(
classify(
"firewall",
"The term 'New-NetFirewallRule' is not recognized"
)
.contains("modules are not available")
);
assert_eq!(classify("route", "boom"), "route: boom");
assert!(classify("route", "").contains("without saying why"));
}
#[test]
fn powershell_is_found_by_absolute_path() {
let path = powershell_path();
assert!(path.ends_with("powershell.exe"));
assert!(path.to_string_lossy().contains("System32"), "{path:?}");
}
}
File diff suppressed because it is too large. Load diff
+80
View File
@@ -0,0 +1,80 @@
//! The overlay interface: the one network interface an agent owns.
//!
//! One agent, one interface. It belongs to the system level rather than to
//! any protocol, and that is the whole point: a packet leaving it is routed
//! to whichever peer owns its destination address, over whichever protocol
//! currently has a link to that peer. Several protocols can be live at once
//! without arguing over who holds the address, because neither of them
//! holds it — the agent does.
//!
//! What lives here:
//!
//! * [`provision`] creates the interface and configures it, and removes it
//! again. Platform mechanics behind one decision function.
//! * [`hostrules`] installs the route and firewall allowance that let a
//! game's LAN discovery reach the interface, while broadcast is on.
//! * [`tun`] is the packet interface itself, real or in memory.
//! * [`packet`] reads just enough of an IP header to route by it.
//! * [`config`] names interfaces and describes addresses.
//!
//! What does not live here: encryption, peer discovery, and how a packet
//! actually reaches another machine. Those belong to a protocol plugin,
//! which this level hands packets to and takes packets from.
/// Why the overlay interface cannot be brought into the state asked for.
///
/// Its own error rather than the plugin one it used to borrow: this level
/// owns the interface now, and a plugin failing is a different event from
/// the interface failing.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum OverlayError {
/// The interface cannot be created or configured right now, with the
/// reason and, where there is one, what to do about it.
#[error("overlay unavailable: {0}")]
Unavailable(String),
/// Anything else.
#[error("{0}")]
Other(String),
}
pub mod broadcast;
pub mod config;
pub mod exit;
pub mod hostrules;
pub mod interface;
pub mod packet;
pub mod provision;
pub mod router;
pub mod tun;
pub use config::{Cidr, DEFAULT_INTERFACE_PREFIX, MAX_INTERFACE_NAME_LEN, interface_name};
pub use exit::{ExitRulesReport, RuleSetReport};
pub use hostrules::{
BroadcastHostRules, BroadcastRulesPlan, BroadcastRulesReport, ExitHostPlan, ExitHostReport,
ExitHostRules, MockExitRules, MockHostRules, RuleOutcome, UnsupportedHostRules,
};
pub use interface::{Counters, Interface, PacketCarrier, Rejected};
pub use packet::IpHeader;
pub use provision::{
Changes, InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, ManagedTunFactory,
MockHost, MockProvisioner, Privilege, Provisioned, UnsupportedProvisioner, plan_changes,
probe_net_admin,
};
pub use router::{ExitPolicy, NetworkRoutes, Route, RouteError, RoutingTable};
pub use tun::{MemoryTun, MemoryTunFactory, TunDevice, TunFactory, TunRequest, address_is_local};
#[cfg(all(feature = "tun-device", target_os = "linux"))]
pub use hostrules::LinuxHostRules;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
pub use hostrules::MacosHostRules;
#[cfg(all(feature = "tun-device", target_os = "windows"))]
pub use hostrules::WindowsHostRules;
#[cfg(all(feature = "tun-device", target_os = "linux"))]
pub use provision::NetlinkProvisioner;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
pub use provision::UtunProvisioner;
#[cfg(all(feature = "tun-device", target_os = "windows"))]
pub use provision::WintunProvisioner;
+141
View File
@@ -0,0 +1,141 @@
//! The little bit of IP parsing the data plane needs.
//!
//! Two questions only: which peer should carry this packet, and did the packet
//! that came back really come from that peer? Everything is bounds checked and
//! nothing here can panic on a hostile packet.
use std::net::{Ipv4Addr, Ipv6Addr};
/// The addresses of an IP packet, as far as routing cares.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IpHeader {
/// An IPv4 packet.
V4 {
/// Source address.
source: Ipv4Addr,
/// Destination address.
destination: Ipv4Addr,
},
/// An IPv6 packet.
V6 {
/// Source address.
source: Ipv6Addr,
/// Destination address.
destination: Ipv6Addr,
},
}
impl IpHeader {
/// Reads the addresses out of a packet, or `None` if it is not one.
pub fn parse(packet: &[u8]) -> Option<Self> {
let version = packet.first()? >> 4;
match version {
4 => {
let source: [u8; 4] = packet.get(12..16)?.try_into().ok()?;
let destination: [u8; 4] = packet.get(16..20)?.try_into().ok()?;
Some(IpHeader::V4 {
source: Ipv4Addr::from(source),
destination: Ipv4Addr::from(destination),
})
}
6 => {
let source: [u8; 16] = packet.get(8..24)?.try_into().ok()?;
let destination: [u8; 16] = packet.get(24..40)?.try_into().ok()?;
Some(IpHeader::V6 {
source: Ipv6Addr::from(source),
destination: Ipv6Addr::from(destination),
})
}
_ => None,
}
}
/// The destination, when the packet is IPv6.
pub fn v6_destination(&self) -> Option<Ipv6Addr> {
match self {
IpHeader::V6 { destination, .. } => Some(*destination),
IpHeader::V4 { .. } => None,
}
}
/// The source, when the packet is IPv6.
pub fn v6_source(&self) -> Option<Ipv6Addr> {
match self {
IpHeader::V6 { source, .. } => Some(*source),
IpHeader::V4 { .. } => None,
}
}
/// The destination, whichever family it is.
pub fn destination(&self) -> std::net::IpAddr {
match self {
IpHeader::V4 { destination, .. } => std::net::IpAddr::V4(*destination),
IpHeader::V6 { destination, .. } => std::net::IpAddr::V6(*destination),
}
}
/// The source, whichever family it is.
pub fn source(&self) -> std::net::IpAddr {
match self {
IpHeader::V4 { source, .. } => std::net::IpAddr::V4(*source),
IpHeader::V6 { source, .. } => std::net::IpAddr::V6(*source),
}
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn ipv6_packet(source: Ipv6Addr, destination: Ipv6Addr) -> Vec<u8> {
let mut packet = vec![0u8; 48];
packet[0] = 6 << 4;
packet[8..24].copy_from_slice(&source.octets());
packet[24..40].copy_from_slice(&destination.octets());
packet
}
#[test]
fn ipv6_addresses_are_read_correctly() {
let source: Ipv6Addr = "fd00::1".parse().unwrap();
let destination: Ipv6Addr = "fd00::2".parse().unwrap();
let header = IpHeader::parse(&ipv6_packet(source, destination)).unwrap();
assert_eq!(header.v6_source(), Some(source));
assert_eq!(header.v6_destination(), Some(destination));
}
#[test]
fn ipv4_addresses_are_read_correctly() {
let mut packet = vec![0u8; 20];
packet[0] = 4 << 4;
packet[12..16].copy_from_slice(&[10, 0, 0, 1]);
packet[16..20].copy_from_slice(&[10, 0, 0, 2]);
let header = IpHeader::parse(&packet).unwrap();
assert_eq!(
header,
IpHeader::V4 {
source: Ipv4Addr::new(10, 0, 0, 1),
destination: Ipv4Addr::new(10, 0, 0, 2),
}
);
// The overlay is IPv6, so the v6 accessors correctly report nothing.
assert_eq!(header.v6_destination(), None);
}
#[test]
fn truncated_and_nonsense_packets_are_rejected_without_panicking() {
assert!(IpHeader::parse(&[]).is_none());
assert!(IpHeader::parse(&[0x60]).is_none());
assert!(IpHeader::parse(&[0x40; 19]).is_none(), "short IPv4");
assert!(IpHeader::parse(&[0x60; 39]).is_none(), "short IPv6");
assert!(IpHeader::parse(&[0x00; 64]).is_none(), "version 0");
assert!(IpHeader::parse(&[0xf0; 64]).is_none(), "version 15");
// Every possible first byte is safe to feed in.
for byte in 0..=u8::MAX {
let _ = IpHeader::parse(&[byte; 64]);
let _ = IpHeader::parse(&[byte]);
}
}
}
@@ -0,0 +1,271 @@
//! The adapter between the plugin's view of a packet interface and the
//! host-management view.
//!
//! The plugin asks a [`TunFactory`] for a device and knows nothing else. This
//! factory answers by reconciling the host — creating the interface, fixing
//! up whatever an earlier run left behind, assigning the addresses — and
//! handing back the device that came out of it.
use std::sync::Arc;
use crate::BoxFuture;
use crate::overlay::OverlayError;
use super::super::hostrules::BroadcastHostRules;
use super::super::tun::{TunDevice, TunFactory, TunRequest};
use super::{InterfacePlan, InterfaceProvisioner};
/// Turns a [`TunRequest`] into the plan for a host interface.
fn plan_for(request: &TunRequest) -> Result<InterfacePlan, OverlayError> {
// Every address, because the plan is exhaustive: the provisioner adds
// what is missing and removes what is not in it. Passing one network's
// address would take every other network's off the host.
Ok(InterfacePlan::new(
request.name.clone(),
request.mtu,
request.addresses.clone(),
))
}
/// A [`TunFactory`] backed by an [`InterfaceProvisioner`].
#[derive(Debug)]
pub struct ManagedTunFactory {
provisioner: Arc<dyn InterfaceProvisioner>,
host_rules: Option<Arc<dyn BroadcastHostRules>>,
}
impl ManagedTunFactory {
/// Wraps a provisioner.
pub fn new(provisioner: Arc<dyn InterfaceProvisioner>) -> Self {
Self {
provisioner,
host_rules: None,
}
}
/// Lets the agent install the broadcast route and firewall allowance for
/// the interface this factory manages.
pub fn with_host_rules(mut self, rules: Arc<dyn BroadcastHostRules>) -> Self {
self.host_rules = Some(rules);
self
}
/// The provisioner underneath.
pub fn provisioner(&self) -> &Arc<dyn InterfaceProvisioner> {
&self.provisioner
}
}
impl TunFactory for ManagedTunFactory {
fn name(&self) -> &str {
self.provisioner.name()
}
/// It creates the interface on the host and holds it open.
fn on_host(&self) -> bool {
true
}
fn host_rules(&self) -> Option<Arc<dyn BroadcastHostRules>> {
self.host_rules.clone()
}
fn create<'a>(
&'a self,
request: TunRequest,
) -> BoxFuture<'a, Result<Arc<dyn TunDevice>, OverlayError>> {
Box::pin(async move {
let plan = plan_for(&request)?;
let provisioned = self.provisioner.reconcile(&plan).await?;
tracing::info!(
interface = %plan.name,
changes = %provisioned.changes.summary(),
"overlay interface reconciled"
);
provisioned.device.ok_or_else(|| {
// Reaching here would mean the interface already existed and
// was held open by us, which cannot be true on the path that
// creates a device.
OverlayError::Other(format!(
"interface `{}` was reconciled but no device came back",
plan.name
))
})
})
}
fn reconfigure<'a>(&'a self, request: TunRequest) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move {
let plan = plan_for(&request)?;
let provisioned = self.provisioner.reconcile(&plan).await?;
if !provisioned.changes.is_empty() {
tracing::info!(
interface = %plan.name,
changes = %provisioned.changes.summary(),
"overlay interface updated"
);
}
Ok(())
})
}
fn destroy<'a>(&'a self, name: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
match self.provisioner.remove(name).await {
Ok(()) => tracing::info!(interface = %name, "overlay interface removed"),
Err(err) => {
tracing::warn!(interface = %name, %err, "cannot remove the overlay interface")
}
}
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::net::Ipv4Addr;
use super::super::super::config::Cidr;
use super::super::{LinkKind, MockHost, MockProvisioner};
use super::*;
fn request(v4: Option<Ipv4Addr>) -> TunRequest {
addressed(v4.into_iter().collect())
}
fn addressed(v4: Vec<Ipv4Addr>) -> TunRequest {
TunRequest {
name: "tsunfactory".into(),
addresses: v4
.into_iter()
.map(|address| Cidr::new(address.into(), 24).unwrap())
.collect(),
mtu: 1280,
}
}
#[tokio::test]
async fn creating_a_device_provisions_the_host_and_returns_it() {
let provisioner = Arc::new(MockProvisioner::default());
let factory = ManagedTunFactory::new(provisioner.clone());
let device = factory
.create(request(Some(Ipv4Addr::new(10, 13, 37, 69))))
.await
.unwrap();
assert_eq!(device.name(), "tsunfactory");
assert_eq!(device.mtu(), 1280);
let state = provisioner.host().get("tsunfactory").unwrap();
assert_eq!(state.kind, LinkKind::Tun);
assert_eq!(state.mtu, 1280);
assert_eq!(state.addresses.len(), 1, "the overlay address is assigned");
}
#[tokio::test]
async fn a_reallocated_address_is_applied_to_the_live_interface() {
let provisioner = Arc::new(MockProvisioner::default());
let factory = ManagedTunFactory::new(provisioner.clone());
factory
.create(request(Some(Ipv4Addr::new(10, 13, 37, 69))))
.await
.unwrap();
factory
.reconfigure(request(Some(Ipv4Addr::new(10, 13, 37, 70))))
.await
.unwrap();
let state = provisioner.host().get("tsunfactory").unwrap();
assert!(state.attached, "the interface was not recreated");
let addresses: Vec<String> = state
.addresses
.iter()
.map(|entry| entry.to_string())
.collect();
assert!(
addresses.contains(&"10.13.37.70/24".to_string()),
"{addresses:?}"
);
assert!(
!addresses.contains(&"10.13.37.69/24".to_string()),
"{addresses:?}"
);
}
#[tokio::test]
async fn destroying_takes_the_interface_off_the_host() {
let provisioner = Arc::new(MockProvisioner::default());
let factory = ManagedTunFactory::new(provisioner.clone());
factory.create(request(None)).await.unwrap();
factory.destroy("tsunfactory").await;
assert!(provisioner.host().names().is_empty());
}
#[tokio::test]
async fn a_host_that_cannot_be_provisioned_fails_the_create() {
let host = MockHost::new();
host.insert(
"tsunfactory",
super::super::InterfaceState {
kind: LinkKind::Foreign("bridge".into()),
attached: true,
up: true,
mtu: 1500,
addresses: Vec::new(),
},
);
let factory = ManagedTunFactory::new(Arc::new(MockProvisioner::new(host)));
let err = factory.create(request(None)).await.unwrap_err();
assert!(err.to_string().contains("bridge"), "{err}");
}
#[tokio::test]
async fn every_network_s_address_reaches_the_host_not_just_the_first() {
// One agent has one interface and a network apiece on it. Carrying
// only the first address is carrying only the first network: the
// rest have addresses the operating system has never heard of, and
// their traffic goes nowhere while the status says all is well.
let provisioner = Arc::new(MockProvisioner::default());
let factory = ManagedTunFactory::new(provisioner.clone());
factory
.create(addressed(vec![
Ipv4Addr::new(10, 13, 37, 69),
Ipv4Addr::new(10, 156, 200, 116),
]))
.await
.unwrap();
let state = provisioner.host().get("tsunfactory").unwrap();
let addresses: Vec<String> = state
.addresses
.iter()
.map(|entry| entry.to_string())
.collect();
assert!(
addresses.contains(&"10.13.37.69/24".to_string()),
"{addresses:?}"
);
assert!(
addresses.contains(&"10.156.200.116/24".to_string()),
"{addresses:?}"
);
// And leaving one network takes its address off, because the plan
// is what the interface should carry and nothing else.
factory
.reconfigure(addressed(vec![Ipv4Addr::new(10, 13, 37, 69)]))
.await
.unwrap();
let state = provisioner.host().get("tsunfactory").unwrap();
let addresses: Vec<String> = state
.addresses
.iter()
.map(|entry| entry.to_string())
.collect();
assert_eq!(addresses, vec!["10.13.37.69/24".to_string()]);
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,531 @@
//! Managing the overlay interface on macOS, through utun and `ifconfig`/`route`.
//!
//! The shape is the same as the Linux and Windows provisioners: observe what
//! is there, [`plan_changes`](super::plan_changes) to decide what to do, apply
//! the difference. Only the mechanisms differ.
//!
//! # Creating the interface
//!
//! The interface is a `utun` device, created through the `tun` dependency's
//! safe wrapper — [`open_tun`](super::super::tun) — over the in-kernel
//! `com.apple.net.utun_control` control socket. No kernel extension and no
//! Network Extension entitlement is involved; what it needs is root. The
//! device is not made persistent, so closing the last file descriptor removes
//! the interface — it goes away when the agent does, however the agent ends.
//!
//! # The kernel names the interface, not us
//!
//! Unlike Linux and Windows, a utun interface cannot be given an arbitrary
//! name: it is always `utunN`, and the unit number is assigned by the kernel.
//! So the name derived from the network id (`tsun…`) is only ever a *request*;
//! the real name is read back off the device after it is opened, and
//! everything downstream — the host rules, the resolver publication — uses the
//! real one. The agent adopts it as soon as the device is created (see
//! [`Interface`](crate::overlay::Interface)).
//!
//! # Configuring it
//!
//! Addresses, MTU and routes are applied with `ifconfig` and `route`. On Linux
//! the same work is done in process over a netlink socket; here there is no
//! equivalent this crate may call, because the in-process route is raw `ioctl`
//! and `PF_ROUTE` and that is `unsafe`. The tools are invoked by absolute path
//! from `/sbin`, so nothing on `PATH` can stand in for them, and every value
//! handed to them is one this agent derived — an interface name chosen by the
//! kernel, an address it allocated — never anything a peer said.
//!
//! # Observing it
//!
//! The interface table is read through `netdev`, a safe wrapper over
//! `getifaddrs`. An interface is matched by its real `utunN` name, which only
//! matters on reconfiguration: the first create always makes a fresh device.
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::sync::{Arc, Mutex};
use crate::BoxFuture;
use crate::overlay::OverlayError;
use super::super::config::Cidr;
use super::super::tun::{TunDevice, TunRequest, open_tun};
use super::privilege::{Privilege, probe_net_admin};
use super::{
InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned, plan_changes,
};
/// Manages the overlay interface with utun and `ifconfig`/`route`.
#[derive(Debug)]
pub struct UtunProvisioner {
/// Interfaces this process created, by their real `utunN` name, so a
/// reconfiguration adjusts them rather than trying to create them again.
ours: Mutex<Vec<String>>,
/// Devices kept alive for as long as the interface should exist. Dropping
/// one closes the last descriptor, which removes the utin interface.
held: Mutex<Vec<(String, Arc<dyn TunDevice>)>>,
}
impl UtunProvisioner {
/// Creates the provisioner, after checking the platform supports one.
///
/// Whether the process is actually root is not checked here — reading the
/// effective uid needs a libc call this crate forbids — so a missing
/// privilege surfaces at utun creation with a precise message, the same
/// way the Linux and Windows paths treat the open itself as the honest
/// answer.
pub fn new() -> Result<Self, OverlayError> {
match probe_net_admin() {
Privilege::Available => {}
Privilege::Missing(reason) => return Err(OverlayError::Unavailable(reason)),
Privilege::Unsupported => {
return Err(OverlayError::Unavailable(
"interface management is not compiled in".to_string(),
));
}
}
Ok(Self {
ours: Mutex::new(Vec::new()),
held: Mutex::new(Vec::new()),
})
}
fn is_ours(&self, name: &str) -> bool {
lock(&self.ours).iter().any(|owned| owned == name)
}
/// Opens the utun device, which is what creates it. The kernel assigns the
/// name, so the returned device reports a `utunN` that is not `plan.name`.
fn create_device(&self, plan: &InterfacePlan) -> Result<Arc<dyn TunDevice>, OverlayError> {
let request = TunRequest::bare(plan.name.clone(), plan.mtu);
open_tun(&request)
}
}
impl InterfaceProvisioner for UtunProvisioner {
fn name(&self) -> &str {
"utun"
}
fn reconcile<'a>(
&'a self,
plan: &'a InterfacePlan,
) -> BoxFuture<'a, Result<Provisioned, OverlayError>> {
Box::pin(async move {
// A reconfiguration: the plan carries the real `utunN` name the
// agent adopted after the first create, so the interface is there
// and only the addresses and MTU need bringing into line.
if self.is_ours(&plan.name) {
let current = observe(&plan.name).await;
let changes = plan_changes(&current, plan, true)?;
configure(&plan.name, &changes).await?;
return Ok(Provisioned {
changes,
device: None,
});
}
// The first create. The kernel assigns the unit, so the device is
// opened first and its real name used for everything after.
let device = self.create_device(plan)?;
let real = device.name().to_string();
// A fresh interface starts with nothing on it, so the whole plan is
// applied. `plan_changes` against an absent interface is exactly
// that decision; its `create_link` flag is already satisfied by the
// open above, so `configure` ignores it and only applies the rest.
let changes = plan_changes(&InterfaceState::absent(), plan, false)?;
// A failure here leaves an interface that exists but cannot carry
// traffic. Drop the device so the utun closes, and report, rather
// than leave a trap behind.
if let Err(err) = configure(&real, &changes).await {
drop(device);
return Err(err);
}
lock(&self.ours).push(real.clone());
lock(&self.held).push((real, Arc::clone(&device)));
Ok(Provisioned {
changes,
device: Some(device),
})
})
}
fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move {
// Dropping the device closes the last descriptor, which removes the
// utun interface and the routes that were bound to it. There is
// nothing to flush: a utun never outlives the process that holds it.
lock(&self.held).retain(|(held, _)| held != name);
lock(&self.ours).retain(|owned| owned != name);
Ok(())
})
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
match mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
/// `fe80::/10` and `169.254.0.0/16`, which macOS assigns on its own.
fn is_link_local(addr: IpAddr) -> bool {
match addr {
IpAddr::V4(addr) => addr.is_link_local(),
IpAddr::V6(addr) => (addr.segments()[0] & 0xffc0) == 0xfe80,
}
}
/// What the interface named `name` currently looks like.
///
/// Never fails: a name nothing is using reads as [`InterfaceState::absent`],
/// which is what the plan then acts on.
async fn observe(name: &str) -> InterfaceState {
let name = name.to_string();
let found = tokio::task::spawn_blocking(move || {
netdev::get_interfaces()
.into_iter()
.find(|iface| iface.name == name)
})
.await
.ok()
.flatten();
let Some(iface) = found else {
return InterfaceState::absent();
};
let mut addresses: Vec<Cidr> = Vec::new();
for net in &iface.ipv4 {
if let Ok(cidr) = Cidr::new(IpAddr::V4(net.addr()), net.prefix_len())
&& !is_link_local(cidr.addr)
{
addresses.push(cidr);
}
}
for net in &iface.ipv6 {
if let Ok(cidr) = Cidr::new(IpAddr::V6(net.addr()), net.prefix_len())
&& !is_link_local(cidr.addr)
{
addresses.push(cidr);
}
}
addresses.sort();
InterfaceState {
// Only ever reached for an interface this agent created (see
// `reconcile`), so it is ours and a TUN by construction.
kind: LinkKind::Tun,
attached: iface.is_up(),
up: iface.is_up(),
mtu: iface.mtu.unwrap_or(0),
addresses,
}
}
/// Applies MTU, link state, addresses and the on-link routes they imply.
async fn configure(name: &str, changes: &super::Changes) -> Result<(), OverlayError> {
if let Some(mtu) = changes.set_mtu {
set_mtu(name, mtu).await?;
}
for cidr in &changes.remove {
// Best effort: an address that is already gone is the outcome wanted.
if let Err(err) = address_del(name, *cidr).await {
tracing::debug!(interface = %name, %cidr, %err, "could not remove address");
}
if let Err(err) = route_del(name, *cidr).await {
tracing::debug!(interface = %name, %cidr, %err, "could not remove route");
}
}
for cidr in &changes.add {
address_add(name, *cidr).await?;
// The subnet route makes the whole overlay range reach the interface;
// a point-to-point utun does not install one from the address alone. A
// host address (/32 or /128) needs none. Best effort: a route that is
// already there is the outcome wanted.
if let Err(err) = route_add(name, *cidr).await {
tracing::debug!(interface = %name, %cidr, %err, "could not add route");
}
}
if changes.bring_up {
// Best effort: a utun with a descriptor open is up already, and a
// failure to nudge it is not a reason to fail the whole reconcile.
if let Err(err) = set_up(name).await {
tracing::debug!(interface = %name, %err, "could not enable the interface");
}
}
Ok(())
}
/// The network address of a CIDR, with the host bits cleared.
fn network_of(cidr: Cidr) -> IpAddr {
match cidr.addr {
IpAddr::V4(addr) => {
let bits = u32::from(addr);
let mask = if cidr.prefix_len == 0 {
0
} else {
u32::MAX << (32 - u32::from(cidr.prefix_len))
};
IpAddr::V4(Ipv4Addr::from(bits & mask))
}
IpAddr::V6(addr) => {
let bits = u128::from(addr);
let mask = if cidr.prefix_len == 0 {
0
} else {
u128::MAX << (128 - u32::from(cidr.prefix_len))
};
IpAddr::V6(Ipv6Addr::from(bits & mask))
}
}
}
/// Whether a CIDR is a single host address, which needs no subnet route.
fn is_host_route(cidr: Cidr) -> bool {
match cidr.addr {
IpAddr::V4(_) => cidr.prefix_len >= 32,
IpAddr::V6(_) => cidr.prefix_len >= 128,
}
}
/// The `ifconfig` arguments that add an address to a utun interface.
///
/// A utun is point-to-point, so the IPv4 form names the address as its own
/// peer; `alias` lets several coexist on the one interface.
fn address_add_args(name: &str, cidr: Cidr) -> Vec<String> {
match cidr.addr {
IpAddr::V4(addr) => vec![
name.into(),
"inet".into(),
format!("{addr}/{}", cidr.prefix_len),
addr.to_string(),
"alias".into(),
],
IpAddr::V6(addr) => vec![
name.into(),
"inet6".into(),
format!("{addr}/{}", cidr.prefix_len),
"alias".into(),
],
}
}
/// The `ifconfig` arguments that remove an address from a utun interface.
fn address_del_args(name: &str, cidr: Cidr) -> Vec<String> {
let family = match cidr.addr {
IpAddr::V4(_) => "inet",
IpAddr::V6(_) => "inet6",
};
vec![
name.into(),
family.into(),
cidr.addr.to_string(),
"-alias".into(),
]
}
/// The `route` arguments that point a CIDR's subnet at the interface.
fn route_add_args(name: &str, cidr: Cidr) -> Vec<String> {
let family = match cidr.addr {
IpAddr::V4(_) => "-inet",
IpAddr::V6(_) => "-inet6",
};
vec![
"-q".into(),
"-n".into(),
"add".into(),
family.into(),
"-net".into(),
format!("{}/{}", network_of(cidr), cidr.prefix_len),
"-interface".into(),
name.into(),
]
}
/// The `route` arguments that remove that subnet route.
fn route_del_args(name: &str, cidr: Cidr) -> Vec<String> {
let family = match cidr.addr {
IpAddr::V4(_) => "-inet",
IpAddr::V6(_) => "-inet6",
};
vec![
"-q".into(),
"-n".into(),
"delete".into(),
family.into(),
"-net".into(),
format!("{}/{}", network_of(cidr), cidr.prefix_len),
"-interface".into(),
name.into(),
]
}
async fn address_add(name: &str, cidr: Cidr) -> Result<(), OverlayError> {
ifconfig(address_add_args(name, cidr)).await
}
async fn address_del(name: &str, cidr: Cidr) -> Result<(), OverlayError> {
ifconfig(address_del_args(name, cidr)).await
}
async fn route_add(name: &str, cidr: Cidr) -> Result<(), OverlayError> {
if is_host_route(cidr) {
return Ok(());
}
route(route_add_args(name, cidr)).await
}
async fn route_del(name: &str, cidr: Cidr) -> Result<(), OverlayError> {
if is_host_route(cidr) {
return Ok(());
}
route(route_del_args(name, cidr)).await
}
async fn set_mtu(name: &str, mtu: u32) -> Result<(), OverlayError> {
ifconfig(vec![name.into(), "mtu".into(), mtu.to_string()]).await
}
async fn set_up(name: &str) -> Result<(), OverlayError> {
ifconfig(vec![name.into(), "up".into()]).await
}
/// Runs `/sbin/ifconfig`, turning a non-zero exit into an error with its text.
async fn ifconfig(args: Vec<String>) -> Result<(), OverlayError> {
run("/sbin/ifconfig", args).await
}
/// Runs `/sbin/route`, turning a non-zero exit into an error with its text.
async fn route(args: Vec<String>) -> Result<(), OverlayError> {
run("/sbin/route", args).await
}
/// Runs a system tool by absolute path, so nothing on `PATH` can stand in for
/// it, and turns a non-zero exit into an error carrying what it said.
async fn run(program: &'static str, args: Vec<String>) -> Result<(), OverlayError> {
let display = format!("{program} {}", args.join(" "));
let output = tokio::task::spawn_blocking(move || {
std::process::Command::new(program).args(&args).output()
})
.await
.map_err(|err| OverlayError::Other(format!("could not run {display}: {err}")))?
.map_err(|err| OverlayError::Unavailable(format!("could not run {display}: {err}")))?;
if output.status.success() {
return Ok(());
}
let message = {
let stderr = String::from_utf8_lossy(&output.stderr);
let text = if stderr.trim().is_empty() {
String::from_utf8_lossy(&output.stdout).trim().to_string()
} else {
stderr.trim().to_string()
};
if text.is_empty() {
format!("exit code {}", output.status)
} else {
text
}
};
Err(OverlayError::Unavailable(format!(
"{display} failed: {message}"
)))
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn v4(last: u8, prefix: u8) -> Cidr {
Cidr::new(IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)), prefix).unwrap()
}
fn v6(last: u16, prefix: u8) -> Cidr {
Cidr::new(
IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, last)),
prefix,
)
.unwrap()
}
#[test]
fn a_v4_address_is_added_point_to_point_with_an_alias() {
let args = address_add_args("utun5", v4(69, 24));
assert_eq!(
args,
vec!["utun5", "inet", "10.13.37.69/24", "10.13.37.69", "alias"]
);
}
#[test]
fn a_v6_address_is_added_with_a_prefix_and_an_alias() {
let args = address_add_args("utun5", v6(1, 128));
assert_eq!(args, vec!["utun5", "inet6", "fd00::1/128", "alias"]);
}
#[test]
fn deleting_an_address_names_the_family_and_removes_the_alias() {
let args = address_del_args("utun5", v4(69, 24));
assert_eq!(args, vec!["utun5", "inet", "10.13.37.69", "-alias"]);
let args = address_del_args("utun5", v6(1, 128));
assert_eq!(args, vec!["utun5", "inet6", "fd00::1", "-alias"]);
}
#[test]
fn a_subnet_route_points_the_masked_network_at_the_interface() {
let args = route_add_args("utun5", v4(69, 24));
assert_eq!(
args,
vec![
"-q",
"-n",
"add",
"-inet",
"-net",
"10.13.37.0/24",
"-interface",
"utun5"
]
);
}
#[test]
fn a_host_address_installs_no_subnet_route() {
assert!(is_host_route(v4(69, 32)));
assert!(is_host_route(v6(1, 128)));
assert!(!is_host_route(v4(69, 24)));
assert!(!is_host_route(v6(1, 64)));
}
#[test]
fn the_network_address_clears_the_host_bits() {
assert_eq!(
network_of(v4(69, 24)),
IpAddr::V4(Ipv4Addr::new(10, 13, 37, 0))
);
assert_eq!(
network_of(Cidr::new(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 69)), 16).unwrap()),
IpAddr::V4(Ipv4Addr::new(10, 13, 0, 0))
);
assert_eq!(
network_of(v6(0x1234, 64)),
IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0))
);
}
#[test]
fn link_local_addresses_are_not_ours_to_manage() {
assert!(is_link_local(IpAddr::V6(Ipv6Addr::new(
0xfe80, 0, 0, 0, 0, 0, 0, 1
))));
assert!(is_link_local(IpAddr::V4(Ipv4Addr::new(169, 254, 1, 1))));
assert!(!is_link_local(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 1))));
assert!(!is_link_local(IpAddr::V6(Ipv6Addr::new(
0xfd00, 0, 0, 0, 0, 0, 0, 1
))));
}
}
@@ -0,0 +1,336 @@
//! An in-memory host, so provisioning is tested without touching this one.
//!
//! [`MockHost`] is a pretend `/sys/class/net`: a test can seed it with the
//! leftovers of a crashed run, or with somebody else's bridge, then check
//! what the provisioner did about it. It is also what the platforms that have
//! no provisioner yet are wired to in their own tests.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use crate::BoxFuture;
use crate::overlay::OverlayError;
use super::super::config::Cidr;
use super::super::tun::{MemoryTun, MemoryTunFactory, TunFactory, TunRequest};
use super::{
Changes, InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned,
plan_changes,
};
/// A pretend host with interfaces on it.
#[derive(Debug, Clone, Default)]
pub struct MockHost {
links: Arc<Mutex<HashMap<String, InterfaceState>>>,
}
impl MockHost {
/// An empty host.
pub fn new() -> Self {
Self::default()
}
fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, InterfaceState>> {
match self.links.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
/// Puts an interface on the host.
pub fn insert(&self, name: impl Into<String>, state: InterfaceState) {
self.lock().insert(name.into(), state);
}
/// Seeds the leftovers of a run that died: a TUN nobody holds open, still
/// carrying whatever addresses it had.
pub fn insert_stale_tun(&self, name: impl Into<String>, addresses: Vec<Cidr>) {
self.insert(
name,
InterfaceState {
kind: LinkKind::Tun,
attached: false,
up: true,
mtu: 1280,
addresses,
},
);
}
/// The state of an interface, if it exists.
pub fn get(&self, name: &str) -> Option<InterfaceState> {
self.lock().get(name).cloned()
}
/// The names currently on the host.
pub fn names(&self) -> Vec<String> {
let mut names: Vec<String> = self.lock().keys().cloned().collect();
names.sort();
names
}
}
/// Applies plans to a [`MockHost`].
#[derive(Debug, Clone)]
pub struct MockProvisioner {
host: MockHost,
ours: Arc<Mutex<Vec<String>>>,
/// The devices handed out, so a test can drive packets through them.
devices: MemoryTunFactory,
/// Set to fail every call, to exercise the error path.
failure: Option<String>,
}
impl Default for MockProvisioner {
fn default() -> Self {
Self::new(MockHost::new())
}
}
impl MockProvisioner {
/// A provisioner over a host.
pub fn new(host: MockHost) -> Self {
Self {
host,
ours: Arc::new(Mutex::new(Vec::new())),
devices: MemoryTunFactory::new(),
failure: None,
}
}
/// A provisioner that refuses everything, with this reason.
pub fn failing(reason: impl Into<String>) -> Self {
Self {
host: MockHost::new(),
ours: Arc::new(Mutex::new(Vec::new())),
devices: MemoryTunFactory::new(),
failure: Some(reason.into()),
}
}
/// The host it applies to.
pub fn host(&self) -> &MockHost {
&self.host
}
/// The device created for an interface name, if any.
pub fn device(&self, name: &str) -> Option<Arc<MemoryTun>> {
self.devices.device(name)
}
fn owned(&self) -> std::sync::MutexGuard<'_, Vec<String>> {
match self.ours.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
fn apply(&self, plan: &InterfacePlan) -> Result<Changes, OverlayError> {
if let Some(reason) = &self.failure {
return Err(OverlayError::Unavailable(reason.clone()));
}
let ours = self.owned().iter().any(|name| name == &plan.name);
let current = self
.host
.get(&plan.name)
.unwrap_or_else(InterfaceState::absent);
let changes = plan_changes(&current, plan, ours)?;
let mut state = current;
if changes.delete_link {
self.host.lock().remove(&plan.name);
state = InterfaceState::absent();
}
if changes.create_link {
state = InterfaceState {
kind: LinkKind::Tun,
// Creating it means holding it open, so it has carrier.
attached: true,
up: false,
mtu: 1500,
addresses: Vec::new(),
};
self.owned().push(plan.name.clone());
}
if let Some(mtu) = changes.set_mtu {
state.mtu = mtu;
}
if changes.bring_up {
state.up = true;
}
state
.addresses
.retain(|addr| !changes.remove.contains(addr));
state.addresses.extend(changes.add.iter().copied());
state.addresses.sort();
state.addresses.dedup();
self.host.insert(plan.name.clone(), state);
Ok(changes)
}
}
impl InterfaceProvisioner for MockProvisioner {
fn name(&self) -> &str {
"mock"
}
fn reconcile<'a>(
&'a self,
plan: &'a InterfacePlan,
) -> BoxFuture<'a, Result<Provisioned, OverlayError>> {
Box::pin(async move {
let changes = self.apply(plan)?;
let device = if changes.create_link {
Some(
self.devices
.create(TunRequest::bare(plan.name.clone(), plan.mtu))
.await?,
)
} else {
None
};
Ok(Provisioned { changes, device })
})
}
fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move {
self.host.lock().remove(name);
self.owned().retain(|owned| owned != name);
Ok(())
})
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
fn v6() -> Cidr {
Cidr {
addr: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1)),
prefix_len: 64,
}
}
fn v4(last: u8) -> Cidr {
Cidr {
addr: IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)),
prefix_len: 24,
}
}
fn plan(addresses: Vec<Cidr>) -> InterfacePlan {
InterfacePlan::new("tsunmock", 1280, addresses)
}
#[tokio::test]
async fn reconciling_an_empty_host_creates_a_configured_interface() {
let provisioner = MockProvisioner::default();
let plan = plan(vec![v6(), v4(69)]);
let changes = provisioner.reconcile(&plan).await.unwrap().changes;
assert!(changes.create_link);
let state = provisioner.host().get("tsunmock").unwrap();
assert_eq!(state.kind, LinkKind::Tun);
assert!(state.up);
assert_eq!(state.mtu, 1280);
assert_eq!(state.addresses, vec![v4(69), v6()]);
}
#[tokio::test]
async fn reconciling_is_idempotent() {
let provisioner = MockProvisioner::default();
let plan = plan(vec![v6(), v4(69)]);
provisioner.reconcile(&plan).await.unwrap();
let second = provisioner.reconcile(&plan).await.unwrap().changes;
assert!(second.is_empty(), "{second:?}");
}
#[tokio::test]
async fn a_crashed_run_is_repaired_on_the_next_start() {
let host = MockHost::new();
// What the previous run left: the interface, with the address it had
// been allocated back then.
host.insert_stale_tun("tsunmock", vec![v4(178)]);
let provisioner = MockProvisioner::new(host);
let changes = provisioner
.reconcile(&plan(vec![v6(), v4(69)]))
.await
.unwrap()
.changes;
assert!(changes.delete_link && changes.create_link);
let state = provisioner.host().get("tsunmock").unwrap();
assert_eq!(
state.addresses,
vec![v4(69), v6()],
"the stale address is gone and the current one is there"
);
assert!(state.attached, "the new interface is held open by us");
}
#[tokio::test]
async fn a_changed_allocation_is_applied_without_recreating_the_interface() {
let provisioner = MockProvisioner::default();
provisioner
.reconcile(&plan(vec![v6(), v4(69)]))
.await
.unwrap();
// The overlay agreed on a different address for us while running.
let changes = provisioner
.reconcile(&plan(vec![v6(), v4(70)]))
.await
.unwrap()
.changes;
assert!(
!changes.delete_link && !changes.create_link,
"recreating would drop every tunnel"
);
assert_eq!(changes.add, vec![v4(70)]);
assert_eq!(changes.remove, vec![v4(69)]);
assert_eq!(
provisioner.host().get("tsunmock").unwrap().addresses,
vec![v4(70), v6()]
);
}
#[tokio::test]
async fn removing_takes_the_interface_off_the_host_and_is_idempotent() {
let provisioner = MockProvisioner::default();
provisioner.reconcile(&plan(vec![v6()])).await.unwrap();
assert_eq!(provisioner.host().names(), vec!["tsunmock".to_string()]);
provisioner.remove("tsunmock").await.unwrap();
assert!(provisioner.host().names().is_empty());
provisioner.remove("tsunmock").await.unwrap();
}
#[tokio::test]
async fn a_foreign_interface_makes_reconciling_fail_and_changes_nothing() {
let host = MockHost::new();
host.insert(
"tsunmock",
InterfaceState {
kind: LinkKind::Foreign("bridge".into()),
attached: true,
up: true,
mtu: 1500,
addresses: vec![v4(1)],
},
);
let provisioner = MockProvisioner::new(host);
assert!(provisioner.reconcile(&plan(vec![v6()])).await.is_err());
let state = provisioner.host().get("tsunmock").unwrap();
assert_eq!(state.kind, LinkKind::Foreign("bridge".into()));
assert_eq!(state.addresses, vec![v4(1)], "left exactly as it was");
}
}
@@ -0,0 +1,456 @@
//! Bringing a host interface into the state the overlay needs.
//!
//! The agent used to print a list of `ip` commands and ask a human to run
//! them. That is fragile in exactly the way hand-held setup always is: the
//! interface does not survive a reboot, a changed address allocation needs
//! another manual round, and a crashed run leaves a half-configured interface
//! that the next run then trips over.
//!
//! So the agent does it itself, and the shape of that is a *reconciliation*:
//! it is handed an [`InterfacePlan`] describing what the interface should look
//! like, it observes what is actually there, and it applies the difference.
//! Running it twice changes nothing the second time, and running it after a
//! crash repairs whatever was left behind.
//!
//! # Why this is a trait
//!
//! Every platform does this differently — netlink on Linux, `SystemConfiguration`
//! on macOS, the Windows IP Helper API — while the *decision* of what to change
//! is the same everywhere. So the decision lives in [`plan_changes`], which is
//! pure and tested on every platform, and only the execution is behind
//! [`InterfaceProvisioner`].
//!
//! Five implementations:
//!
//! * `NetlinkProvisioner` on Linux, which needs `CAP_NET_ADMIN`.
//! * `WintunProvisioner` on Windows, a Wintun adapter configured with `netsh`,
//! which needs an elevated process.
//! * `UtunProvisioner` on macOS, a utun interface configured with `ifconfig`
//! and `route`, which needs root.
//! * [`MockProvisioner`], an in-memory host used by the tests.
//! * [`UnsupportedProvisioner`] elsewhere, which fails with an explanation
//! and a pointer at the manual route rather than pretending to work.
//!
//! # What it is not allowed to do
//!
//! Nothing here takes a name, an address or a command from the network. The
//! interface name is derived from the network id, the addresses come from the
//! local plugin, and an interface this agent did not create is never deleted
//! or reconfigured — see [`plan_changes`].
use std::collections::BTreeSet;
use std::sync::Arc;
use crate::BoxFuture;
use crate::overlay::OverlayError;
use super::config::Cidr;
use super::tun::TunDevice;
mod mock;
pub use mock::{MockHost, MockProvisioner};
#[cfg(all(feature = "tun-device", target_os = "linux"))]
mod linux;
#[cfg(all(feature = "tun-device", target_os = "linux"))]
pub use linux::NetlinkProvisioner;
#[cfg(all(feature = "tun-device", target_os = "linux"))]
pub(crate) use linux::RouteHandle;
#[cfg(all(feature = "tun-device", target_os = "windows"))]
mod windows;
#[cfg(all(feature = "tun-device", target_os = "windows"))]
pub use windows::WintunProvisioner;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
mod macos;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
pub use macos::UtunProvisioner;
mod privilege;
pub use privilege::{Privilege, probe_net_admin};
mod unsupported;
pub use unsupported::UnsupportedProvisioner;
mod factory;
pub use factory::ManagedTunFactory;
/// What an interface should look like.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InterfacePlan {
/// Interface name. Derived from the network id, never from the network.
pub name: String,
/// Interface MTU.
pub mtu: u32,
/// Every address the interface should carry, and no others.
pub addresses: Vec<Cidr>,
}
impl InterfacePlan {
/// Builds a plan, normalising the address list.
pub fn new(name: impl Into<String>, mtu: u32, addresses: Vec<Cidr>) -> Self {
let unique: BTreeSet<Cidr> = addresses.into_iter().collect();
Self {
name: name.into(),
mtu,
addresses: unique.into_iter().collect(),
}
}
}
/// What kind of link is sitting on a name.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LinkKind {
/// Nothing is there.
Absent,
/// A TUN interface.
Tun,
/// Something else entirely — a bridge, a physical device, a VPN from
/// another program. Never ours to touch.
Foreign(String),
}
/// What an interface currently looks like.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InterfaceState {
/// What kind of link holds the name.
pub kind: LinkKind,
/// Whether a process is attached to the TUN, which the kernel reports as
/// carrier. A leftover interface from a crashed run has none.
pub attached: bool,
/// Whether the link is administratively up.
pub up: bool,
/// The current MTU.
pub mtu: u32,
/// The addresses currently assigned.
pub addresses: Vec<Cidr>,
}
impl InterfaceState {
/// The state of a name nothing is using.
pub fn absent() -> Self {
Self {
kind: LinkKind::Absent,
attached: false,
up: false,
mtu: 0,
addresses: Vec::new(),
}
}
}
/// The steps that turn an [`InterfaceState`] into an [`InterfacePlan`].
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Changes {
/// Remove a stale interface left behind by an earlier run.
pub delete_link: bool,
/// Create the interface.
pub create_link: bool,
/// Set the MTU, when it is not already right.
pub set_mtu: Option<u32>,
/// Bring the link up.
pub bring_up: bool,
/// Addresses to add.
pub add: Vec<Cidr>,
/// Addresses to remove, because the plan no longer contains them.
pub remove: Vec<Cidr>,
}
impl Changes {
/// Whether anything at all needs doing.
pub fn is_empty(&self) -> bool {
!self.delete_link
&& !self.create_link
&& self.set_mtu.is_none()
&& !self.bring_up
&& self.add.is_empty()
&& self.remove.is_empty()
}
/// A one-line summary for a log line.
pub fn summary(&self) -> String {
if self.is_empty() {
return "already as planned".to_string();
}
let mut parts = Vec::new();
if self.delete_link {
parts.push("remove a stale interface".to_string());
}
if self.create_link {
parts.push("create the interface".to_string());
}
if let Some(mtu) = self.set_mtu {
parts.push(format!("set mtu {mtu}"));
}
if self.bring_up {
parts.push("bring it up".to_string());
}
for address in &self.add {
parts.push(format!("add {address}"));
}
for address in &self.remove {
parts.push(format!("remove {address}"));
}
parts.join(", ")
}
}
/// Decides what to change, or refuses.
///
/// `ours` says whether this process created the interface in its current run.
/// It is the whole reason this can be safe: an interface we made is adjusted
/// in place, and an interface we did not make is only ever *replaced* when it
/// is plainly abandoned.
///
/// The refusals matter more than the changes:
///
/// * A link that is not a TUN is never touched. Deriving the name from the
/// network id makes a collision with a real device unlikely, not impossible,
/// and destroying somebody's bridge because it happened to share a name is
/// not a recoverable mistake.
/// * A TUN with a process attached to it is never deleted. It is somebody
/// else's working interface — most likely another agent on this host — and
/// yanking it out from under them would break a running overlay.
///
/// What is left is a TUN with nothing attached, which is precisely the
/// footprint of a run that died: those are removed and rebuilt.
pub fn plan_changes(
current: &InterfaceState,
plan: &InterfacePlan,
ours: bool,
) -> Result<Changes, OverlayError> {
let mut changes = Changes::default();
match &current.kind {
LinkKind::Foreign(kind) => {
return Err(OverlayError::Unavailable(format!(
"`{}` already exists and is a {kind} interface, not one of ours. \
Refusing to touch it. Run with a different interface prefix.",
plan.name
)));
}
LinkKind::Tun if !ours && current.attached => {
return Err(OverlayError::Unavailable(format!(
"`{}` already exists and another process is attached to it. \
That is most likely a second agent on this host in the same \
network; give one of them a different interface prefix.",
plan.name
)));
}
LinkKind::Tun if !ours => {
// Abandoned: a TUN with no carrier is one nothing holds open. It
// is either a leftover from a run that died or an interface made
// by the old manual recipe. Either way it is replaced, which also
// discards whatever stale addresses it carried.
changes.delete_link = true;
changes.create_link = true;
}
LinkKind::Tun => {}
LinkKind::Absent => changes.create_link = true,
}
if changes.create_link {
// A fresh interface starts down, with the kernel default MTU and no
// addresses, so everything in the plan has to be applied.
changes.set_mtu = Some(plan.mtu);
changes.bring_up = true;
changes.add = plan.addresses.clone();
return Ok(changes);
}
if current.mtu != plan.mtu {
changes.set_mtu = Some(plan.mtu);
}
if !current.up {
changes.bring_up = true;
}
let wanted: BTreeSet<Cidr> = plan.addresses.iter().copied().collect();
let present: BTreeSet<Cidr> = current.addresses.iter().copied().collect();
changes.add = wanted.difference(&present).copied().collect();
changes.remove = present.difference(&wanted).copied().collect();
Ok(changes)
}
/// The result of a reconciliation.
pub struct Provisioned {
/// What was changed to get here.
pub changes: Changes,
/// The packet interface, when this call is what created it.
///
/// Creating the interface and configuring it are the same privileged act
/// and belong together, so the provisioner owns both. A reconciliation
/// that only adjusted an interface already in place returns `None`.
pub device: Option<Arc<dyn TunDevice>>,
}
impl std::fmt::Debug for Provisioned {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Provisioned")
.field("changes", &self.changes)
.field("device", &self.device.as_ref().map(|device| device.name()))
.finish()
}
}
/// Applies an [`InterfacePlan`] to the host.
///
/// Implementations are expected to be idempotent: calling [`reconcile`] twice
/// with the same plan changes nothing the second time.
///
/// [`reconcile`]: InterfaceProvisioner::reconcile
pub trait InterfaceProvisioner: Send + Sync + std::fmt::Debug + 'static {
/// A short name used in diagnostics.
fn name(&self) -> &str;
/// Brings the interface in line with the plan, and reports what it did.
fn reconcile<'a>(
&'a self,
plan: &'a InterfacePlan,
) -> BoxFuture<'a, Result<Provisioned, OverlayError>>;
/// Removes an interface this provisioner created.
///
/// Removing one that is already gone succeeds: this runs on the shutdown
/// path, where the interface having vanished is the desired outcome.
fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), OverlayError>>;
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
fn v6(last: u16) -> Cidr {
Cidr {
addr: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, last)),
prefix_len: 64,
}
}
fn v4(last: u8) -> Cidr {
Cidr {
addr: IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)),
prefix_len: 24,
}
}
fn plan() -> InterfacePlan {
InterfacePlan::new("tsuntest", 1280, vec![v6(1), v4(69)])
}
#[test]
fn an_absent_interface_is_created_and_fully_configured() {
let changes = plan_changes(&InterfaceState::absent(), &plan(), false).unwrap();
assert!(changes.create_link);
assert!(!changes.delete_link);
assert_eq!(changes.set_mtu, Some(1280));
assert!(changes.bring_up);
assert_eq!(changes.add, vec![v4(69), v6(1)]);
assert!(changes.remove.is_empty());
}
#[test]
fn an_abandoned_tun_from_a_crashed_run_is_replaced() {
// The footprint of a run that died: the interface is still there, it
// still has its old addresses, and nothing holds it open.
let current = InterfaceState {
kind: LinkKind::Tun,
attached: false,
up: true,
mtu: 1280,
addresses: vec![v4(178)],
};
let changes = plan_changes(&current, &plan(), false).unwrap();
assert!(changes.delete_link, "the stale interface goes");
assert!(changes.create_link);
// Replacing it discards the stale address, so it need not be removed
// one by one.
assert_eq!(changes.add, vec![v4(69), v6(1)]);
assert!(changes.remove.is_empty());
}
#[test]
fn a_foreign_interface_is_never_touched() {
let current = InterfaceState {
kind: LinkKind::Foreign("bridge".into()),
attached: true,
up: true,
mtu: 1500,
addresses: vec![v4(1)],
};
let err = plan_changes(&current, &plan(), false).unwrap_err();
let message = err.to_string();
assert!(message.contains("bridge"), "{message}");
assert!(message.contains("Refusing to touch it"), "{message}");
}
#[test]
fn a_tun_another_process_is_using_is_never_deleted() {
let current = InterfaceState {
kind: LinkKind::Tun,
attached: true,
up: true,
mtu: 1280,
addresses: vec![v6(1)],
};
let err = plan_changes(&current, &plan(), false).unwrap_err();
assert!(err.to_string().contains("another process"), "{err}");
}
#[test]
fn our_own_interface_is_adjusted_in_place() {
// The live case: our address allocation changed while running. The
// interface must not be recreated, or every tunnel on it would drop.
let current = InterfaceState {
kind: LinkKind::Tun,
attached: true,
up: true,
mtu: 1280,
addresses: vec![v6(1), v4(178)],
};
let changes = plan_changes(&current, &plan(), true).unwrap();
assert!(!changes.delete_link && !changes.create_link);
assert_eq!(changes.add, vec![v4(69)]);
assert_eq!(changes.remove, vec![v4(178)]);
}
#[test]
fn reconciling_an_interface_that_already_matches_changes_nothing() {
let current = InterfaceState {
kind: LinkKind::Tun,
attached: true,
up: true,
mtu: 1280,
addresses: vec![v4(69), v6(1)],
};
let changes = plan_changes(&current, &plan(), true).unwrap();
assert!(changes.is_empty(), "{changes:?}");
assert_eq!(changes.summary(), "already as planned");
}
#[test]
fn a_wrong_mtu_or_a_down_link_is_corrected() {
let current = InterfaceState {
kind: LinkKind::Tun,
attached: true,
up: false,
mtu: 1500,
addresses: vec![v4(69), v6(1)],
};
let changes = plan_changes(&current, &plan(), true).unwrap();
assert_eq!(changes.set_mtu, Some(1280));
assert!(changes.bring_up);
assert!(changes.add.is_empty() && changes.remove.is_empty());
}
#[test]
fn a_plan_deduplicates_and_orders_its_addresses() {
let plan = InterfacePlan::new("x", 1280, vec![v6(1), v4(69), v6(1)]);
assert_eq!(plan.addresses, vec![v4(69), v6(1)]);
}
}
@@ -0,0 +1,222 @@
//! Holding `CAP_NET_ADMIN` for as short a time as possible.
//!
//! Creating a TUN interface and assigning addresses to it needs
//! `CAP_NET_ADMIN`, and there is no way around that on Linux. What *is* in
//! our control is how long the process can actually use it.
//!
//! Linux splits capabilities into sets. The *permitted* set is what a process
//! may use; the *effective* set is what it may use **right now**. A process
//! can lower a capability out of effective and raise it back later, but it
//! can never add to permitted. So the agent keeps `CAP_NET_ADMIN` out of the
//! effective set and raises it only around the handful of netlink calls that
//! need it, which is a few milliseconds at startup and again whenever the
//! address allocation changes.
//!
//! Grant it with:
//!
//! ```text
//! sudo setcap cap_net_admin,cap_net_bind_service+p /usr/local/bin/tsng
//! ```
//!
//! `+p` rather than `+ep`: with `+p` the capability is permitted but not
//! effective at exec, which is exactly the resting state this module wants.
//! `+ep` also works — [`NetAdmin::acquire`] lowers it on the way in.
//!
//! # Capabilities are per thread
//!
//! `capset` affects the calling thread only, so raising one inside an async
//! block would be a bug the moment the task migrated to another worker. Every
//! caller here runs on a single dedicated thread; see
//! [`linux`](super::linux).
/// Whether this process can configure interfaces.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Privilege {
/// `CAP_NET_ADMIN` is available. The agent manages the interface itself.
Available,
/// It is not, with a description of what was found.
Missing(String),
/// This platform has no provisioner yet.
Unsupported,
}
impl Privilege {
/// Whether interfaces can be managed.
pub fn is_available(&self) -> bool {
matches!(self, Privilege::Available)
}
/// How to obtain it, for a diagnostic.
pub fn how_to_grant(program: &str) -> String {
format!(
"Grant it once with `sudo setcap cap_net_admin,cap_net_bind_service+p {program}` \
and the agent manages its own interface. Without it, run with \
`--no-tun`: the tunnels still form, they just do not reach the \
operating system."
)
}
}
#[cfg(all(feature = "tun-device", target_os = "linux"))]
pub use linux_impl::{NetAdmin, probe_net_admin};
#[cfg(all(feature = "tun-device", target_os = "windows"))]
pub use windows_impl::probe_net_admin;
#[cfg(all(feature = "tun-device", target_os = "macos"))]
pub use macos_impl::probe_net_admin;
#[cfg(not(any(
all(feature = "tun-device", target_os = "linux"),
all(feature = "tun-device", target_os = "windows"),
all(feature = "tun-device", target_os = "macos")
)))]
pub use other_impl::probe_net_admin;
#[cfg(not(any(
all(feature = "tun-device", target_os = "linux"),
all(feature = "tun-device", target_os = "windows"),
all(feature = "tun-device", target_os = "macos")
)))]
mod other_impl {
use super::Privilege;
/// Whether this process can configure interfaces.
pub fn probe_net_admin() -> Privilege {
Privilege::Unsupported
}
}
#[cfg(all(feature = "tun-device", target_os = "windows"))]
mod windows_impl {
use super::Privilege;
/// Whether this process can configure interfaces.
///
/// Windows has no capability to hold and lower the way Linux does:
/// creating an adapter simply needs the process to be elevated. Reading
/// whether it *is* elevated means inspecting the process token through a
/// raw call this crate forbids, so the probe is deliberately optimistic —
/// it reports that the platform can manage interfaces — and the real
/// check is left to Wintun's adapter creation, which fails with a precise
/// message when the process is not elevated. This matches how the Linux
/// path treats the open itself as the honest answer.
pub fn probe_net_admin() -> Privilege {
Privilege::Available
}
}
#[cfg(all(feature = "tun-device", target_os = "macos"))]
mod macos_impl {
use super::Privilege;
/// Whether this process can configure interfaces.
///
/// macOS has no capability to hold and lower the way Linux does: creating
/// a utun interface simply needs the process to be root. Reading the
/// effective uid means a libc call this crate forbids, so the probe is
/// deliberately optimistic — it reports that the platform can manage
/// interfaces — and the real check is left to the utun open, which fails
/// with a precise "run with sudo" message when the process is not root.
/// This matches how the Linux path treats the open itself as the honest
/// answer.
pub fn probe_net_admin() -> Privilege {
Privilege::Available
}
}
#[cfg(all(feature = "tun-device", target_os = "linux"))]
mod linux_impl {
use caps::{CapSet, Capability};
use super::Privilege;
use crate::overlay::OverlayError;
/// Whether this thread holds `CAP_NET_ADMIN` in its permitted set.
pub fn probe_net_admin() -> Privilege {
match caps::has_cap(None, CapSet::Permitted, Capability::CAP_NET_ADMIN) {
Ok(true) => Privilege::Available,
Ok(false) => Privilege::Missing(
"this process does not hold CAP_NET_ADMIN, so it cannot create \
or configure a network interface"
.to_string(),
),
Err(err) => {
Privilege::Missing(format!("cannot read this process's capabilities: {err}"))
}
}
}
/// `CAP_NET_ADMIN`, raised for as long as this value is alive.
///
/// Dropping it lowers the capability again, including on the error paths,
/// which is the point of it being a guard rather than a pair of calls.
#[derive(Debug)]
pub struct NetAdmin {
/// Whether this guard is the one that raised it, and so the one that
/// must lower it. Nested acquisition leaves the inner guard inert.
raised: bool,
}
impl NetAdmin {
/// Raises `CAP_NET_ADMIN` into the effective set.
pub fn acquire() -> Result<Self, OverlayError> {
let already = caps::has_cap(None, CapSet::Effective, Capability::CAP_NET_ADMIN)
.map_err(|err| {
OverlayError::Unavailable(format!("cannot read capabilities: {err}"))
})?;
if already {
return Ok(Self { raised: false });
}
caps::raise(None, CapSet::Effective, Capability::CAP_NET_ADMIN).map_err(|err| {
OverlayError::Unavailable(format!(
"cannot raise CAP_NET_ADMIN: {err}. {}",
Privilege::how_to_grant("tsng")
))
})?;
Ok(Self { raised: true })
}
/// Lowers `CAP_NET_ADMIN` out of the effective set of this thread.
///
/// Called on the way in as well as on the way out, so that a binary
/// granted `cap_net_admin+ep` — which starts with it effective — still
/// spends almost all of its life unable to use it.
pub fn lower() {
let _ = caps::drop(None, CapSet::Effective, Capability::CAP_NET_ADMIN);
}
}
impl Drop for NetAdmin {
fn drop(&mut self) {
if self.raised {
Self::lower();
}
}
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn the_grant_instructions_name_the_program() {
let text = Privilege::how_to_grant("/usr/local/bin/tsng");
assert!(text.contains("setcap cap_net_admin,cap_net_bind_service+p /usr/local/bin/tsng"));
assert!(text.contains("--no-tun"), "the fallback is offered too");
}
#[test]
fn probing_says_something_definite_about_this_host() {
// Whatever the answer is, it must be one of the three, and a missing
// capability must come with a reason rather than a bare `false`.
match probe_net_admin() {
Privilege::Available => {}
Privilege::Missing(reason) => assert!(!reason.is_empty()),
Privilege::Unsupported => {}
}
}
}
@@ -0,0 +1,80 @@
//! The provisioner for platforms that do not have one yet.
//!
//! macOS and Windows both need real work here — `utun` plus the
//! `SystemConfiguration` framework on one, the IP Helper API and a Wintun
//! adapter on the other — and neither is written. Rather than let the agent
//! come up and fail obscurely at the first packet, this refuses at the point
//! of provisioning and says what to do instead.
use crate::BoxFuture;
use crate::overlay::OverlayError;
use super::{InterfacePlan, InterfaceProvisioner, Provisioned};
/// Refuses to provision, with an explanation.
#[derive(Debug, Clone)]
pub struct UnsupportedProvisioner {
platform: &'static str,
}
impl Default for UnsupportedProvisioner {
fn default() -> Self {
Self::new()
}
}
impl UnsupportedProvisioner {
/// A provisioner naming the platform it is standing in for.
pub fn new() -> Self {
Self {
platform: std::env::consts::OS,
}
}
fn refusal(&self) -> OverlayError {
OverlayError::Unavailable(format!(
"managing the overlay interface is not implemented on {} yet. \
Run with `--no-tun` until it is: the tunnels still form, they just \
do not reach the operating system.",
self.platform
))
}
}
impl InterfaceProvisioner for UnsupportedProvisioner {
fn name(&self) -> &str {
"unsupported"
}
fn reconcile<'a>(
&'a self,
_plan: &'a InterfacePlan,
) -> BoxFuture<'a, Result<Provisioned, OverlayError>> {
Box::pin(async move { Err(self.refusal()) })
}
fn remove<'a>(&'a self, _name: &'a str) -> BoxFuture<'a, Result<(), OverlayError>> {
// Nothing was ever created, so there is nothing to clean up and no
// reason to fail a shutdown path.
Box::pin(async move { Ok(()) })
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[tokio::test]
async fn it_refuses_with_a_usable_pointer_and_still_cleans_up_quietly() {
let provisioner = UnsupportedProvisioner::new();
let plan = InterfacePlan::new("tsuntest", 1280, Vec::new());
let err = provisioner.reconcile(&plan).await.unwrap_err();
let message = err.to_string();
assert!(message.contains("--no-tun"), "{message}");
assert!(message.contains(std::env::consts::OS), "{message}");
provisioner.remove("tsuntest").await.unwrap();
}
}
@@ -0,0 +1,513 @@
//! Managing the overlay interface on Windows, through Wintun and `netsh`.
//!
//! The shape is the same as the Linux provisioner: observe what is there,
//! [`plan_changes`](super::plan_changes) to decide what to do, apply the
//! difference. Only the two mechanisms differ.
//!
//! # Creating the interface
//!
//! The interface is a Wintun adapter, created through the `tun` dependency's
//! safe wrapper — [`open_tun`](super::super::tun) — because loading
//! `wintun.dll` is an `unsafe` call and this crate forbids `unsafe`. The
//! adapter is created, not made persistent: closing the last handle to it
//! removes it, so it goes away when the agent does, however the agent ends.
//! The reader and writer inside the device hold that handle, so the adapter
//! lives exactly as long as the device does.
//!
//! # Configuring it
//!
//! Addresses and MTU are applied with `netsh`. On Linux the same work is done
//! in process over a netlink socket; here there is no equivalent that this
//! crate may call, because the in-process route is the IP Helper API and that
//! is `unsafe`. `netsh` is the supported tool for the job, it is invoked by
//! absolute path from `%SystemRoot%` so nothing on `PATH` can stand in for it,
//! and every value handed to it is one this agent derived — an interface name
//! from the network id, an address it allocated — never anything a peer said.
//!
//! # Observing it
//!
//! The interface table is read through `netdev`, whose enumeration is a safe
//! wrapper over the same IP Helper API. An adapter is matched by the friendly
//! name it was created with. A match that carries a default gateway is treated
//! as foreign and left alone: a tsunagi overlay interface never has one, so a
//! gateway is the mark of a real device that happens to share the name.
//!
//! # What Windows cannot do that Linux can
//!
//! Wintun exposes no way to delete an adapter this process did not create, so
//! a leftover from a run that was killed is *reused* rather than replaced: its
//! addresses are flushed and the planned ones applied, which reaches the same
//! end state. A reused adapter is not removed on exit, only closed — that is a
//! Wintun limitation, and the common case of a clean start and a clean stop is
//! unaffected.
use std::net::{IpAddr, Ipv4Addr};
use std::sync::{Arc, Mutex};
use crate::BoxFuture;
use crate::overlay::OverlayError;
use super::super::config::Cidr;
use super::super::tun::{TunDevice, TunRequest, open_tun};
use super::privilege::{Privilege, probe_net_admin};
use super::{
InterfacePlan, InterfaceProvisioner, InterfaceState, LinkKind, Provisioned, plan_changes,
};
/// Manages the overlay interface with Wintun and `netsh`.
#[derive(Debug)]
pub struct WintunProvisioner {
/// Interfaces this process created, so they are adjusted rather than
/// replaced. See [`plan_changes`].
ours: Mutex<Vec<String>>,
/// Devices kept alive for as long as the interface should exist. Dropping
/// one closes the adapter, which removes it if this process created it.
held: Mutex<Vec<(String, Arc<dyn TunDevice>)>>,
}
impl WintunProvisioner {
/// Creates the provisioner, after checking the platform supports one.
///
/// Whether the process is actually elevated is not checked here — that
/// cannot be read without an `unsafe` call — so a missing privilege
/// surfaces at adapter creation with a precise message, the same way the
/// Linux path treats the open itself as the honest answer.
pub fn new() -> Result<Self, OverlayError> {
match probe_net_admin() {
Privilege::Available => {}
Privilege::Missing(reason) => return Err(OverlayError::Unavailable(reason)),
Privilege::Unsupported => {
return Err(OverlayError::Unavailable(
"interface management is not compiled in".to_string(),
));
}
}
Ok(Self {
ours: Mutex::new(Vec::new()),
held: Mutex::new(Vec::new()),
})
}
fn is_ours(&self, name: &str) -> bool {
lock(&self.ours)
.iter()
.any(|owned| owned.eq_ignore_ascii_case(name))
}
/// Opens the Wintun adapter, which is what creates it.
fn create_device(&self, plan: &InterfacePlan) -> Result<Arc<dyn TunDevice>, OverlayError> {
let request = TunRequest::bare(plan.name.clone(), plan.mtu);
open_tun(&request)
}
}
impl InterfaceProvisioner for WintunProvisioner {
fn name(&self) -> &str {
"wintun"
}
fn reconcile<'a>(
&'a self,
plan: &'a InterfacePlan,
) -> BoxFuture<'a, Result<Provisioned, OverlayError>> {
Box::pin(async move {
let current = observe(&plan.name).await;
let changes = plan_changes(&current, plan, self.is_ours(&plan.name))?;
// Wintun cannot delete an adapter this process did not create, so a
// "delete then create" from `plan_changes` becomes "flush then
// reuse": clear the stale addresses off the interface that is
// there, and let the create step below reopen the same adapter.
// The end state — an adapter carrying exactly the plan's addresses
// — is identical.
let flush: Vec<Cidr> = if changes.delete_link {
tracing::info!(
interface = %plan.name,
"reusing an abandoned interface left by an earlier run"
);
current.addresses.clone()
} else {
Vec::new()
};
for cidr in flush.iter().chain(changes.remove.iter()) {
// Best effort: an address that is already gone is the outcome
// wanted, and refusing to start over one is not worth it.
if let Err(err) = address_del(&plan.name, *cidr).await {
tracing::debug!(interface = %plan.name, %cidr, %err, "could not remove address");
}
}
let device = if changes.create_link {
let device = self.create_device(plan)?;
lock(&self.ours).push(plan.name.clone());
lock(&self.held).push((plan.name.clone(), Arc::clone(&device)));
Some(device)
} else {
None
};
// A failure past this point leaves an interface that exists but
// cannot carry traffic. Undo the bookkeeping for one we just
// created so it is not mistaken for a working interface, and drop
// the device so the adapter closes.
let configured = configure(plan, &changes).await;
if let Err(err) = configured {
if changes.create_link {
lock(&self.held).retain(|(held, _)| held != &plan.name);
lock(&self.ours).retain(|owned| owned != &plan.name);
}
return Err(err);
}
Ok(Provisioned { changes, device })
})
}
fn remove<'a>(&'a self, name: &'a str) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move {
// Dropping the device closes the adapter, which removes it if this
// process created it. A reused adapter is only closed, not removed;
// flushing its addresses first keeps no stale configuration behind.
let current = observe(name).await;
for cidr in &current.addresses {
if let Err(err) = address_del(name, *cidr).await {
tracing::debug!(interface = %name, %cidr, %err, "could not remove address");
}
}
lock(&self.held).retain(|(held, _)| !held.eq_ignore_ascii_case(name));
lock(&self.ours).retain(|owned| !owned.eq_ignore_ascii_case(name));
Ok(())
})
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
match mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
/// `fe80::/10` and `169.254.0.0/16`, which Windows assigns on its own.
fn is_link_local(addr: IpAddr) -> bool {
match addr {
IpAddr::V4(addr) => addr.is_link_local(),
IpAddr::V6(addr) => (addr.segments()[0] & 0xffc0) == 0xfe80,
}
}
/// What the interface named `name` currently looks like.
///
/// Never fails: a name nothing is using reads as [`InterfaceState::absent`],
/// which is what the plan then acts on.
async fn observe(name: &str) -> InterfaceState {
let name = name.to_string();
let found = tokio::task::spawn_blocking(move || {
netdev::get_interfaces()
.into_iter()
.find(|iface| match &iface.friendly_name {
Some(friendly) => friendly.eq_ignore_ascii_case(&name),
None => false,
})
})
.await
.ok()
.flatten();
let Some(iface) = found else {
return InterfaceState::absent();
};
let mut addresses: Vec<Cidr> = Vec::new();
for net in &iface.ipv4 {
if let Ok(cidr) = Cidr::new(IpAddr::V4(net.addr()), net.prefix_len())
&& !is_link_local(cidr.addr)
{
addresses.push(cidr);
}
}
for net in &iface.ipv6 {
if let Ok(cidr) = Cidr::new(IpAddr::V6(net.addr()), net.prefix_len())
&& !is_link_local(cidr.addr)
{
addresses.push(cidr);
}
}
addresses.sort();
// A tsunagi overlay interface only ever carries the addresses it was given
// and no gateway. One that has a gateway is a real device sharing the name,
// so it is foreign and never touched.
let kind = if iface.gateway.is_some() {
LinkKind::Foreign("gatewayed".to_string())
} else {
LinkKind::Tun
};
InterfaceState {
kind,
// Wintun reports an adapter with a live session as up and an abandoned
// one as down, so "up" doubles as "some process is holding it".
attached: iface.is_up(),
up: iface.is_up(),
mtu: iface.mtu.unwrap_or(0),
addresses,
}
}
/// Applies MTU, link state and addresses.
async fn configure(plan: &InterfacePlan, changes: &super::Changes) -> Result<(), OverlayError> {
if let Some(mtu) = changes.set_mtu {
set_mtu(&plan.name, mtu).await?;
}
for cidr in &changes.add {
if let Err(err) = address_add(&plan.name, *cidr).await {
let err_str = err.to_string();
if !err_str.contains("already exists") && !err_str.contains("0x80071392") {
return Err(err);
}
}
}
if changes.bring_up {
// Best effort: a Wintun adapter with a session is up already, and a
// failure to nudge it is not a reason to fail the whole reconcile.
if let Err(err) = set_up(&plan.name).await {
tracing::debug!(interface = %plan.name, %err, "could not enable the interface");
}
}
Ok(())
}
/// The dotted netmask for an IPv4 prefix length.
fn ipv4_mask(prefix_len: u8) -> Ipv4Addr {
if prefix_len == 0 {
Ipv4Addr::UNSPECIFIED
} else {
Ipv4Addr::from(u32::MAX << (32 - u32::from(prefix_len)))
}
}
/// The `netsh` arguments that add an address to an interface.
fn address_add_args(name: &str, cidr: Cidr) -> Vec<String> {
match cidr.addr {
IpAddr::V4(addr) => vec![
"interface".into(),
"ipv4".into(),
"add".into(),
"address".into(),
format!("name={name}"),
format!("address={addr}"),
format!("mask={}", ipv4_mask(cidr.prefix_len)),
"store=active".into(),
],
IpAddr::V6(addr) => vec![
"interface".into(),
"ipv6".into(),
"add".into(),
"address".into(),
format!("interface={name}"),
format!("address={addr}/{}", cidr.prefix_len),
"store=active".into(),
],
}
}
/// The `netsh` arguments that remove an address from an interface.
fn address_del_args(name: &str, cidr: Cidr) -> Vec<String> {
match cidr.addr {
IpAddr::V4(addr) => vec![
"interface".into(),
"ipv4".into(),
"delete".into(),
"address".into(),
format!("name={name}"),
format!("address={addr}"),
"store=active".into(),
],
IpAddr::V6(addr) => vec![
"interface".into(),
"ipv6".into(),
"delete".into(),
"address".into(),
format!("interface={name}"),
format!("address={addr}"),
"store=active".into(),
],
}
}
/// The `netsh` arguments that set an interface's MTU, one call per family.
fn mtu_args(name: &str, family: &str, mtu: u32) -> Vec<String> {
vec![
"interface".into(),
family.into(),
"set".into(),
"subinterface".into(),
name.into(),
format!("mtu={mtu}"),
"store=active".into(),
]
}
async fn address_add(name: &str, cidr: Cidr) -> Result<(), OverlayError> {
netsh(address_add_args(name, cidr)).await
}
async fn address_del(name: &str, cidr: Cidr) -> Result<(), OverlayError> {
netsh(address_del_args(name, cidr)).await
}
/// Sets the MTU on both families.
///
/// A family that is turned off on the adapter cannot take an MTU, so this
/// fails only when *neither* would — one family carrying the plan's MTU is
/// enough, and refusing the whole interface because the other is disabled
/// would be wrong.
async fn set_mtu(name: &str, mtu: u32) -> Result<(), OverlayError> {
let v4 = netsh(mtu_args(name, "ipv4", mtu)).await;
let v6 = netsh(mtu_args(name, "ipv6", mtu)).await;
match (&v4, &v6) {
(Err(_), Err(_)) => v4,
_ => {
if let Err(err) = &v4 {
tracing::debug!(interface = %name, %err, "could not set the IPv4 MTU");
}
if let Err(err) = &v6 {
tracing::debug!(interface = %name, %err, "could not set the IPv6 MTU");
}
Ok(())
}
}
}
async fn set_up(name: &str) -> Result<(), OverlayError> {
netsh(vec![
"interface".into(),
"set".into(),
"interface".into(),
format!("name={name}"),
"admin=enabled".into(),
])
.await
}
/// Runs `netsh` from `%SystemRoot%\System32`, so nothing on `PATH` can stand
/// in for it, and turns a non-zero exit into an error carrying what it said.
async fn netsh(args: Vec<String>) -> Result<(), OverlayError> {
let program = system32("netsh.exe");
let display = format!("netsh {}", args.join(" "));
let output = tokio::task::spawn_blocking(move || {
std::process::Command::new(program).args(&args).output()
})
.await
.map_err(|err| OverlayError::Other(format!("could not run {display}: {err}")))?
.map_err(|err| OverlayError::Unavailable(format!("could not run {display}: {err}")))?;
if output.status.success() {
return Ok(());
}
let message = {
let stderr = String::from_utf8_lossy(&output.stderr);
let text = if stderr.trim().is_empty() {
String::from_utf8_lossy(&output.stdout).trim().to_string()
} else {
stderr.trim().to_string()
};
if text.is_empty() {
format!("exit code {}", output.status)
} else {
text
}
};
Err(OverlayError::Unavailable(format!(
"{display} failed: {message}"
)))
}
/// The absolute path to a program in `System32`.
fn system32(exe: &str) -> std::path::PathBuf {
let root = std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into());
std::path::Path::new(&root).join("System32").join(exe)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use std::net::Ipv6Addr;
fn v4(last: u8, prefix: u8) -> Cidr {
Cidr::new(IpAddr::V4(Ipv4Addr::new(10, 13, 37, last)), prefix).unwrap()
}
fn v6(last: u16) -> Cidr {
Cidr::new(
IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, last)),
64,
)
.unwrap()
}
#[test]
fn a_prefix_length_becomes_the_right_dotted_mask() {
assert_eq!(ipv4_mask(24), Ipv4Addr::new(255, 255, 255, 0));
assert_eq!(ipv4_mask(16), Ipv4Addr::new(255, 255, 0, 0));
assert_eq!(ipv4_mask(32), Ipv4Addr::new(255, 255, 255, 255));
assert_eq!(ipv4_mask(0), Ipv4Addr::UNSPECIFIED);
}
#[test]
fn a_v4_address_is_added_by_name_with_a_dotted_mask() {
let args = address_add_args("tsun0", v4(69, 24));
assert!(args.contains(&"ipv4".to_string()));
assert!(args.contains(&"add".to_string()));
assert!(args.contains(&"name=tsun0".to_string()));
assert!(args.contains(&"address=10.13.37.69".to_string()));
assert!(args.contains(&"mask=255.255.255.0".to_string()));
}
#[test]
fn a_v6_address_is_added_by_interface_with_a_prefix() {
let args = address_add_args("tsun0", v6(1));
assert!(args.contains(&"ipv6".to_string()));
assert!(args.contains(&"interface=tsun0".to_string()));
assert!(args.contains(&"address=fd00::1/64".to_string()));
}
#[test]
fn deleting_an_address_names_no_mask() {
let args = address_del_args("tsun0", v4(69, 24));
assert!(args.contains(&"delete".to_string()));
assert!(args.contains(&"address=10.13.37.69".to_string()));
assert!(!args.iter().any(|arg| arg.starts_with("mask=")));
}
#[test]
fn the_mtu_is_set_per_family_on_the_subinterface() {
let args = mtu_args("tsun0", "ipv4", 1280);
assert!(args.contains(&"subinterface".to_string()));
assert!(args.contains(&"tsun0".to_string()));
assert!(args.contains(&"mtu=1280".to_string()));
}
#[test]
fn link_local_addresses_are_not_ours_to_manage() {
assert!(is_link_local(IpAddr::V6(Ipv6Addr::new(
0xfe80, 0, 0, 0, 0, 0, 0, 1
))));
assert!(is_link_local(IpAddr::V4(Ipv4Addr::new(169, 254, 1, 1))));
assert!(!is_link_local(IpAddr::V4(Ipv4Addr::new(10, 13, 37, 1))));
assert!(!is_link_local(IpAddr::V6(Ipv6Addr::new(
0xfd00, 0, 0, 0, 0, 0, 0, 1
))));
}
#[test]
fn system32_is_absolute_and_ends_with_the_program() {
let path = system32("netsh.exe");
assert!(path.is_absolute(), "{path:?}");
assert!(path.ends_with("netsh.exe"), "{path:?}");
}
}
+925
View File
@@ -0,0 +1,925 @@
//! Which peer a packet is for, and whether a peer may have sent it.
//!
//! One agent has one interface, so a packet coming off it could belong to any
//! network the agent is in and to any protocol currently carrying traffic.
//! Deciding which is this table's job, and it is the reason the interface
//! belongs to the system level: the answer comes from signed state, which no
//! protocol owns.
//!
//! Two questions, and they are not the same one:
//!
//! * **Outbound**: this destination address — whose is it? A packet for
//! nobody is dropped rather than broadcast.
//! * **Inbound**: this peer decrypted a packet with this source address —
//! is that address actually its? A peer may not speak for anybody else,
//! and the check consults the signed claim rather than anything the peer
//! said.
//!
//! Nothing here is derived from a protocol's key. An address is allocated at
//! the system level and signed by the member that holds it, so every protocol
//! carries traffic for the same addresses and the table is the same whichever
//! one is in use.
use super::broadcast::{BroadcastPolicy, BroadcastTable};
use arc_swap::ArcSwap;
use std::collections::HashMap;
use std::net::Ipv4Addr;
use std::sync::{Arc, RwLock};
use iroh::EndpointId;
use crate::identity::NetworkId;
use crate::state::Ipv4Range;
/// Where an outbound packet is going.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Route {
/// The network the destination belongs to.
pub network: NetworkId,
/// The member that holds the destination address.
pub peer: EndpointId,
}
/// A network's exit-node settings, as far as packets are concerned.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ExitPolicy {
/// This agent offers itself as an exit node here: packets from members
/// for addresses outside every overlay range are handed to the host.
pub offer: bool,
/// The member all other traffic is sent to, if this device uses one.
pub via: Option<EndpointId>,
}
/// What one network contributes to the table.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct NetworkRoutes {
/// Exit-node settings of this network.
pub exit: ExitPolicy,
/// Local broadcast policy and authenticated willing recipients.
pub broadcast: BroadcastPolicy,
/// The range this network allocates from, once it has agreed one.
pub range: Option<Ipv4Range>,
/// This agent's own address in the network.
pub local: Option<Ipv4Addr>,
/// Every other member's address, as the signed state has it.
pub peers: Vec<(Ipv4Addr, EndpointId)>,
}
/// Why a network cannot be added to the table.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum RouteError {
/// Two networks would answer for the same addresses.
#[error(
"network {network} uses {range}, which overlaps {other_range} already in use by \
{other}. One agent has one interface, so an address can belong to only one of \
them: give one network a different range."
)]
Overlap {
/// The network being added.
network: NetworkId,
/// The range it wants.
range: Ipv4Range,
/// The network already using an overlapping range.
other: NetworkId,
/// That network's range.
other_range: Ipv4Range,
},
}
/// Address ownership across every network this agent is in.
#[derive(Debug, Default)]
pub struct RoutingTable {
broadcast: ArcSwap<BroadcastTable>,
networks: RwLock<HashMap<NetworkId, NetworkRoutes>>,
}
impl RoutingTable {
/// An empty table.
pub fn new() -> Self {
Self::default()
}
fn read(&self) -> std::sync::RwLockReadGuard<'_, HashMap<NetworkId, NetworkRoutes>> {
match self.networks.read() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
fn write(&self) -> std::sync::RwLockWriteGuard<'_, HashMap<NetworkId, NetworkRoutes>> {
match self.networks.write() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
/// Replaces what one network contributes.
///
/// Rejected when its range overlaps another network's. With one
/// interface an address can belong to only one network, and guessing
/// which would send somebody's traffic to a stranger.
pub fn set_network(&self, network: NetworkId, routes: NetworkRoutes) -> Result<(), RouteError> {
let mut networks = self.write();
if let Some(range) = routes.range {
for (other, existing) in networks.iter() {
if *other == network {
continue;
}
let Some(other_range) = existing.range else {
continue;
};
if ranges_overlap(range, other_range) {
return Err(RouteError::Overlap {
network,
range,
other: *other,
other_range,
});
}
}
}
networks.insert(network, routes);
self.broadcast
.store(Arc::new(BroadcastTable::build(&networks)));
Ok(())
}
/// Forgets a network.
pub fn remove_network(&self, network: NetworkId) {
let mut networks = self.write();
networks.remove(&network);
self.broadcast
.store(Arc::new(BroadcastTable::build(&networks)));
}
/// Replaces only a network's broadcast participation, without changing addresses.
pub fn set_broadcast(&self, network: NetworkId, policy: BroadcastPolicy) {
let mut networks = self.write();
if let Some(routes) = networks.get_mut(&network) {
if routes.broadcast == policy {
return;
}
routes.broadcast = policy;
self.broadcast
.store(Arc::new(BroadcastTable::build(&networks)));
}
}
/// Replaces only a network's exit-node settings, without changing addresses.
///
/// A network that has no entry yet (its range is not agreed) takes them
/// from [`NetworkRoutes::exit`] when it is added.
pub fn set_exit(&self, network: NetworkId, policy: ExitPolicy) {
if let Some(routes) = self.write().get_mut(&network) {
routes.exit = policy;
}
}
/// Recognizes limited and configured subnet broadcasts before unicast lookup.
pub fn is_broadcast(&self, destination: Ipv4Addr) -> bool {
self.broadcast.load().is_destination(destination)
}
/// A precomputed domain-scoped recipient list; no per-packet graph search.
pub fn broadcast_recipients(
&self,
source: Ipv4Addr,
destination: Ipv4Addr,
) -> Option<Arc<[Route]>> {
self.broadcast.load().outgoing(source, destination)
}
/// Local admission after the plugin authenticated and decrypted the sender.
pub fn accepts_broadcast(
&self,
network: NetworkId,
peer: EndpointId,
destination: Ipv4Addr,
) -> bool {
self.broadcast.load().accepts(network, peer, destination)
}
/// The peer that holds a destination address, if anybody does.
///
/// This agent's own address returns `None`: a packet for ourselves does
/// not go over a tunnel, and answering with a peer would send it to one.
///
/// An address nobody holds, outside every overlay range and an ordinary
/// unicast destination, goes to this device's exit node when it has one:
/// that is what sending *all* traffic through a member means.
pub fn route(&self, destination: Ipv4Addr) -> Option<Route> {
let networks = self.read();
for (network, routes) in networks.iter() {
if routes.local == Some(destination) {
return None;
}
if let Some((_, peer)) = routes
.peers
.iter()
.find(|(address, _)| *address == destination)
{
return Some(Route {
network: *network,
peer: *peer,
});
}
}
// Inside an overlay range, an unowned address is nobody's, not the
// internet's.
if !is_internet_destination(destination) || in_any_range(&networks, destination) {
return None;
}
exit_via(&networks).map(|(network, peer)| Route { network, peer })
}
/// Whether a peer may send from a source address.
///
/// A peer speaks only for the address the network agreed it holds. The
/// one exception is this device's exit node: what comes back from the
/// internet carries the internet's addresses, so from *that* peer any
/// source outside the overlay ranges is accepted.
pub fn may_send_from(&self, network: NetworkId, peer: EndpointId, source: Ipv4Addr) -> bool {
let networks = self.read();
let Some(routes) = networks.get(&network) else {
return false;
};
if routes
.peers
.iter()
.any(|(address, holder)| *address == source && *holder == peer)
{
return true;
}
routes.exit.via == Some(peer)
&& is_internet_destination(source)
&& !in_any_range(&networks, source)
}
/// Whether this agent, as an exit node of `network`, takes a member's
/// packet for `destination` to the host.
///
/// Only an ordinary unicast address outside every overlay range, so a
/// packet for another member, for this agent or for the local network
/// segment never leaves by this door. The sender's own source address is
/// checked separately, by [`Self::may_send_from`].
pub fn accepts_exit_traffic(&self, network: NetworkId, destination: Ipv4Addr) -> bool {
let networks = self.read();
networks
.get(&network)
.is_some_and(|routes| routes.exit.offer)
&& is_internet_destination(destination)
&& !in_any_range(&networks, destination)
}
/// The network and member this device sends its internet traffic through.
///
/// One default route, so one answer; if several networks claim one (they
/// should not) the lowest network id decides, so two runs agree.
pub fn exit_via(&self) -> Option<(NetworkId, EndpointId)> {
exit_via(&self.read())
}
/// Networks that offer this agent as an exit node, with the range each
/// agreed. A network that has not agreed one yet has nothing to
/// masquerade and is left out.
pub fn exit_offers(&self) -> Vec<(NetworkId, Ipv4Range)> {
let mut offers: Vec<(NetworkId, Ipv4Range)> = self
.read()
.iter()
.filter(|(_, routes)| routes.exit.offer)
.filter_map(|(network, routes)| Some((*network, routes.range?)))
.collect();
offers.sort_by_key(|(network, _)| *network);
offers
}
/// One network's exit-node settings and its range, if it is in the table.
pub fn exit_settings(&self, network: NetworkId) -> Option<(ExitPolicy, Option<Ipv4Range>)> {
self.read()
.get(&network)
.map(|routes| (routes.exit, routes.range))
}
/// Whether an ordinary packet terminates on this host in this network.
/// Exported LAN subnets must extend this admission policy explicitly later.
pub fn is_local_destination(&self, network: NetworkId, destination: Ipv4Addr) -> bool {
self.read()
.get(&network)
.is_some_and(|routes| routes.local == Some(destination))
}
/// Every address this agent should answer to, with its prefix length.
pub fn local_addresses(&self) -> Vec<(Ipv4Addr, u8)> {
let mut addresses: Vec<(Ipv4Addr, u8)> = self
.read()
.values()
.filter_map(|routes| Some((routes.local?, routes.range?.prefix_len)))
.collect();
addresses.sort();
addresses
}
/// The one network whose address the limited broadcast route is bound to.
///
/// The host has a single `255.255.255.255` route per interface, and one
/// interface carries every network, so only one overlay address can be its
/// source. The choice is the lowest [`NetworkId`] among networks that have
/// broadcast enabled, a range and a local address: deterministic, so two
/// runs with the same networks pick the same one.
pub fn broadcast_host_target(&self) -> Option<(Ipv4Addr, Ipv4Range)> {
let networks = self.read();
networks
.iter()
.filter(|(_, routes)| routes.broadcast.enabled)
.filter_map(|(network, routes)| Some((*network, routes.local?, routes.range?)))
.min_by_key(|(network, _, _)| *network)
.map(|(_, local, range)| (local, range))
}
/// Whether a range would collide with one another network already uses.
///
/// Asked before proposing a range rather than after: with one interface
/// an agent that claimed an address it could not route would also be
/// telling everybody else to use that range, and "the range of the
/// lowest author wins" would spread the collision instead of containing
/// it.
pub fn would_overlap(&self, network: NetworkId, range: Ipv4Range) -> Option<Ipv4Range> {
self.read().iter().find_map(|(other, existing)| {
if *other == network {
return None;
}
let other_range = existing.range?;
ranges_overlap(range, other_range).then_some(other_range)
})
}
/// How many networks the table covers.
pub fn len(&self) -> usize {
self.read().len()
}
/// Whether it covers none.
pub fn is_empty(&self) -> bool {
self.read().is_empty()
}
}
/// Whether an address is an ordinary unicast one: not this host, not a
/// multicast or broadcast group, not link-local, not unspecified.
fn is_internet_destination(address: Ipv4Addr) -> bool {
!(address.is_unspecified()
|| address.is_loopback()
|| address.is_multicast()
|| address.is_broadcast()
|| address.is_link_local())
}
/// Whether an address lies in any network's range.
fn in_any_range(networks: &HashMap<NetworkId, NetworkRoutes>, address: Ipv4Addr) -> bool {
networks
.values()
.any(|routes| routes.range.is_some_and(|range| range.contains(address)))
}
/// The exit node in use, if any, from the lowest network that names one.
fn exit_via(networks: &HashMap<NetworkId, NetworkRoutes>) -> Option<(NetworkId, EndpointId)> {
networks
.iter()
.filter_map(|(network, routes)| Some((*network, routes.exit.via?)))
.min_by_key(|(network, _)| *network)
}
/// Whether two ranges share any address.
fn ranges_overlap(one: Ipv4Range, other: Ipv4Range) -> bool {
// A range contains the other's base, or the other way round. With
// prefix-aligned ranges that is the whole of it: two blocks either nest
// or are disjoint.
one.contains(other.base) || other.contains(one.base)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::identity::{NetworkKeys, NetworkName, NetworkSecret};
fn network(name: &str) -> NetworkId {
NetworkKeys::derive(
&NetworkName::new(name).unwrap(),
&NetworkSecret::from_bytes(vec![4u8; 32]).unwrap(),
)
.network_id()
}
fn peer(seed: u8) -> EndpointId {
iroh::SecretKey::from_bytes(&[seed; 32]).public()
}
fn addr(last: u8) -> Ipv4Addr {
Ipv4Addr::new(10, 13, 37, last)
}
fn routes() -> NetworkRoutes {
NetworkRoutes {
exit: Default::default(),
broadcast: Default::default(),
range: Some("10.13.37.0/24".parse().unwrap()),
local: Some(addr(1)),
peers: vec![(addr(2), peer(2)), (addr(3), peer(3))],
}
}
#[test]
fn a_packet_goes_to_the_member_that_holds_its_destination() {
let table = RoutingTable::new();
let id = network("routing");
table.set_network(id, routes()).unwrap();
assert_eq!(
table.route(addr(2)),
Some(Route {
network: id,
peer: peer(2)
})
);
assert_eq!(
table.route(addr(3)),
Some(Route {
network: id,
peer: peer(3)
})
);
}
#[test]
fn a_packet_for_nobody_has_nowhere_to_go() {
// Dropped rather than sent to everybody: a tunnel is not a broadcast
// domain, and flooding would hand one member another's traffic.
let table = RoutingTable::new();
table.set_network(network("routing"), routes()).unwrap();
assert_eq!(table.route(addr(200)), None);
assert_eq!(table.route(Ipv4Addr::new(192, 0, 2, 1)), None);
}
#[test]
fn a_packet_for_ourselves_is_not_sent_to_a_peer() {
let table = RoutingTable::new();
table.set_network(network("routing"), routes()).unwrap();
assert_eq!(table.route(addr(1)), None, "that address is ours");
}
#[test]
fn a_peer_may_only_send_from_the_address_it_holds() {
let table = RoutingTable::new();
let id = network("ownership");
table.set_network(id, routes()).unwrap();
assert!(table.may_send_from(id, peer(2), addr(2)));
// Somebody else's address, and its own network's.
assert!(!table.may_send_from(id, peer(2), addr(3)));
assert!(!table.may_send_from(id, peer(2), addr(1)));
assert!(!table.may_send_from(id, peer(2), addr(99)));
// A peer that holds nothing here.
assert!(!table.may_send_from(id, peer(9), addr(2)));
// And the question is per network, not global.
assert!(!table.may_send_from(network("elsewhere"), peer(2), addr(2)));
}
#[test]
fn two_networks_with_overlapping_ranges_are_refused() {
// One interface means an address belongs to one network. Accepting
// both and guessing would send somebody's traffic to a stranger.
let table = RoutingTable::new();
let first = network("first");
let second = network("second");
table.set_network(first, routes()).unwrap();
let err = table.set_network(second, routes()).unwrap_err();
let RouteError::Overlap { other, .. } = err;
assert_eq!(other, first);
assert!(err.to_string().contains("different range"), "{err}");
assert_eq!(table.len(), 1, "the overlapping network is not added");
// A range of its own is fine.
table
.set_network(
second,
NetworkRoutes {
exit: Default::default(),
broadcast: Default::default(),
range: Some("10.99.0.0/16".parse().unwrap()),
local: Some(Ipv4Addr::new(10, 99, 0, 1)),
peers: vec![(Ipv4Addr::new(10, 99, 0, 2), peer(4))],
},
)
.unwrap();
assert_eq!(table.len(), 2);
assert_eq!(
table.route(Ipv4Addr::new(10, 99, 0, 2)),
Some(Route {
network: second,
peer: peer(4)
})
);
// And the first network still answers for its own.
assert_eq!(table.route(addr(2)).map(|route| route.network), Some(first));
}
#[test]
fn the_host_broadcast_target_is_the_lowest_enabled_network_with_an_address() {
let table = RoutingTable::new();
assert_eq!(table.broadcast_host_target(), None);
let (low, high) = {
let (a, b) = (network("first"), network("second"));
if a < b { (a, b) } else { (b, a) }
};
let mk = |enabled: bool, base: [u8; 4], local: Option<Ipv4Addr>| NetworkRoutes {
exit: Default::default(),
broadcast: BroadcastPolicy {
enabled,
..Default::default()
},
range: Some(
format!("{}.{}.{}.0/24", base[0], base[1], base[2])
.parse()
.unwrap(),
),
local,
peers: Vec::new(),
};
let a = Ipv4Addr::new(10, 1, 0, 1);
let b = Ipv4Addr::new(10, 2, 0, 1);
// The lower id has broadcast off, so the higher one is chosen.
table
.set_network(low, mk(false, [10, 1, 0, 0], Some(a)))
.unwrap();
table
.set_network(high, mk(true, [10, 2, 0, 0], Some(b)))
.unwrap();
assert_eq!(
table.broadcast_host_target(),
Some((b, "10.2.0.0/24".parse().unwrap()))
);
// Both enabled: the lowest id wins, whatever the insertion order.
table
.set_network(low, mk(true, [10, 1, 0, 0], Some(a)))
.unwrap();
assert_eq!(
table.broadcast_host_target(),
Some((a, "10.1.0.0/24".parse().unwrap()))
);
// No local address yet: it cannot be the route's source.
table
.set_network(low, mk(true, [10, 1, 0, 0], None))
.unwrap();
assert_eq!(
table.broadcast_host_target(),
Some((b, "10.2.0.0/24".parse().unwrap()))
);
table.remove_network(high);
assert_eq!(table.broadcast_host_target(), None);
}
#[test]
fn a_range_can_be_asked_about_before_it_is_proposed() {
// The point of asking first: an agent that claims an address it
// cannot route also tells everybody else to use that range.
let table = RoutingTable::new();
let first = network("first");
let second = network("second");
table.set_network(first, routes()).unwrap();
let mine: Ipv4Range = "10.13.37.0/24".parse().unwrap();
assert_eq!(table.would_overlap(second, mine), Some(mine));
// Its own range is not a collision with itself.
assert_eq!(table.would_overlap(first, mine), None);
assert_eq!(
table.would_overlap(second, "10.99.0.0/16".parse().unwrap()),
None
);
}
#[test]
fn a_nested_range_counts_as_overlapping() {
let table = RoutingTable::new();
table
.set_network(
network("wide"),
NetworkRoutes {
exit: Default::default(),
broadcast: Default::default(),
range: Some("10.0.0.0/8".parse().unwrap()),
..Default::default()
},
)
.unwrap();
// Inside the /8, in either direction.
assert!(
table
.set_network(
network("narrow"),
NetworkRoutes {
exit: Default::default(),
broadcast: Default::default(),
range: Some("10.13.37.0/24".parse().unwrap()),
..Default::default()
}
)
.is_err()
);
}
#[test]
fn replacing_a_network_does_not_count_as_overlapping_itself() {
let table = RoutingTable::new();
let id = network("refresh");
table.set_network(id, routes()).unwrap();
let mut updated = routes();
updated.peers.push((addr(4), peer(4)));
table.set_network(id, updated).unwrap();
assert_eq!(table.len(), 1);
assert!(table.route(addr(4)).is_some());
}
#[test]
fn forgetting_a_network_takes_its_addresses_with_it() {
let table = RoutingTable::new();
let id = network("leaving");
table.set_network(id, routes()).unwrap();
table.remove_network(id);
assert!(table.is_empty());
assert_eq!(table.route(addr(2)), None);
assert!(!table.may_send_from(id, peer(2), addr(2)));
}
#[test]
fn the_local_addresses_are_what_the_interface_should_carry() {
let table = RoutingTable::new();
table.set_network(network("one"), routes()).unwrap();
table
.set_network(
network("two"),
NetworkRoutes {
exit: Default::default(),
broadcast: Default::default(),
range: Some("10.99.0.0/16".parse().unwrap()),
local: Some(Ipv4Addr::new(10, 99, 0, 1)),
peers: Vec::new(),
},
)
.unwrap();
assert_eq!(
table.local_addresses(),
vec![(addr(1), 24), (Ipv4Addr::new(10, 99, 0, 1), 16)]
);
}
#[test]
fn a_network_with_no_address_yet_contributes_nothing() {
let table = RoutingTable::new();
let id = network("waiting");
table.set_network(id, NetworkRoutes::default()).unwrap();
assert!(table.local_addresses().is_empty());
assert_eq!(table.route(addr(2)), None);
assert_eq!(table.len(), 1, "the network is known, it just has nothing");
}
fn with_exit(offer: bool, via: Option<EndpointId>) -> NetworkRoutes {
NetworkRoutes {
exit: ExitPolicy { offer, via },
..routes()
}
}
const INTERNET: Ipv4Addr = Ipv4Addr::new(93, 184, 216, 34);
#[test]
fn without_an_exit_node_the_internet_has_nowhere_to_go() {
let table = RoutingTable::new();
table.set_network(network("plain"), routes()).unwrap();
assert_eq!(table.route(INTERNET), None);
assert_eq!(table.exit_via(), None);
}
#[test]
fn with_an_exit_node_every_ordinary_destination_goes_to_it() {
let table = RoutingTable::new();
let id = network("client");
table
.set_network(id, with_exit(false, Some(peer(2))))
.unwrap();
let exit = Route {
network: id,
peer: peer(2),
};
assert_eq!(table.route(INTERNET), Some(exit));
assert_eq!(table.route(Ipv4Addr::new(192, 168, 1, 10)), Some(exit));
assert_eq!(table.exit_via(), Some((id, peer(2))));
// A member's own address still goes to that member.
assert_eq!(
table.route(addr(3)),
Some(Route {
network: id,
peer: peer(3)
})
);
}
#[test]
fn the_exit_node_is_never_the_answer_for_what_is_not_the_internet() {
let table = RoutingTable::new();
table
.set_network(network("client"), with_exit(false, Some(peer(2))))
.unwrap();
// Ourselves, a free address inside the overlay range, and everything
// that is not an ordinary unicast destination.
for destination in [
addr(1),
addr(200),
Ipv4Addr::new(127, 0, 0, 1),
Ipv4Addr::new(224, 0, 0, 251),
Ipv4Addr::new(239, 255, 255, 250),
Ipv4Addr::BROADCAST,
Ipv4Addr::UNSPECIFIED,
Ipv4Addr::new(169, 254, 1, 1),
] {
assert_eq!(table.route(destination), None, "{destination}");
}
}
#[test]
fn another_networks_range_is_not_the_internet_either() {
let table = RoutingTable::new();
table
.set_network(network("client"), with_exit(false, Some(peer(2))))
.unwrap();
table
.set_network(
network("other"),
NetworkRoutes {
range: Some("10.99.0.0/16".parse().unwrap()),
local: Some(Ipv4Addr::new(10, 99, 0, 1)),
..Default::default()
},
)
.unwrap();
assert_eq!(table.route(Ipv4Addr::new(10, 99, 7, 7)), None);
}
#[test]
fn two_exit_choices_resolve_to_the_lowest_network_id() {
let table = RoutingTable::new();
let (a, b) = (network("one"), network("two"));
table
.set_network(a, with_exit(false, Some(peer(2))))
.unwrap();
table
.set_network(
b,
NetworkRoutes {
exit: ExitPolicy {
offer: false,
via: Some(peer(9)),
},
range: Some("10.99.0.0/16".parse().unwrap()),
..Default::default()
},
)
.unwrap();
let (chosen, _) = table.exit_via().unwrap();
assert_eq!(chosen, a.min(b));
assert_eq!(table.route(INTERNET).unwrap().network, a.min(b));
}
#[test]
fn the_exit_peer_may_send_from_any_internet_address_but_nobody_else_may() {
let table = RoutingTable::new();
let id = network("client");
table
.set_network(id, with_exit(false, Some(peer(2))))
.unwrap();
// What comes back from the internet carries the internet's addresses.
assert!(table.may_send_from(id, peer(2), INTERNET));
// Another member still speaks only for its own address.
assert!(!table.may_send_from(id, peer(3), INTERNET));
assert!(table.may_send_from(id, peer(3), addr(3)));
// The exit peer cannot pass itself off as somebody else in the
// overlay, nor as this agent, nor as something that is not a source.
assert!(!table.may_send_from(id, peer(2), addr(3)));
assert!(!table.may_send_from(id, peer(2), addr(1)));
assert!(!table.may_send_from(id, peer(2), addr(200)));
for source in [
Ipv4Addr::LOCALHOST,
Ipv4Addr::UNSPECIFIED,
Ipv4Addr::new(224, 0, 0, 1),
Ipv4Addr::BROADCAST,
] {
assert!(!table.may_send_from(id, peer(2), source), "{source}");
}
// Without choosing an exit node nobody gets the exception.
let plain = RoutingTable::new();
plain.set_network(id, routes()).unwrap();
assert!(!plain.may_send_from(id, peer(2), INTERNET));
}
#[test]
fn an_exit_node_takes_a_members_internet_packets_only_while_it_offers() {
let table = RoutingTable::new();
let id = network("server");
table.set_network(id, with_exit(false, None)).unwrap();
assert!(!table.accepts_exit_traffic(id, INTERNET));
table.set_exit(
id,
ExitPolicy {
offer: true,
via: None,
},
);
assert!(table.accepts_exit_traffic(id, INTERNET));
assert!(table.accepts_exit_traffic(id, Ipv4Addr::new(192, 168, 1, 10)));
// Never for another member, for this agent, a free overlay address,
// or anything that is not an ordinary unicast destination.
for destination in [
addr(1),
addr(2),
addr(200),
Ipv4Addr::LOCALHOST,
Ipv4Addr::new(224, 0, 0, 251),
Ipv4Addr::BROADCAST,
Ipv4Addr::UNSPECIFIED,
] {
assert!(
!table.accepts_exit_traffic(id, destination),
"{destination}"
);
}
// An unknown network offers nothing.
assert!(!table.accepts_exit_traffic(network("elsewhere"), INTERNET));
table.set_exit(id, ExitPolicy::default());
assert!(!table.accepts_exit_traffic(id, INTERNET));
}
#[test]
fn offers_are_listed_by_network_with_the_range_each_agreed() {
let table = RoutingTable::new();
let (a, b) = (network("a"), network("b"));
table.set_network(a, with_exit(true, None)).unwrap();
table
.set_network(
b,
NetworkRoutes {
exit: ExitPolicy {
offer: true,
via: None,
},
// No range agreed yet: nothing to masquerade.
..Default::default()
},
)
.unwrap();
assert_eq!(
table.exit_offers(),
vec![(a, "10.13.37.0/24".parse().unwrap())]
);
assert_eq!(
table.exit_settings(a),
Some((
ExitPolicy {
offer: true,
via: None
},
Some("10.13.37.0/24".parse().unwrap())
))
);
assert_eq!(table.exit_settings(network("none")), None);
}
#[test]
fn setting_exit_for_a_network_with_no_entry_is_remembered_by_the_caller() {
// The runtime keeps the policy and passes it with the network when
// its range is agreed; a setter for a network the table does not know
// yet changes nothing.
let table = RoutingTable::new();
table.set_exit(
network("later"),
ExitPolicy {
offer: true,
via: None,
},
);
assert!(table.is_empty());
}
}
+569
View File
@@ -0,0 +1,569 @@
//! The boundary to the operating system's packet interface.
//!
//! One agent has one of these, and it belongs to the agent rather than to
//! any protocol: a packet arriving here is routed to whichever peer owns its
//! destination, over whichever protocol has a link to that peer.
//!
//! Two implementations:
//!
//! * [`MemoryTun`] keeps packets in memory. It needs no privileges at all and
//! is what the test suite uses, so the entire data plane — handshake,
//! encryption, routing — is exercised without touching the host.
//! * `SystemTun`, behind the `tun-device` feature, is a real TUN interface.
//! Creating one needs `CAP_NET_ADMIN`, and it is
//! [`provision`](super::provision) that holds that and creates it.
use std::sync::Arc;
use bytes::Bytes;
use crate::BoxFuture;
use crate::overlay::OverlayError;
/// What a device should look like once created.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TunRequest {
/// Interface name to ask for.
pub name: String,
/// Every overlay address this host answers to, and no others.
///
/// One per network the agent is in — one agent has one interface, and
/// it carries them all. A list rather than one address because
/// carrying only the first is carrying only the first network: the
/// others have addresses the operating system has never heard of, and
/// their traffic goes nowhere.
///
/// Allocated and signed at the system level, never derived from a
/// protocol's key: every protocol carries traffic for the same
/// addresses.
pub addresses: Vec<super::Cidr>,
/// Interface MTU.
pub mtu: u32,
}
impl TunRequest {
/// A request carrying nothing but a name and an MTU.
///
/// Used where the addresses have already been applied to the host, so the
/// device itself only needs opening.
pub fn bare(name: impl Into<String>, mtu: u32) -> Self {
Self {
name: name.into(),
addresses: Vec::new(),
mtu,
}
}
}
/// A packet interface.
///
/// `recv` yields packets the operating system wants sent; `send` delivers
/// packets that arrived from a peer.
pub trait TunDevice: Send + Sync + std::fmt::Debug + 'static {
/// The interface name the operating system actually gave us.
fn name(&self) -> &str;
/// The interface MTU.
fn mtu(&self) -> u32;
/// The next packet the operating system wants to send, or `None` once the
/// device is gone.
fn recv(&self) -> BoxFuture<'_, Option<Bytes>>;
/// Delivers a packet to the operating system.
fn send<'a>(&'a self, packet: Bytes) -> BoxFuture<'a, Result<(), OverlayError>>;
}
/// Creates packet interfaces.
pub trait TunFactory: Send + Sync + std::fmt::Debug + 'static {
/// A short name used in diagnostics.
fn name(&self) -> &str;
/// Whether what this factory creates exists on the host.
///
/// `false` for an in-memory device: it has a name and an MTU and nothing
/// else — no link, no addresses, nothing for the operating system to
/// route through or to attach a resolver setting to. Stated rather than
/// guessed from the name, because a name is not evidence: an in-memory
/// `tsun0` and a real `tsun0` belonging to another agent look identical
/// from here, and configuring the operating system for the first would
/// land on the second.
fn on_host(&self) -> bool;
/// Creates a device.
fn create<'a>(
&'a self,
request: TunRequest,
) -> BoxFuture<'a, Result<Arc<dyn TunDevice>, OverlayError>>;
/// Applies a changed request to an interface that already exists.
///
/// The overlay IPv4 address is allocated at run time, so it can change
/// while the agent runs. A factory that manages the host applies that to
/// the live interface, without recreating it: recreating would drop every
/// tunnel riding on it.
///
/// The default does nothing, which is right for a factory that only
/// attaches to an interface somebody else prepared.
fn reconfigure<'a>(&'a self, _request: TunRequest) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move { Ok(()) })
}
/// The host rules for this factory's interface, when it manages the host.
///
/// `None` for an in-memory device or an interface somebody else prepared:
/// there is nothing of ours to route or to open in a firewall.
fn host_rules(&self) -> Option<Arc<dyn super::hostrules::BroadcastHostRules>> {
None
}
/// Removes an interface this factory created.
///
/// Runs on the teardown path, so it reports rather than fails: there is
/// nothing useful to do about a failure at that point, and an interface
/// that is already gone is the desired outcome anyway.
fn destroy<'a>(&'a self, _name: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {})
}
}
/// An in-memory packet interface.
///
/// Nothing reaches the operating system. Packets the device "sends" can be
/// read back with [`MemoryTun::pop_to_os`], and packets can be injected as if
/// the operating system produced them with [`MemoryTun::push_from_os`].
#[derive(Debug)]
pub struct MemoryTun {
name: String,
mtu: u32,
from_os_tx: tokio::sync::mpsc::UnboundedSender<Bytes>,
from_os_rx: tokio::sync::Mutex<tokio::sync::mpsc::UnboundedReceiver<Bytes>>,
to_os_tx: tokio::sync::mpsc::UnboundedSender<Bytes>,
to_os_rx: tokio::sync::Mutex<tokio::sync::mpsc::UnboundedReceiver<Bytes>>,
/// Set once the device is gone, so a reader stops rather than waiting
/// for a packet that will never come.
///
/// A `watch` rather than a `Notify`: closing must be seen by a reader
/// that has not started waiting yet as well as by one already waiting —
/// a device can be removed before the loop reading it has been polled
/// even once. The receiver is kept for the same reason: a `watch` send
/// with nobody subscribed does nothing at all.
closed: tokio::sync::watch::Sender<bool>,
closed_rx: tokio::sync::watch::Receiver<bool>,
}
impl MemoryTun {
/// Creates a device with the given name and MTU.
pub fn new(name: impl Into<String>, mtu: u32) -> Arc<Self> {
let (from_os_tx, from_os_rx) = tokio::sync::mpsc::unbounded_channel();
let (to_os_tx, to_os_rx) = tokio::sync::mpsc::unbounded_channel();
let (closed, closed_rx) = tokio::sync::watch::channel(false);
Arc::new(Self {
name: name.into(),
mtu,
from_os_tx,
from_os_rx: tokio::sync::Mutex::new(from_os_rx),
to_os_tx,
to_os_rx: tokio::sync::Mutex::new(to_os_rx),
closed,
closed_rx,
})
}
/// Marks the device gone, so the packet loop reading it ends.
///
/// A real interface reports end of stream when it is removed; this is
/// how that happens here. Without it teardown waits out the whole grace
/// period for a reader that had no way of knowing.
pub fn close(&self) {
let _ = self.closed.send(true);
}
/// Injects a packet as if the operating system had produced it.
pub fn push_from_os(&self, packet: Bytes) {
let _ = self.from_os_tx.send(packet);
}
/// Takes the next packet the device delivered to the operating system.
pub async fn pop_to_os(&self) -> Option<Bytes> {
self.to_os_rx.lock().await.recv().await
}
}
impl TunDevice for MemoryTun {
fn name(&self) -> &str {
&self.name
}
fn mtu(&self) -> u32 {
self.mtu
}
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
Box::pin(async move {
let mut closed = self.closed_rx.clone();
if *closed.borrow_and_update() {
return None;
}
let mut queue = self.from_os_rx.lock().await;
tokio::select! {
packet = queue.recv() => packet,
_ = closed.changed() => None,
}
})
}
fn send<'a>(&'a self, packet: Bytes) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move {
let _ = self.to_os_tx.send(packet);
Ok(())
})
}
}
/// Whether an address is assigned to some interface on this host.
///
/// Binding a UDP socket to a specific address only succeeds when the address
/// is local, which makes this a cheap check that needs no privileges and no
/// platform-specific code. It does not say *which* interface has it, which is
/// enough here: the agent chose the address, so anything else holding it is a
/// problem in its own right.
pub fn address_is_local(address: std::net::IpAddr) -> bool {
if std::net::UdpSocket::bind((address, 0)).is_ok() {
return true;
}
#[cfg(all(feature = "tun-device", target_os = "windows"))]
{
if let Ok(interfaces) = std::panic::catch_unwind(netdev::get_interfaces) {
for iface in interfaces {
if iface.is_up() {
match address {
std::net::IpAddr::V4(v4) => {
if iface.ipv4.iter().any(|ip| ip.addr() == v4) {
return true;
}
}
std::net::IpAddr::V6(v6) => {
if iface.ipv6.iter().any(|ip| ip.addr() == v6) {
return true;
}
}
}
}
}
}
}
false
}
/// Creates [`MemoryTun`] devices.
#[derive(Debug, Clone, Default)]
pub struct MemoryTunFactory {
created: Arc<std::sync::Mutex<Vec<Arc<MemoryTun>>>>,
}
impl MemoryTunFactory {
/// Creates a factory.
pub fn new() -> Self {
Self::default()
}
/// The device created for an interface name, if any.
pub fn device(&self, name: &str) -> Option<Arc<MemoryTun>> {
let guard = match self.created.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard
.iter()
.find(|device| device.name() == name)
.map(Arc::clone)
}
/// Every device created so far.
pub fn devices(&self) -> Vec<Arc<MemoryTun>> {
let guard = match self.created.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard.clone()
}
}
impl TunFactory for MemoryTunFactory {
fn name(&self) -> &str {
"memory"
}
/// Nothing here is on the host, which is the whole point of it.
fn on_host(&self) -> bool {
false
}
/// Closes the device so whatever is reading it stops.
///
/// The device itself is kept, because a test asks what went through it
/// after the agent that owned it has gone.
fn destroy<'a>(&'a self, name: &'a str) -> BoxFuture<'a, ()> {
Box::pin(async move {
if let Some(device) = self.device(name) {
device.close();
}
})
}
fn create<'a>(
&'a self,
request: TunRequest,
) -> BoxFuture<'a, Result<Arc<dyn TunDevice>, OverlayError>> {
Box::pin(async move {
let device = MemoryTun::new(request.name, request.mtu);
let mut guard = match self.created.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard.push(Arc::clone(&device));
Ok(device as Arc<dyn TunDevice>)
})
}
}
#[cfg(feature = "tun-device")]
pub(crate) use system::open_tun;
#[cfg(feature = "tun-device")]
mod system {
use std::sync::Arc;
use bytes::Bytes;
use tokio::sync::Mutex;
use super::{TunDevice, TunRequest};
use crate::BoxFuture;
use crate::overlay::OverlayError;
/// A real TUN interface.
///
/// On Linux it is created by opening `/dev/net/tun`, which needs
/// `CAP_NET_ADMIN`; on Windows it is a Wintun adapter, which needs the
/// process to be elevated and `wintun.dll` to be reachable. Either way
/// [`open_tun`] is only ever called from
/// [`provision`](super::super::provision), which holds whatever privilege
/// the platform requires for the length of the call and no longer.
///
/// It is deliberately **not** made persistent, so the operating system
/// removes the interface when this value is dropped — however the process
/// ends. (A Wintun adapter created this way is likewise torn down when the
/// last handle to it closes, which the reader and writer below hold.)
pub struct SystemTun {
name: String,
mtu: u32,
reader: Mutex<tokio::io::ReadHalf<tun::AsyncDevice>>,
writer: Mutex<tokio::io::WriteHalf<tun::AsyncDevice>>,
}
impl std::fmt::Debug for SystemTun {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SystemTun")
.field("name", &self.name)
.field("mtu", &self.mtu)
.finish()
}
}
impl TunDevice for SystemTun {
fn name(&self) -> &str {
&self.name
}
fn mtu(&self) -> u32 {
self.mtu
}
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
Box::pin(async move {
use tokio::io::AsyncReadExt;
let mut buffer = vec![0u8; self.mtu as usize + 64];
let mut reader = self.reader.lock().await;
match reader.read(&mut buffer).await {
Ok(0) => None,
Ok(read) => {
buffer.truncate(read);
Some(Bytes::from(buffer))
}
Err(err) => {
tracing::debug!(%err, "tun read failed");
None
}
}
})
}
fn send<'a>(&'a self, packet: Bytes) -> BoxFuture<'a, Result<(), OverlayError>> {
Box::pin(async move {
use tokio::io::AsyncWriteExt;
let mut writer = self.writer.lock().await;
writer
.write_all(&packet)
.await
.map_err(|err| OverlayError::Other(format!("tun write failed: {err}")))
})
}
}
/// Creates the TUN interface by opening it.
///
/// Synchronous, and deliberately so: the caller holds a capability guard
/// across this call, and such a guard must not span an `await` because
/// Linux capabilities are per thread.
pub(crate) fn open_tun(request: &TunRequest) -> Result<Arc<dyn TunDevice>, OverlayError> {
// macOS mutates neither field below (it neither names the interface nor
// sets a platform config), so `mut` is unused only there.
#[cfg_attr(target_os = "macos", allow(unused_mut))]
let mut config = tun::Configuration::default();
// macOS names its tunnels `utunN` and assigns the unit number itself.
// A name derived from the network id (`tsun…`) is not a legal utun
// name, so there the kernel is left to choose and the real name is read
// back below. Linux and Windows take the requested name verbatim.
#[cfg(not(target_os = "macos"))]
config.tun_name(&request.name);
#[cfg(target_os = "linux")]
config.platform_config(|platform| {
// The crate's own root check is not the check we want: this holds
// CAP_NET_ADMIN without being root. Whether the open succeeds is
// the honest answer.
platform.ensure_root_privileges(false);
});
// Packet information stays off, so reads and writes are raw IP
// packets. `ip tuntap add ... mode tun` also defaults to no packet
// information, so the flags match when attaching to one. The address
// and MTU are left off the configuration on purpose: the provisioner
// applies those to the interface separately, the same way on every
// platform, so the plan stays the single source of what it carries.
let device = tun::create_as_async(&config).map_err(|err| open_error(&request.name, err))?;
// The interface name the operating system actually gave us: the
// requested one everywhere except macOS, where the kernel chose the
// utun unit and so the real name has to be read back off the device.
#[cfg(target_os = "macos")]
let name = {
use tun::AbstractDevice;
device.tun_name().unwrap_or_else(|_| request.name.clone())
};
#[cfg(not(target_os = "macos"))]
let name = request.name.clone();
let (reader, writer) = tokio::io::split(device);
Ok(Arc::new(SystemTun {
name,
mtu: request.mtu,
reader: Mutex::new(reader),
writer: Mutex::new(writer),
}) as Arc<dyn TunDevice>)
}
fn open_error(name: &str, error: tun::Error) -> OverlayError {
// Wintun preserves the Windows code in its I/O variant. Normalize it
// before testing permissions; matching localized error text would
// miss the same failure on another Windows language.
#[cfg(target_os = "windows")]
let error = match error {
tun::Error::WintunError(error) => tun::Error::Io(error.into()),
error => error,
};
let denied = match &error {
tun::Error::Io(error) => {
error.kind() == std::io::ErrorKind::PermissionDenied
|| (cfg!(target_os = "windows")
&& matches!(error.raw_os_error(), Some(5 | 740 | 1314)))
}
_ => false,
};
if denied {
let hint = if cfg!(target_os = "windows") {
"Open PowerShell or Command Prompt with 'Run as administrator' and \
start `tsng up` there."
} else if cfg!(target_os = "linux") {
"Start `tsng up` as root or grant CAP_NET_ADMIN with \
`sudo setcap cap_net_admin,cap_net_bind_service+p /path/to/tsunagi`."
} else if cfg!(target_os = "macos") {
"Creating a utun interface needs root on macOS; start `tsng up` \
with `sudo`, or install it as a root LaunchDaemon."
} else {
"Start `tsng up` with the privileges required to create a TUN interface."
};
return OverlayError::Unavailable(format!(
"permission denied while creating the TUN interface `{name}`. \
{hint} OS error: {error}"
));
}
OverlayError::Unavailable(format!(
"cannot create the TUN interface `{name}`: {error}. {}",
open_hint()
))
}
/// The platform-appropriate tail of a "cannot create the interface" error:
/// what privilege it needs and the fallback that always works.
fn open_hint() -> &'static str {
#[cfg(target_os = "linux")]
{
"Creating one needs CAP_NET_ADMIN; grant it with `setcap cap_net_admin,cap_net_bind_service+p`, \
or run with `--no-tun` to keep the tunnels off the operating system."
}
#[cfg(target_os = "windows")]
{
"Creating one needs an elevated process and `wintun.dll` on the search path; \
run as Administrator with the DLL beside the executable, or run with `--no-tun` \
to keep the tunnels off the operating system."
}
#[cfg(target_os = "macos")]
{
"Creating a utun interface needs root; start the agent with `sudo` or as a root \
LaunchDaemon, or run with `--no-tun` to keep the tunnels off the operating system."
}
#[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))]
{
"Run with `--no-tun` to keep the tunnels off the operating system."
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_tun_permission_error_explains_how_to_start_the_agent() {
let error = tun::Error::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied));
let message = open_error("tsun0", error).to_string();
assert!(message.contains("permission denied"), "{message}");
assert!(message.contains("tsun0"), "{message}");
assert!(message.contains("tsng up"), "{message}");
#[cfg(target_os = "windows")]
assert!(message.contains("Run as administrator"), "{message}");
#[cfg(target_os = "linux")]
assert!(message.contains("CAP_NET_ADMIN"), "{message}");
}
#[cfg(target_os = "windows")]
#[test]
fn wintun_access_and_elevation_errors_get_the_permission_hint() {
for code in [5, 740, 1314] {
let error = tun::Error::WintunError(std::io::Error::from_raw_os_error(code).into());
let message = open_error("tsun0", error).to_string();
assert!(message.contains("permission denied"), "{message}");
assert!(message.contains("Run as administrator"), "{message}");
assert!(message.contains(&format!("os error {code}")), "{message}");
assert!(!message.contains("wintun.dll"), "{message}");
}
}
#[test]
fn other_tun_errors_keep_their_cause_without_claiming_access_was_denied() {
let message = open_error("tsun0", tun::Error::InvalidName).to_string();
assert!(message.contains("invalid device tun name"), "{message}");
assert!(!message.contains("permission denied"), "{message}");
}
}
}
+80
View File
@@ -0,0 +1,80 @@
//! Length-prefixed framing over one QUIC bidirectional stream.
//!
//! A frame is `u32_be(len) || payload`. The announced length is validated
//! against the configured limit **before** a buffer of that size is allocated,
//! so a hostile peer cannot make the agent allocate arbitrary memory with a
//! four byte header.
//!
//! No custom encryption is layered on top: the iroh/QUIC connection already
//! provides confidentiality, integrity and endpoint authentication.
use iroh::endpoint::{RecvStream, SendStream};
use crate::error::ProtocolError;
/// Size of the frame length prefix, in bytes.
pub const LENGTH_PREFIX_LEN: usize = 4;
/// Writes one frame.
pub async fn write_frame(
stream: &mut SendStream,
payload: &[u8],
max_frame_len: usize,
) -> Result<(), ProtocolError> {
if payload.len() > max_frame_len {
return Err(ProtocolError::FrameTooLarge {
announced: payload.len() as u64,
limit: max_frame_len,
});
}
let len = u32::try_from(payload.len()).map_err(|_| ProtocolError::FrameTooLarge {
announced: payload.len() as u64,
limit: max_frame_len,
})?;
stream
.write_all(&len.to_be_bytes())
.await
.map_err(|err| ProtocolError::Stream(err.to_string()))?;
stream
.write_all(payload)
.await
.map_err(|err| ProtocolError::Stream(err.to_string()))?;
Ok(())
}
/// Reads one frame, rejecting oversized headers before allocating.
pub async fn read_frame(
stream: &mut RecvStream,
max_frame_len: usize,
) -> Result<Vec<u8>, ProtocolError> {
let mut header = [0u8; LENGTH_PREFIX_LEN];
match stream.read_exact(&mut header).await {
Ok(()) => {}
Err(err) => return Err(classify_read_error(err)),
}
let announced = u32::from_be_bytes(header) as u64;
if announced > max_frame_len as u64 {
return Err(ProtocolError::FrameTooLarge {
announced,
limit: max_frame_len,
});
}
// Safe: `announced` was just bounded by `max_frame_len`, a usize.
let mut payload = vec![0u8; announced as usize];
if !payload.is_empty() {
match stream.read_exact(&mut payload).await {
Ok(()) => {}
Err(err) => return Err(classify_read_error(err)),
}
}
Ok(payload)
}
fn classify_read_error(err: iroh::endpoint::ReadExactError) -> ProtocolError {
match err {
iroh::endpoint::ReadExactError::FinishedEarly(_) => ProtocolError::StreamClosed,
other => ProtocolError::Stream(other.to_string()),
}
}
+575
View File
@@ -0,0 +1,575 @@
//! Mutual proof that both ends belong to the same network space.
//!
//! # Why a successful iroh connection is not enough
//!
//! iroh authenticates *endpoints*: after the QUIC/TLS handshake each side knows
//! the other's [`EndpointId`], because that id is the public key in the
//! certificate. It says nothing about network membership — anybody can dial us.
//! So on top of the authenticated connection we run an explicit mutual proof of
//! knowledge of the derived network authentication key.
//!
//! # Channel binding is not a proof by itself
//!
//! iroh exposes the TLS exporter (RFC 5705) via
//! [`Connection::export_keying_material`]. That gives both sides the same
//! secret bytes for *this* connection, which is exactly what is needed to stop
//! a proof being replayed on another connection. It proves nothing about the
//! shared network secret on its own, because both ends of any connection can
//! compute it. The proof of membership is the HMAC keyed by `auth_key`; the
//! exporter output is only one of its inputs.
//!
//! # The scheme
//!
//! ```text
//! cb = TLS-Exporter(label = "tsunagi/handshake/v1", context = network_id, 32)
//! LP(x)= u32_be(len(x)) || x
//!
//! transcript(role) = LP("tsunagi-handshake-v1")
//! || LP(role) // "initiator-proof" | "responder-proof"
//! || LP(u16_be(protocol_version))
//! || LP(network_id) // 32 bytes
//! || LP(initiator_endpoint_id) // 32 bytes
//! || LP(responder_endpoint_id) // 32 bytes
//! || LP(cb) // 32 bytes
//! || LP(nonce_initiator) // 16 bytes
//! || LP(nonce_responder) // 16 bytes
//!
//! proof(role) = HMAC-SHA256(auth_key, transcript(role))
//! ```
//!
//! What each input buys:
//!
//! * `auth_key` — membership. Derived from name+secret only, see
//! [`crate::identity`].
//! * `cb` — binding to this connection. A proof captured elsewhere is useless
//! here, because `cb` differs per TLS session.
//! * `network_id` — binding to this network space.
//! * both endpoint ids — binding to these two identities.
//! * distinct `role` labels — no reflection: the responder cannot bounce the
//! initiator's own proof back at it.
//! * both nonces — freshness contributed by each side.
//!
//! # Message order
//!
//! ```text
//! initiator -> responder : Hello { version, network_id, nonce_i }
//! initiator <- responder : HelloAck { version, nonce_r }
//! initiator -> responder : AuthProof{ proof(initiator) }
//! initiator <- responder : AuthProof{ proof(responder) } // only if the first proof verified
//! ```
//!
//! The responder emits nothing derived from `auth_key` until the initiator's
//! proof has verified, so a caller who does not know the secret learns nothing.
//! Until both steps complete, no regular control message is accepted in either
//! direction.
//!
//! [`Connection::export_keying_material`]: iroh::endpoint::Connection::export_keying_material
use hmac::{Hmac, KeyInit, Mac};
use iroh::EndpointId;
use iroh::endpoint::{Connection, RecvStream, SendStream};
use sha2::Sha256;
use zeroize::Zeroizing;
use crate::config::Limits;
use crate::error::ProtocolError;
use crate::identity::{NetworkId, NetworkKeys};
use crate::proto::frame::{read_frame, write_frame};
use crate::proto::message::{AuthProof, Hello, HelloAck, PROTOCOL_VERSION, decode, encode};
/// Frozen domain separator of the handshake transcript.
pub const TRANSCRIPT_DOMAIN: &str = "tsunagi-handshake-v1";
/// TLS exporter label used for channel binding.
pub const EXPORTER_LABEL: &[u8] = b"tsunagi/handshake/v1";
/// Transcript role label of the side that dialled.
pub const ROLE_INITIATOR: &str = "initiator-proof";
/// Transcript role label of the side that accepted.
pub const ROLE_RESPONDER: &str = "responder-proof";
/// Length of the channel binding material, in bytes.
pub const CHANNEL_BINDING_LEN: usize = 32;
/// Which side of the handshake this agent played.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Role {
/// This agent dialled.
Initiator,
/// This agent accepted.
Responder,
}
impl Role {
/// Short label for diagnostics.
pub fn as_str(&self) -> &'static str {
match self {
Role::Initiator => "initiator",
Role::Responder => "responder",
}
}
}
/// Result of a completed handshake.
#[derive(Debug, Clone)]
pub struct HandshakeOutcome {
/// Network both sides proved membership of.
pub network_id: NetworkId,
/// Authenticated endpoint id of the peer, taken from the TLS certificate.
pub peer: EndpointId,
/// Which side this agent played.
pub role: Role,
}
fn push_lp(out: &mut Vec<u8>, bytes: &[u8]) {
let len = u32::try_from(bytes.len()).unwrap_or(u32::MAX);
out.extend_from_slice(&len.to_be_bytes());
out.extend_from_slice(bytes);
}
/// Builds the role-specific transcript. Pure function, unit tested.
#[allow(clippy::too_many_arguments)]
pub fn transcript(
role: &str,
version: u16,
network_id: &[u8; 32],
initiator: &[u8; 32],
responder: &[u8; 32],
channel_binding: &[u8],
nonce_initiator: &[u8; 16],
nonce_responder: &[u8; 16],
) -> Vec<u8> {
let mut out = Vec::with_capacity(256);
push_lp(&mut out, TRANSCRIPT_DOMAIN.as_bytes());
push_lp(&mut out, role.as_bytes());
push_lp(&mut out, &version.to_be_bytes());
push_lp(&mut out, network_id);
push_lp(&mut out, initiator);
push_lp(&mut out, responder);
push_lp(&mut out, channel_binding);
push_lp(&mut out, nonce_initiator);
push_lp(&mut out, nonce_responder);
out
}
/// Computes one proof.
#[allow(clippy::too_many_arguments)]
fn proof(
auth_key: &[u8; 32],
role: &str,
version: u16,
network_id: &[u8; 32],
initiator: &[u8; 32],
responder: &[u8; 32],
channel_binding: &[u8],
nonce_initiator: &[u8; 16],
nonce_responder: &[u8; 16],
) -> [u8; 32] {
let message = transcript(
role,
version,
network_id,
initiator,
responder,
channel_binding,
nonce_initiator,
nonce_responder,
);
let mut mac = match <Hmac<Sha256> as KeyInit>::new_from_slice(auth_key) {
Ok(mac) => mac,
// HMAC-SHA256 accepts keys of any length, so a 32 byte key cannot fail.
Err(_) => unreachable!("HMAC-SHA256 accepts a 32 byte key"),
};
mac.update(&message);
let tag = mac.finalize().into_bytes();
let mut out = [0u8; 32];
out.copy_from_slice(&tag);
out
}
/// Verifies a proof in constant time.
#[allow(clippy::too_many_arguments)]
fn verify(
auth_key: &[u8; 32],
role: &str,
version: u16,
network_id: &[u8; 32],
initiator: &[u8; 32],
responder: &[u8; 32],
channel_binding: &[u8],
nonce_initiator: &[u8; 16],
nonce_responder: &[u8; 16],
candidate: &[u8; 32],
) -> Result<(), ProtocolError> {
let message = transcript(
role,
version,
network_id,
initiator,
responder,
channel_binding,
nonce_initiator,
nonce_responder,
);
let mut mac = match <Hmac<Sha256> as KeyInit>::new_from_slice(auth_key) {
Ok(mac) => mac,
Err(_) => unreachable!("HMAC-SHA256 accepts a 32 byte key"),
};
mac.update(&message);
mac.verify_slice(candidate)
.map_err(|_| ProtocolError::AuthenticationFailed)
}
/// Extracts channel binding material from the connection.
fn channel_binding(
conn: &Connection,
network_id: &[u8; 32],
) -> Result<Zeroizing<[u8; CHANNEL_BINDING_LEN]>, ProtocolError> {
let mut out = Zeroizing::new([0u8; CHANNEL_BINDING_LEN]);
conn.export_keying_material(out.as_mut(), EXPORTER_LABEL, network_id)
.map_err(|err| ProtocolError::NoChannelBinding(format!("{err:?}")))?;
Ok(out)
}
fn fresh_nonce() -> [u8; 16] {
let mut nonce = [0u8; 16];
rand::fill(&mut nonce);
nonce
}
fn check_version(found: u16) -> Result<(), ProtocolError> {
if found != PROTOCOL_VERSION {
return Err(ProtocolError::UnsupportedVersion {
found,
supported: PROTOCOL_VERSION,
});
}
Ok(())
}
/// Runs the initiator side of the handshake.
///
/// Bounded by [`Limits::handshake_timeout`].
pub async fn initiate(
conn: &Connection,
send: &mut SendStream,
recv: &mut RecvStream,
local_id: EndpointId,
keys: &NetworkKeys,
limits: &Limits,
) -> Result<HandshakeOutcome, ProtocolError> {
tokio::time::timeout(
limits.handshake_timeout,
initiate_inner(conn, send, recv, local_id, keys, limits),
)
.await
.unwrap_or(Err(ProtocolError::HandshakeTimeout))
}
async fn initiate_inner(
conn: &Connection,
send: &mut SendStream,
recv: &mut RecvStream,
local_id: EndpointId,
keys: &NetworkKeys,
limits: &Limits,
) -> Result<HandshakeOutcome, ProtocolError> {
let network_id = keys.network_id();
let network_bytes = *network_id.as_bytes();
let peer = conn.remote_id();
let initiator = *local_id.as_bytes();
let responder = *peer.as_bytes();
let cb = channel_binding(conn, &network_bytes)?;
let nonce_i = fresh_nonce();
let hello = Hello {
version: PROTOCOL_VERSION,
network_id: network_bytes,
nonce: nonce_i,
};
write_frame(send, &encode(&hello)?, limits.max_frame_len).await?;
let ack: HelloAck = decode(&read_frame(recv, limits.max_frame_len).await?)?;
check_version(ack.version)?;
let nonce_r = ack.nonce;
let mine = proof(
keys.auth_key(),
ROLE_INITIATOR,
PROTOCOL_VERSION,
&network_bytes,
&initiator,
&responder,
cb.as_ref(),
&nonce_i,
&nonce_r,
);
write_frame(
send,
&encode(&AuthProof { proof: mine })?,
limits.max_frame_len,
)
.await?;
let theirs: AuthProof = decode(&read_frame(recv, limits.max_frame_len).await?)?;
verify(
keys.auth_key(),
ROLE_RESPONDER,
PROTOCOL_VERSION,
&network_bytes,
&initiator,
&responder,
cb.as_ref(),
&nonce_i,
&nonce_r,
&theirs.proof,
)?;
Ok(HandshakeOutcome {
network_id,
peer,
role: Role::Initiator,
})
}
/// Runs the responder side of the handshake.
///
/// `lookup` maps the network id the peer asked for to the local key material,
/// returning `None` if this agent does not have that network active. Routing
/// stays in the agent; the protocol stays here.
///
/// Bounded by [`Limits::handshake_timeout`].
pub async fn respond<F>(
conn: &Connection,
send: &mut SendStream,
recv: &mut RecvStream,
local_id: EndpointId,
limits: &Limits,
lookup: F,
) -> Result<HandshakeOutcome, ProtocolError>
where
F: FnOnce(NetworkId) -> Option<NetworkKeys>,
{
tokio::time::timeout(
limits.handshake_timeout,
respond_inner(conn, send, recv, local_id, limits, lookup),
)
.await
.unwrap_or(Err(ProtocolError::HandshakeTimeout))
}
async fn respond_inner<F>(
conn: &Connection,
send: &mut SendStream,
recv: &mut RecvStream,
local_id: EndpointId,
limits: &Limits,
lookup: F,
) -> Result<HandshakeOutcome, ProtocolError>
where
F: FnOnce(NetworkId) -> Option<NetworkKeys>,
{
let hello: Hello = decode(&read_frame(recv, limits.max_frame_len).await?)?;
check_version(hello.version)?;
let network_id = NetworkId::from_bytes(hello.network_id);
let keys = lookup(network_id).ok_or(ProtocolError::UnknownNetwork)?;
let peer = conn.remote_id();
let network_bytes = hello.network_id;
let initiator = *peer.as_bytes();
let responder = *local_id.as_bytes();
let cb = channel_binding(conn, &network_bytes)?;
let nonce_i = hello.nonce;
let nonce_r = fresh_nonce();
let ack = HelloAck {
version: PROTOCOL_VERSION,
nonce: nonce_r,
};
write_frame(send, &encode(&ack)?, limits.max_frame_len).await?;
let theirs: AuthProof = decode(&read_frame(recv, limits.max_frame_len).await?)?;
verify(
keys.auth_key(),
ROLE_INITIATOR,
PROTOCOL_VERSION,
&network_bytes,
&initiator,
&responder,
cb.as_ref(),
&nonce_i,
&nonce_r,
&theirs.proof,
)?;
// Only now, after the peer proved membership, do we emit our own proof.
let mine = proof(
keys.auth_key(),
ROLE_RESPONDER,
PROTOCOL_VERSION,
&network_bytes,
&initiator,
&responder,
cb.as_ref(),
&nonce_i,
&nonce_r,
);
write_frame(
send,
&encode(&AuthProof { proof: mine })?,
limits.max_frame_len,
)
.await?;
Ok(HandshakeOutcome {
network_id,
peer,
role: Role::Responder,
})
}
/// Test-only re-export of [`channel_binding`]. See [`crate::test_support`].
#[doc(hidden)]
pub fn channel_binding_for_test(
conn: &Connection,
network_id: &[u8; 32],
) -> Result<[u8; 32], ProtocolError> {
channel_binding(conn, network_id).map(|cb| *cb)
}
/// Test-only re-export of the proof function. See [`crate::test_support`].
#[doc(hidden)]
#[allow(clippy::too_many_arguments)]
pub fn proof_for_test(
auth_key: &[u8; 32],
role: &str,
version: u16,
network_id: &[u8; 32],
initiator: &[u8; 32],
responder: &[u8; 32],
channel_binding: &[u8],
nonce_initiator: &[u8; 16],
nonce_responder: &[u8; 16],
) -> [u8; 32] {
proof(
auth_key,
role,
version,
network_id,
initiator,
responder,
channel_binding,
nonce_initiator,
nonce_responder,
)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
/// `(auth_key, network_id, initiator_id, responder_id, nonce_i, nonce_r)`
type Fixture = ([u8; 32], [u8; 32], [u8; 32], [u8; 32], [u8; 16], [u8; 16]);
fn fixture() -> Fixture {
(
[1u8; 32], [2u8; 32], [3u8; 32], [4u8; 32], [5u8; 16], [6u8; 16],
)
}
#[test]
fn role_labels_produce_different_transcripts() {
let (key, net, ini, res, ni, nr) = fixture();
let cb = [7u8; 32];
let a = proof(&key, ROLE_INITIATOR, 1, &net, &ini, &res, &cb, &ni, &nr);
let b = proof(&key, ROLE_RESPONDER, 1, &net, &ini, &res, &cb, &ni, &nr);
assert_ne!(a, b, "reflecting a proof back must not verify");
}
#[test]
fn channel_binding_changes_the_proof() {
let (key, net, ini, res, ni, nr) = fixture();
let a = proof(
&key,
ROLE_INITIATOR,
1,
&net,
&ini,
&res,
&[7u8; 32],
&ni,
&nr,
);
let b = proof(
&key,
ROLE_INITIATOR,
1,
&net,
&ini,
&res,
&[8u8; 32],
&ni,
&nr,
);
assert_ne!(a, b, "a proof must not be replayable on another connection");
}
#[test]
fn identities_and_network_are_bound() {
let (key, net, ini, res, ni, nr) = fixture();
let cb = [7u8; 32];
let base = proof(&key, ROLE_INITIATOR, 1, &net, &ini, &res, &cb, &ni, &nr);
let other_net = proof(
&key,
ROLE_INITIATOR,
1,
&[9u8; 32],
&ini,
&res,
&cb,
&ni,
&nr,
);
let swapped = proof(&key, ROLE_INITIATOR, 1, &net, &res, &ini, &cb, &ni, &nr);
assert_ne!(base, other_net);
assert_ne!(base, swapped);
}
#[test]
fn transcript_encoding_is_unambiguous() {
// Two different field splits that would collide under naive concatenation.
let a = transcript(
"ab", 1, &[0u8; 32], &[0u8; 32], &[0u8; 32], b"cd", &[0u8; 16], &[0u8; 16],
);
let b = transcript(
"a", 1, &[0u8; 32], &[0u8; 32], &[0u8; 32], b"bcd", &[0u8; 16], &[0u8; 16],
);
assert_ne!(a, b);
}
#[test]
fn wrong_key_fails_verification() {
let (key, net, ini, res, ni, nr) = fixture();
let cb = [7u8; 32];
let tag = proof(&key, ROLE_INITIATOR, 1, &net, &ini, &res, &cb, &ni, &nr);
let wrong = [0xAAu8; 32];
let result = verify(
&wrong,
ROLE_INITIATOR,
1,
&net,
&ini,
&res,
&cb,
&ni,
&nr,
&tag,
);
assert!(matches!(result, Err(ProtocolError::AuthenticationFailed)));
}
}
+329
View File
@@ -0,0 +1,329 @@
//! Control message formats.
//!
//! This is deliberately a small, closed set of messages, not a general RPC
//! framework. Bodies are encoded with [postcard], a compact, deterministic,
//! non-self-describing serde format.
//!
//! Every message that arrives from the network passes [`validate`] against the
//! configured [`Limits`] before it reaches anything else.
//!
//! [postcard]: https://docs.rs/postcard
use serde::{Deserialize, Serialize};
use crate::config::Limits;
use crate::dataplane::{MAX_PROTOCOL_ID_LEN, PluginCapability};
use crate::error::ProtocolError;
/// ALPN of the tsunagi control plane.
///
/// The version in the ALPN is the wire-compatibility version of the control
/// protocol. It is independent of the network identity scheme version, so
/// bumping it must not change any existing [`crate::NetworkId`].
pub const ALPN: &[u8] = b"tsunagi/ctrl/4";
/// ALPN of the tsunagi data plane.
///
/// Data plane connections are deliberately separate from control plane ones.
/// They carry one IP plugin's packets for one network and nothing else, so a
/// saturated or broken data plane cannot disturb control traffic, and the
/// transport underneath can be replaced without touching the control protocol.
///
/// Version 4 adds source, destination, hop limit and stable flow id above
/// transport fragmentation. All members must upgrade together. Persistent
/// identities and network configuration do not change.
pub const DATA_ALPN: &[u8] = b"tsunagi/data/4";
/// Largest plugin protocol identifier accepted when opening a data channel.
pub const MAX_DATA_PROTOCOL_LEN: usize = 32;
/// Largest accepted signature on a signed record, in bytes.
pub const MAX_SIGNATURE_LEN: usize = 64;
/// Control protocol version carried inside the handshake.
pub const PROTOCOL_VERSION: u16 = 4;
/// First message of the handshake, sent by the initiator.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Hello {
/// Control protocol version the initiator speaks.
pub version: u16,
/// Public network identifier the initiator wants to join.
pub network_id: [u8; 32],
/// Initiator's fresh handshake nonce.
pub nonce: [u8; 16],
}
/// Responder's reply to [`Hello`]. Carries no proof yet.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HelloAck {
/// Control protocol version the responder speaks.
pub version: u16,
/// Responder's fresh handshake nonce.
pub nonce: [u8; 16],
}
/// A handshake proof, i.e. one HMAC tag over a role-specific transcript.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AuthProof {
/// HMAC-SHA256 tag.
pub proof: [u8; 32],
}
/// What this agent tells a peer about itself.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Announcement {
/// Local opt-in to receive LAN broadcasts in this network.
pub broadcast: bool,
/// This agent offers itself as an exit node in this network: members may
/// send all their internet traffic through it. Off unless its owner
/// turned it on, and only said once the rules that make it work are in
/// place.
pub exit_node: bool,
/// Human-readable hostname. A mutable binding, not an identity.
pub hostname: String,
/// Announced IP plugin capabilities. Opaque to the core.
pub capabilities: Vec<PluginCapability>,
}
/// Opens a data channel, sent by the initiator right after the membership
/// handshake on a [`DATA_ALPN`] connection.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DataOpen {
/// Which IP plugin's packets this channel will carry.
pub protocol: String,
/// Maximum reassembled payload the initiator is willing to receive.
pub max_datagram: u32,
}
/// The responder's answer to [`DataOpen`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DataOpenAck {
/// Whether the channel was accepted.
///
/// A channel is refused when the responder has no plugin for that
/// protocol in that network. That is an ordinary outcome, not an error.
pub accepted: bool,
/// Largest datagram the responder is willing to receive, in bytes.
pub max_datagram: u32,
}
/// A control message exchanged after a successful handshake.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub enum ControlMessage {
/// Hostname and capability announcement.
Announce(Announcement),
/// A small request used to verify that the exchange works.
Ping {
/// Caller-chosen sequence number, echoed back.
seq: u64,
/// Opaque bounded payload, echoed back.
payload: Vec<u8>,
},
/// The reply to a [`ControlMessage::Ping`].
Pong {
/// Sequence number of the request being answered.
seq: u64,
/// Echoed payload.
payload: Vec<u8>,
},
/// A snapshot of signed records this agent holds for the network.
///
/// A snapshot is merged into what the receiver already has, never
/// substituted for it: an author missing from the batch is left alone,
/// because absence is not deletion.
State {
/// The records. Bounded by [`crate::config::Limits::max_state_records`].
records: Vec<crate::state::SignedRecord>,
},
/// Members this sender knows of, and where it has seen them.
///
/// An introduction, not a vouching: everything here is an unverified
/// candidate, and membership is still decided by the handshake. It is
/// what turns a star around whoever was named on the command line into
/// a mesh — a device that was told about one member ends up talking to
/// all of them — and it is the same shape a lookup in a distributed
/// hash table would return.
Peers {
/// The members, with whatever addresses the sender has for them.
peers: Vec<PeerHint>,
},
/// Which peers this sender has a live data link with, right now.
///
/// First-hand and nothing else: a sender speaks only for itself, never
/// about what somebody else can reach. That is what makes it usable
/// without weighing hearsay — the claim is proved or disproved by
/// sending through it.
///
/// A snapshot rather than a change, because a snapshot is idempotent
/// and repairs itself; and volatile, so it lives here and not in
/// signed state, which is for what has to survive a member being away.
Reachable {
/// Direct links, scoped to the plugin protocol they actually carry.
links: Vec<ReachableLink>,
},
/// Graceful goodbye.
///
/// A peer going away is not a revocation of anything.
Bye {
/// Short free-text reason.
reason: String,
},
}
/// Somewhere a member has been seen.
///
/// Addresses are in iroh's own text form, which is what the endpoint takes
/// back: this is a hint to try, never a fact about where anybody is.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PeerHint {
/// The member's endpoint key.
pub endpoint: [u8; 32],
/// Where the sender has seen it, as `ip:<socket>` or `relay:<url>`.
///
/// The same spelling the disposable cache uses, so one decoder serves
/// both and neither can drift from the other. An empty list is still
/// worth sending: an id alone is enough for a receiver whose endpoint
/// can resolve it.
pub addrs: Vec<String>,
}
/// One directly observed edge, advertised only by its authenticated source.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ReachableLink {
/// The other end of this transport link.
pub peer: [u8; 32],
/// Protocol carried by the link.
pub protocol: String,
}
/// Longest address text accepted in a hint.
pub const MAX_HINT_ADDR_LEN: usize = 128;
/// Most addresses accepted for one member in a hint.
pub const MAX_HINT_ADDRS: usize = 8;
/// A control message together with the network it belongs to.
///
/// Every session is bound to exactly one network at handshake time. The
/// `network_id` here is re-checked on every message, so an authenticated
/// session for network A can never be used to speak to network B, even over a
/// shared physical connection.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Envelope {
/// Network this message belongs to.
pub network_id: [u8; 32],
/// The message itself.
pub message: ControlMessage,
}
/// Encodes a value into a postcard byte vector.
pub fn encode<T: Serialize>(value: &T) -> Result<Vec<u8>, ProtocolError> {
postcard::to_stdvec(value).map_err(|_| ProtocolError::Malformed("cannot encode message"))
}
/// Decodes a value from postcard bytes.
pub fn decode<T: for<'de> Deserialize<'de>>(bytes: &[u8]) -> Result<T, ProtocolError> {
postcard::from_bytes(bytes).map_err(|_| ProtocolError::Malformed("cannot decode message"))
}
fn check_len(field: &'static str, len: usize, limit: usize) -> Result<(), ProtocolError> {
if len > limit {
return Err(ProtocolError::FieldTooLarge { field, len, limit });
}
Ok(())
}
/// Validates a decoded capability against the configured limits.
pub fn validate_capability(
capability: &PluginCapability,
limits: &Limits,
) -> Result<(), ProtocolError> {
if capability.protocol.is_empty() {
return Err(ProtocolError::Malformed("empty plugin protocol id"));
}
check_len(
"capability.protocol",
capability.protocol.len(),
MAX_PROTOCOL_ID_LEN,
)?;
check_len(
"capability.data",
capability.data.len(),
limits.max_capability_data_len,
)?;
Ok(())
}
/// Validates a decoded control message against the configured limits.
///
/// Returning an error rejects that single message. It never stops the session's
/// network, the other networks or the agent.
pub fn validate(message: &ControlMessage, limits: &Limits) -> Result<(), ProtocolError> {
match message {
ControlMessage::Peers { peers } => {
check_len("peers", peers.len(), limits.max_state_records)?;
for hint in peers {
check_len("peers.addrs", hint.addrs.len(), MAX_HINT_ADDRS)?;
for addr in &hint.addrs {
check_len("peers.addr", addr.len(), MAX_HINT_ADDR_LEN)?;
}
}
}
ControlMessage::Reachable { links } => {
check_len("reachable.links", links.len(), limits.max_state_records)?;
for link in links {
if link.protocol.is_empty() {
return Err(ProtocolError::Malformed("empty routing protocol"));
}
check_len(
"reachable.protocol",
link.protocol.len(),
MAX_PROTOCOL_ID_LEN,
)?;
}
}
ControlMessage::Announce(announcement) => {
check_len(
"announce.hostname",
announcement.hostname.len(),
limits.max_hostname_len,
)?;
check_len(
"announce.capabilities",
announcement.capabilities.len(),
limits.max_capabilities,
)?;
for capability in &announcement.capabilities {
validate_capability(capability, limits)?;
}
}
ControlMessage::Ping { payload, .. } | ControlMessage::Pong { payload, .. } => {
check_len("echo.payload", payload.len(), limits.max_echo_payload_len)?;
}
ControlMessage::State { records } => {
check_len("state.records", records.len(), limits.max_state_records)?;
for record in records {
check_len("state.signature", record.signature.len(), MAX_SIGNATURE_LEN)?;
}
}
ControlMessage::Bye { reason } => {
check_len("bye.reason", reason.len(), limits.max_reason_len)?;
}
}
Ok(())
}
/// A short, stable label for a message kind, for metrics and diagnostics.
pub fn kind(message: &ControlMessage) -> &'static str {
match message {
ControlMessage::Announce(_) => "announce",
ControlMessage::Ping { .. } => "ping",
ControlMessage::Pong { .. } => "pong",
ControlMessage::State { .. } => "state",
ControlMessage::Reachable { .. } => "reachable",
ControlMessage::Peers { .. } => "peers",
ControlMessage::Bye { .. } => "bye",
}
}
+35
View File
@@ -0,0 +1,35 @@
//! The control protocol: framing, messages and the network membership
//! handshake.
//!
//! Only control messages travel over iroh. User IP traffic is never tunnelled
//! through this protocol.
//!
//! Layering, outermost first:
//!
//! 1. iroh/QUIC connection with ALPN [`message::ALPN`] — endpoint
//! authentication, confidentiality and integrity.
//! 2. One bidirectional stream per session, carrying length-prefixed frames
//! ([`frame`]).
//! 3. The [`handshake`], which must complete before anything else is accepted.
//! 4. [`message::Envelope`]s carrying [`message::ControlMessage`]s, each
//! re-checked against the session's network id.
//!
//! Nothing here adds its own encryption on top of iroh.
//!
//! # The data plane speaks a different protocol
//!
//! IP plugin packets never travel on a control connection. They use their own
//! ALPN, [`message::DATA_ALPN`], with the same membership handshake followed by
//! [`message::DataOpen`]. Keeping them apart is what lets the data plane's
//! transport be replaced without touching anything above.
pub mod frame;
pub mod handshake;
pub mod message;
pub use frame::{read_frame, write_frame};
pub use handshake::{HandshakeOutcome, Role};
pub use message::{
ALPN, Announcement, AuthProof, ControlMessage, DATA_ALPN, DataOpen, DataOpenAck, Envelope,
Hello, HelloAck, PROTOCOL_VERSION,
};
+284
View File
@@ -0,0 +1,284 @@
//! Choosing a free overlay address.
//!
//! Allocation, not derivation. Derivation needs no coordination but cannot
//! avoid collisions in a space as small as IPv4; allocation avoids them but
//! has to look at what everybody else already holds. The signed records in
//! [`super`] are what makes that possible without a coordinator.
//!
//! The rules:
//!
//! * an address a participant already holds is kept, because stability across
//! an absence is the whole point;
//! * otherwise the search starts at a position derived from the participant's
//! own identity, so two participants joining at once rarely start in the
//! same place;
//! * the search then walks the range, so a free address is found whenever one
//! exists.
use std::collections::HashSet;
use std::net::Ipv4Addr;
use iroh::EndpointId;
use sha2::{Digest, Sha256};
use super::Ipv4Range;
use crate::identity::NetworkId;
/// Why no address could be allocated.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum AllocationError {
/// Every address in the range is taken.
#[error("the overlay range {range} is full: {holders} of {usable} addresses are taken")]
RangeFull {
/// The range that is full.
range: Ipv4Range,
/// How many are held.
holders: usize,
/// How many the range has.
usable: u64,
},
/// The range has no usable host addresses.
#[error("the overlay range {range} has no room for hosts")]
NoRoom {
/// The offending range.
range: Ipv4Range,
},
}
/// How many host addresses a range holds, excluding network and broadcast.
pub fn usable_addresses(range: Ipv4Range) -> u64 {
let host_bits = 32u32.saturating_sub(u32::from(range.prefix_len));
if host_bits < 2 {
return 0;
}
(1u64 << host_bits) - 2
}
/// The nth host address of a range.
fn address_at(range: Ipv4Range, offset: u64) -> Ipv4Addr {
let host_bits = 32u32.saturating_sub(u32::from(range.prefix_len));
let mask = if host_bits >= 32 {
0
} else {
u32::MAX << host_bits
};
let network_part = u32::from(range.base) & mask;
// Offsets run 1..=usable, so the network address is never handed out.
Ipv4Addr::from(network_part | ((offset % (1u64 << host_bits)) as u32))
}
/// Picks an address for `author`, keeping `current` if it is still usable.
///
/// `taken` is what every other participant is known to hold.
pub fn allocate(
network: NetworkId,
author: EndpointId,
range: Ipv4Range,
taken: &HashSet<Ipv4Addr>,
current: Option<Ipv4Addr>,
) -> Result<Ipv4Addr, AllocationError> {
let usable = usable_addresses(range);
if usable == 0 {
return Err(AllocationError::NoRoom { range });
}
// Keeping what we already hold is what lets a participant come back to
// the same address after any length of absence.
if let Some(current) = current
&& range.contains(current)
&& !taken.contains(&current)
{
return Ok(current);
}
// Start somewhere derived from who we are, so two newcomers do not both
// begin at the first address and collide every time.
let mut hash = Sha256::new();
hash.update(b"tsunagi-ipv4-allocation-v1");
hash.update(network.as_bytes());
hash.update(author.as_bytes());
let digest = hash.finalize();
let seed = u64::from_be_bytes([
digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
]);
for step in 0..usable {
let offset = ((seed.wrapping_add(step)) % usable) + 1;
let candidate = address_at(range, offset);
if !taken.contains(&candidate) {
return Ok(candidate);
}
}
Err(AllocationError::RangeFull {
range,
holders: taken.len(),
usable,
})
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::identity::{NetworkKeys, NetworkName, NetworkSecret};
use iroh::SecretKey;
fn network(name: &str) -> NetworkId {
NetworkKeys::derive(
&NetworkName::new(name).unwrap(),
&NetworkSecret::from_bytes(vec![7u8; 32]).unwrap(),
)
.network_id()
}
fn slash24() -> Ipv4Range {
"10.13.37.0/24".parse().unwrap()
}
#[test]
fn a_range_reports_its_usable_size() {
assert_eq!(usable_addresses(slash24()), 254);
assert_eq!(usable_addresses("10.0.0.0/16".parse().unwrap()), 65534);
assert_eq!(usable_addresses("10.0.0.0/30".parse().unwrap()), 2);
// The type refuses a /31, but the function is defensive anyway.
assert!("10.0.0.0/31".parse::<Ipv4Range>().is_err());
assert_eq!(
usable_addresses(Ipv4Range {
base: "10.0.0.0".parse().unwrap(),
prefix_len: 31
}),
0
);
}
#[test]
fn an_allocated_address_is_inside_the_range_and_not_its_edges() {
let id = network("inside");
let taken = HashSet::new();
for _ in 0..64 {
let author = SecretKey::generate().public();
let address = allocate(id, author, slash24(), &taken, None).unwrap();
assert!(slash24().contains(address));
assert_ne!(address.octets()[3], 0, "never the network address");
assert_ne!(address.octets()[3], 255, "never the broadcast address");
}
}
#[test]
fn an_address_already_held_is_kept() {
let id = network("sticky");
let author = SecretKey::generate().public();
let mine: Ipv4Addr = "10.13.37.42".parse().unwrap();
// This is what lets a participant return to the same address.
let taken = HashSet::new();
assert_eq!(
allocate(id, author, slash24(), &taken, Some(mine)).unwrap(),
mine
);
// Unless somebody else took it while we were away.
let taken = HashSet::from([mine]);
assert_ne!(
allocate(id, author, slash24(), &taken, Some(mine)).unwrap(),
mine
);
// Or unless the range changed under us.
let elsewhere: Ipv4Range = "10.99.0.0/16".parse().unwrap();
let moved = allocate(id, author, elsewhere, &HashSet::new(), Some(mine)).unwrap();
assert!(elsewhere.contains(moved));
}
#[test]
fn allocation_is_deterministic_and_spread_out() {
let id = network("spread");
// Fixed keys, not random ones. With 40 random authors in a /24 the
// birthday problem alone makes a handful of collisions likely, so a
// threshold on the count was a coin flip rather than a property.
let authors: Vec<_> = (0..40u8)
.map(|seed| SecretKey::from_bytes(&[seed; 32]).public())
.collect();
let first: Vec<_> = authors
.iter()
.map(|author| allocate(id, *author, slash24(), &HashSet::new(), None).unwrap())
.collect();
let again: Vec<_> = authors
.iter()
.map(|author| allocate(id, *author, slash24(), &HashSet::new(), None).unwrap())
.collect();
assert_eq!(first, again, "the same inputs give the same answer");
// Starting points are spread, so concurrent newcomers rarely clash.
let distinct: HashSet<_> = first.iter().collect();
assert!(
distinct.len() >= 35,
"only {} distinct starting points out of 40",
distinct.len()
);
}
#[test]
fn the_search_walks_past_everything_taken() {
let id = network("crowded");
let author = SecretKey::generate().public();
// Everything taken except one address.
let free: Ipv4Addr = "10.13.37.200".parse().unwrap();
let taken: HashSet<Ipv4Addr> = (1..=254u8)
.map(|host| Ipv4Addr::new(10, 13, 37, host))
.filter(|addr| *addr != free)
.collect();
assert_eq!(allocate(id, author, slash24(), &taken, None).unwrap(), free);
}
#[test]
fn a_full_range_is_an_error_rather_than_a_duplicate() {
let id = network("full");
let author = SecretKey::generate().public();
let taken: HashSet<Ipv4Addr> = (1..=254u8)
.map(|host| Ipv4Addr::new(10, 13, 37, host))
.collect();
assert!(matches!(
allocate(id, author, slash24(), &taken, None),
Err(AllocationError::RangeFull { .. })
));
assert!(matches!(
allocate(
id,
author,
Ipv4Range {
base: "10.0.0.0".parse().unwrap(),
prefix_len: 31
},
&HashSet::new(),
None
),
Err(AllocationError::NoRoom { .. })
));
}
#[test]
fn every_address_in_a_small_range_can_be_handed_out() {
let id = network("exhaustive");
let small: Ipv4Range = "10.13.37.0/29".parse().unwrap();
let mut taken = HashSet::new();
let mut handed = Vec::new();
for _ in 0..usable_addresses(small) {
let author = SecretKey::generate().public();
let address = allocate(id, author, small, &taken, None).unwrap();
assert!(taken.insert(address), "handed out {address} twice");
handed.push(address);
}
assert_eq!(handed.len(), 6);
// And then it is genuinely full.
let author = SecretKey::generate().public();
assert!(allocate(id, author, small, &taken, None).is_err());
}
}
File diff suppressed because it is too large. Load diff
+231
View File
@@ -0,0 +1,231 @@
//! The disposable cache store, `cache.sqlite`.
//!
//! Everything here is recoverable. A missing cache is recreated, a corrupt one
//! is thrown away and recreated, and a stale one is simply wrong data that the
//! rest of the system is expected to tolerate.
//!
//! Crucially, a stale cache never bypasses identity or network authentication:
//! cached hints only produce *candidates*, which still have to pass the
//! handshake.
use std::path::{Path, PathBuf};
use rusqlite::{Connection, params};
use crate::error::{Error, Result};
use crate::identity::NetworkId;
/// Schema version written by this build.
pub const SCHEMA_VERSION: i64 = 1;
/// A cached address hint for one peer in one network.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AddressHint {
/// Network the hint belongs to.
pub network_id: NetworkId,
/// Peer endpoint id, 32 bytes.
pub endpoint_id: [u8; 32],
/// Serialised address, currently `ip:<socketaddr>` or `relay:<url>`.
pub addr: String,
/// Unix seconds when this hint was last confirmed.
pub last_seen: i64,
}
/// Why the cache had to be recreated, if it did.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CacheOutcome {
/// Opened normally.
Opened,
/// Created because nothing was there.
Created,
/// Discarded and recreated. The reason is free of secrets.
Reset(String),
}
/// The disposable cache store.
#[derive(Debug)]
pub struct CacheStore {
conn: Connection,
path: PathBuf,
}
impl CacheStore {
/// Opens the cache, discarding and recreating it if it is unusable.
pub fn open_or_reset(path: impl AsRef<Path>) -> Result<(Self, CacheOutcome)> {
let path = path.as_ref().to_path_buf();
let existed = path.exists();
match Self::try_open(&path, existed) {
Ok(store) => Ok((
store,
if existed {
CacheOutcome::Opened
} else {
CacheOutcome::Created
},
)),
Err(reason) => {
tracing::warn!(path = %path.display(), %reason, "discarding unusable cache");
Self::remove_files(&path);
let store = Self::try_open(&path, false)
.map_err(|err| Error::Storage(format!("cannot recreate cache: {err}")))?;
Ok((store, CacheOutcome::Reset(reason)))
}
}
}
fn try_open(path: &Path, check_integrity: bool) -> std::result::Result<Self, String> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|err| format!("cannot create cache directory: {err}"))?;
}
let conn = Connection::open(path).map_err(|err| format!("cannot open: {err}"))?;
super::restrict_path_permissions(path).map_err(|err| format!("{err}"))?;
super::apply_common_pragmas(&conn).map_err(|err| format!("cannot configure: {err}"))?;
if check_integrity {
let integrity: String = conn
.query_row("PRAGMA integrity_check", [], |row| row.get(0))
.map_err(|err| format!("integrity check failed: {err}"))?;
if integrity != "ok" {
return Err(format!("integrity check reported: {integrity}"));
}
}
let found: i64 = conn
.query_row("PRAGMA user_version", [], |row| row.get(0))
.map_err(|err| format!("cannot read schema version: {err}"))?;
if found > SCHEMA_VERSION {
return Err(format!(
"cache schema version {found} is newer than {SCHEMA_VERSION}"
));
}
if found < SCHEMA_VERSION {
conn.execute_batch(
"BEGIN;
DROP TABLE IF EXISTS address_hints;
CREATE TABLE address_hints (
network_id BLOB NOT NULL,
endpoint_id BLOB NOT NULL,
addr TEXT NOT NULL,
last_seen INTEGER NOT NULL,
PRIMARY KEY (network_id, endpoint_id, addr)
);
PRAGMA user_version = 1;
COMMIT;",
)
.map_err(|err| format!("cannot create cache schema: {err}"))?;
} else {
conn.query_row("SELECT count(*) FROM address_hints", [], |row| {
row.get::<_, i64>(0)
})
.map_err(|err| format!("cache schema is unusable: {err}"))?;
}
Ok(Self {
conn,
path: path.to_path_buf(),
})
}
fn remove_files(path: &Path) {
for suffix in ["", "-wal", "-shm", "-journal"] {
let mut name = path.as_os_str().to_os_string();
name.push(suffix);
let _ = std::fs::remove_file(PathBuf::from(name));
}
}
/// Path of the underlying file.
pub fn path(&self) -> &Path {
&self.path
}
/// Records an address hint, keeping at most `max_per_peer` newest entries.
pub fn record_hint(
&self,
network_id: NetworkId,
endpoint_id: &[u8; 32],
addr: &str,
max_per_peer: usize,
) -> Result<()> {
self.conn
.execute(
"INSERT INTO address_hints (network_id, endpoint_id, addr, last_seen)
VALUES (?1, ?2, ?3, ?4)
ON CONFLICT(network_id, endpoint_id, addr)
DO UPDATE SET last_seen = excluded.last_seen",
params![
network_id.as_bytes().as_slice(),
endpoint_id.as_slice(),
addr,
super::state::now_unix()
],
)
.map_err(|err| Error::Storage(format!("cannot record address hint: {err}")))?;
self.conn
.execute(
"DELETE FROM address_hints
WHERE network_id = ?1 AND endpoint_id = ?2 AND addr NOT IN (
SELECT addr FROM address_hints
WHERE network_id = ?1 AND endpoint_id = ?2
ORDER BY last_seen DESC LIMIT ?3
)",
params![
network_id.as_bytes().as_slice(),
endpoint_id.as_slice(),
max_per_peer as i64
],
)
.map_err(|err| Error::Storage(format!("cannot prune address hints: {err}")))?;
Ok(())
}
/// Returns every hint known for a network.
pub fn hints_for_network(&self, network_id: NetworkId) -> Result<Vec<AddressHint>> {
let mut stmt = self
.conn
.prepare(
"SELECT endpoint_id, addr, last_seen FROM address_hints
WHERE network_id = ?1 ORDER BY last_seen DESC",
)
.map_err(|err| Error::Storage(format!("cannot read address hints: {err}")))?;
let rows = stmt
.query_map(params![network_id.as_bytes().as_slice()], |row| {
let endpoint_id: Vec<u8> = row.get(0)?;
let addr: String = row.get(1)?;
let last_seen: i64 = row.get(2)?;
Ok((endpoint_id, addr, last_seen))
})
.map_err(|err| Error::Storage(format!("cannot read address hints: {err}")))?;
let mut out = Vec::new();
for row in rows {
let (endpoint_id, addr, last_seen) =
row.map_err(|err| Error::Storage(format!("cannot read hint row: {err}")))?;
// A malformed row in a disposable store is skipped, not fatal.
let Ok(endpoint_id) = <[u8; 32]>::try_from(endpoint_id.as_slice()) else {
continue;
};
out.push(AddressHint {
network_id,
endpoint_id,
addr,
last_seen,
});
}
Ok(out)
}
/// Drops all hints for a network.
pub fn forget_network(&self, network_id: NetworkId) -> Result<()> {
self.conn
.execute(
"DELETE FROM address_hints WHERE network_id = ?1",
params![network_id.as_bytes().as_slice()],
)
.map_err(|err| Error::Storage(format!("cannot clear address hints: {err}")))?;
Ok(())
}
}
+64
View File
@@ -0,0 +1,64 @@
//! Ownership lock for a state directory.
//!
//! One persistent state directory belongs to exactly one live agent instance.
//! Checking whether a file exists is not enough — a stale file from a crashed
//! process must not block a restart, and two concurrently starting agents must
//! not both win. An advisory OS file lock gives both properties.
use std::fs::{File, OpenOptions};
use std::path::{Path, PathBuf};
use fs4::{FileExt, TryLockError};
use crate::error::{Error, Result};
/// An exclusive lock held for the lifetime of an agent.
///
/// Dropping it releases the lock, so a cleanly stopped agent leaves the
/// directory immediately reopenable.
#[derive(Debug)]
pub struct DirectoryLock {
file: File,
path: PathBuf,
}
impl DirectoryLock {
/// Acquires the lock, failing fast if another live agent holds it.
pub fn acquire(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref().to_path_buf();
let file = OpenOptions::new()
.create(true)
.read(true)
.write(true)
.truncate(false)
.open(&path)
.map_err(|source| Error::Io {
path: path.clone(),
source,
})?;
super::restrict_permissions(&file, &path)?;
match FileExt::try_lock(&file) {
Ok(()) => Ok(Self { file, path }),
Err(TryLockError::WouldBlock) => Err(Error::StateLocked {
path: path
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(|| path.clone()),
}),
Err(TryLockError::Error(source)) => Err(Error::Io { path, source }),
}
}
/// The path of the lock file.
pub fn path(&self) -> &Path {
&self.path
}
}
impl Drop for DirectoryLock {
fn drop(&mut self) {
// Best effort: the OS releases the lock when the descriptor closes anyway.
let _ = FileExt::unlock(&self.file);
}
}
+525
View File
@@ -0,0 +1,525 @@
//! Persistence, split into mandatory state and a disposable cache.
//!
//! | store | contents | on damage |
//! |----------------|--------------------------------------------------------|-----------|
//! | `state.sqlite` | device identity, network configuration, hostname | hard error |
//! | `cache.sqlite` | address hints and other recoverable data | discarded and recreated |
//!
//! Both files are created with owner-only permissions where the platform
//! supports it. The state directory additionally carries an ownership lock, see
//! [`DirectoryLock`].
//!
//! SQLite is synchronous. Every call that touches a database therefore runs on
//! a blocking pool via [`tokio::task::spawn_blocking`], and no database lock is
//! ever held across a network `await`.
mod cache;
mod lock;
mod state;
pub use cache::{AddressHint, CacheOutcome, CacheStore};
pub use lock::DirectoryLock;
pub use state::{SCHEMA_VERSION, StateStore, StoredNetwork};
use std::path::Path;
use std::sync::{Arc, Mutex, MutexGuard};
use crate::config::StoragePaths;
use crate::error::{Error, Result};
use crate::identity::{DeviceIdentity, NetworkId, NetworkName, NetworkSecret};
use crate::state::SignedRecord;
/// Applies the pragmas both stores share.
fn apply_common_pragmas(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
conn.busy_timeout(std::time::Duration::from_secs(5))?;
conn.pragma_update(None, "journal_mode", "WAL")?;
conn.pragma_update(None, "synchronous", "NORMAL")?;
conn.pragma_update(None, "foreign_keys", "ON")?;
Ok(())
}
/// Restricts a file to the current user where the platform supports it.
#[cfg(unix)]
fn restrict_permissions(file: &std::fs::File, path: &Path) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let perms = std::fs::Permissions::from_mode(0o600);
file.set_permissions(perms).map_err(|source| Error::Io {
path: path.to_path_buf(),
source,
})
}
/// On Windows, files inherit the parent directory's ACL, which for the
/// per-user application data directory is already restricted to that user.
#[cfg(not(unix))]
fn restrict_permissions(_file: &std::fs::File, _path: &Path) -> Result<()> {
Ok(())
}
/// Restricts a file to the user that owns it.
///
/// Part of what the system level offers a protocol: a plugin keeping keys of
/// its own on disk has the same obligation as the agent, and should not have
/// to work out the platform details again to meet it.
#[cfg(unix)]
pub fn restrict_path_permissions(path: &Path) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).map_err(|source| {
Error::Io {
path: path.to_path_buf(),
source,
}
})
}
/// Restricts a file to the user that owns it. A no-op off Unix.
#[cfg(not(unix))]
pub fn restrict_path_permissions(_path: &Path) -> Result<()> {
Ok(())
}
/// Restricts a directory to the user that owns it.
#[cfg(unix)]
pub fn restrict_dir_permissions(path: &Path) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).map_err(|source| {
Error::Io {
path: path.to_path_buf(),
source,
}
})
}
/// Restricts a directory to the user that owns it. A no-op off Unix.
#[cfg(not(unix))]
pub fn restrict_dir_permissions(_path: &Path) -> Result<()> {
Ok(())
}
/// Creates a directory and every parent, restricted to this user.
pub fn create_dir(path: &Path) -> Result<()> {
std::fs::create_dir_all(path).map_err(|source| Error::Io {
path: path.to_path_buf(),
source,
})?;
restrict_dir_permissions(path)
}
/// Async facade over both stores, holding the directory ownership lock.
///
/// Cloning shares the same underlying connections and the same lock.
#[derive(Debug, Clone)]
pub struct Storage {
inner: Arc<Inner>,
}
#[derive(Debug)]
struct Inner {
state: Mutex<StateStore>,
cache: Mutex<Option<CacheStore>>,
cache_outcome: CacheOutcome,
paths: StoragePaths,
lock: Mutex<Option<DirectoryLock>>,
}
impl Storage {
/// Opens both stores and takes the ownership lock on the state directory.
///
/// Fails with [`Error::StateLocked`] if another live agent owns the state
/// directory, and with [`Error::StateCorrupted`] if the mandatory state is
/// unusable. A broken cache is silently discarded and reported through
/// [`Storage::cache_outcome`].
pub fn open(paths: &StoragePaths) -> Result<Self> {
create_dir(&paths.state_dir)?;
create_dir(&paths.cache_dir)?;
let lock = DirectoryLock::acquire(paths.lock_file())?;
let state = StateStore::open(paths.state_db())?;
let (cache, cache_outcome) = CacheStore::open_or_reset(paths.cache_db())?;
Ok(Self {
inner: Arc::new(Inner {
state: Mutex::new(state),
cache: Mutex::new(Some(cache)),
cache_outcome,
paths: paths.clone(),
lock: Mutex::new(Some(lock)),
}),
})
}
/// What happened to the cache when the agent started.
pub fn cache_outcome(&self) -> &CacheOutcome {
&self.inner.cache_outcome
}
/// The configured paths.
pub fn paths(&self) -> &StoragePaths {
&self.inner.paths
}
fn lock_state(&self) -> MutexGuard<'_, StateStore> {
match self.inner.state.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
fn lock_cache(&self) -> MutexGuard<'_, Option<CacheStore>> {
match self.inner.cache.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
/// Runs a closure against the mandatory state store on the blocking pool.
async fn with_state<T, F>(&self, f: F) -> Result<T>
where
T: Send + 'static,
F: FnOnce(&StateStore) -> Result<T> + Send + 'static,
{
let inner = Arc::clone(&self.inner);
tokio::task::spawn_blocking(move || {
let guard = match inner.state.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
f(&guard)
})
.await
.map_err(|err| Error::Storage(format!("state task failed: {err}")))?
}
/// Runs a closure against the cache, tolerating an unavailable cache.
///
/// If the cache has been disabled because it misbehaved, the closure is
/// skipped and `default` is returned.
async fn with_cache<T, F>(&self, default: T, f: F) -> T
where
T: Send + 'static,
F: FnOnce(&CacheStore) -> Result<T> + Send + 'static,
{
let inner = Arc::clone(&self.inner);
let joined = tokio::task::spawn_blocking(move || {
let guard = match inner.cache.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
match guard.as_ref() {
Some(cache) => f(cache),
None => Err(Error::Storage("cache is unavailable".into())),
}
})
.await;
match joined {
Ok(Ok(value)) => value,
Ok(Err(err)) => {
tracing::debug!(%err, "cache operation failed; continuing without it");
default
}
Err(err) => {
tracing::debug!(%err, "cache task failed; continuing without it");
default
}
}
}
/// Loads or creates the persistent device identity.
pub async fn device_identity(&self) -> Result<DeviceIdentity> {
self.with_state(|state| state.load_or_create_device_identity())
.await
}
/// Lists configured networks.
pub async fn list_networks(&self) -> Result<Vec<StoredNetwork>> {
self.with_state(|state| state.list_networks()).await
}
/// Stores or updates a network configuration.
pub async fn upsert_network(
&self,
network_id: NetworkId,
name: NetworkName,
secret: NetworkSecret,
auto_start: bool,
) -> Result<()> {
self.with_state(move |state| state.upsert_network(network_id, &name, &secret, auto_start))
.await
}
/// Persists local broadcast participation for one configured network.
pub async fn set_broadcast(&self, network_id: NetworkId, enabled: bool) -> Result<()> {
self.with_state(move |state| state.set_broadcast(network_id, enabled))
.await
}
/// Persists whether this agent offers itself as an exit node in a network.
pub async fn set_exit_node(&self, network_id: NetworkId, enabled: bool) -> Result<()> {
self.with_state(move |state| state.set_exit_node(network_id, enabled))
.await
}
/// Persists the member this device sends its internet traffic through.
pub async fn set_exit_via(&self, network_id: NetworkId, via: Option<[u8; 32]>) -> Result<()> {
self.with_state(move |state| state.set_exit_via(network_id, via.as_ref()))
.await
}
/// Updates the auto-start flag of a network.
pub async fn set_auto_start(&self, network_id: NetworkId, auto_start: bool) -> Result<()> {
self.with_state(move |state| state.set_auto_start(network_id, auto_start))
.await
}
/// Removes a network configuration and its cached hints.
pub async fn remove_network(&self, network_id: NetworkId) -> Result<()> {
self.with_state(move |state| {
state.remove_network(network_id)?;
state.forget_signed_records(network_id)?;
state.forget_peer_hostnames(network_id)
})
.await?;
self.with_cache((), move |cache| cache.forget_network(network_id))
.await;
Ok(())
}
/// Reads the stored hostname.
pub async fn hostname(&self) -> Result<Option<String>> {
self.with_state(|state| state.hostname()).await
}
/// Writes the stored hostname.
pub async fn set_hostname(&self, hostname: String) -> Result<()> {
self.with_state(move |state| state.set_hostname(&hostname))
.await
}
/// Remembers the hostname a member announced, so it can still be named
/// once it is away.
pub async fn remember_peer_hostname(
&self,
network_id: NetworkId,
endpoint_id: [u8; 32],
hostname: String,
) -> Result<()> {
self.with_state(move |state| {
state.remember_peer_hostname(network_id, &endpoint_id, &hostname)
})
.await
}
/// The hostnames members of a network last announced.
pub async fn peer_hostnames(&self, network_id: NetworkId) -> Result<Vec<([u8; 32], String)>> {
self.with_state(move |state| state.peer_hostnames(network_id))
.await
}
/// Records an address hint. Failures are non-fatal.
pub async fn record_hint(
&self,
network_id: NetworkId,
endpoint_id: [u8; 32],
addr: String,
max_per_peer: usize,
) {
self.with_cache((), move |cache| {
cache.record_hint(network_id, &endpoint_id, &addr, max_per_peer)
})
.await;
}
/// Loads every signed record known for a network.
pub async fn signed_records(&self, network_id: NetworkId) -> Result<Vec<SignedRecord>> {
self.with_state(move |state| state.signed_records(network_id))
.await
}
/// Stores a record received from another replica.
pub async fn put_signed_record(&self, record: SignedRecord) -> Result<()> {
self.with_state(move |state| state.put_signed_record(&record))
.await
}
/// Stores one of this agent's own records and bumps its counter in one
/// transaction, which must happen before the record is announced.
pub async fn publish_own_record(&self, record: SignedRecord) -> Result<()> {
self.with_state(move |state| state.publish_own_record(&record))
.await
}
/// The highest version this agent has ever published for a network.
pub async fn own_record_version(
&self,
network_id: NetworkId,
author: iroh::EndpointId,
) -> Result<u64> {
self.with_state(move |state| state.own_record_version(network_id, author))
.await
}
/// Reads cached address hints. Returns an empty list if the cache is gone.
pub async fn hints_for_network(&self, network_id: NetworkId) -> Vec<AddressHint> {
self.with_cache(Vec::new(), move |cache| cache.hints_for_network(network_id))
.await
}
/// Synchronously reads the hostname. Used only during startup.
pub(crate) fn hostname_blocking(&self) -> Result<Option<String>> {
self.lock_state().hostname()
}
/// Whether the cache is currently usable.
pub fn cache_healthy(&self) -> bool {
self.lock_cache().is_some()
}
/// Releases the state directory ownership lock.
///
/// Called by [`crate::Agent::shutdown`] so that a cleanly stopped agent
/// leaves its directory immediately claimable by another instance. The
/// databases stay open and readable, but this handle no longer owns the
/// directory and must not be used to write after this point.
pub fn release_ownership_lock(&self) {
let mut guard = match self.inner.lock.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard.take();
}
}
/// What a wipe covers: every entry in the two directories.
///
/// Reported before anything is removed, and again afterwards, because a
/// command whose whole job is to destroy state has to be able to say exactly
/// what it is about to destroy.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct WipePlan {
/// Entries in the state directory, which is the mandatory state and
/// anything a plugin keeps beside it.
pub state: Vec<std::path::PathBuf>,
/// Entries in the cache directory.
pub cache: Vec<std::path::PathBuf>,
}
impl WipePlan {
/// Whether there is anything to remove at all.
pub fn is_empty(&self) -> bool {
self.state.is_empty() && self.cache.is_empty()
}
/// Every entry, state first.
pub fn entries(&self) -> impl Iterator<Item = &std::path::Path> {
self.state
.iter()
.chain(self.cache.iter())
.map(std::path::PathBuf::as_path)
}
}
/// Lists what [`wipe`] would remove, without removing anything.
pub fn wipe_plan(paths: &StoragePaths) -> Result<WipePlan> {
Ok(WipePlan {
state: removable(&paths.state_dir, &paths.state_db())?,
cache: removable(&paths.cache_dir, &paths.cache_db())?,
})
}
/// Removes everything in both directories, leaving this device a stranger.
///
/// The device identity, every network it belongs to, every signed record and
/// everything a plugin kept beside them. The next start generates a new
/// identity and knows nothing, which is the point: after this there is no
/// membership left to be surprised by.
///
/// What it is **not** is a goodbye. Other members keep the claims this device
/// signed, because signed state has no expiry and there is nobody left here
/// to sign a release. Leaving each network first with
/// [`crate::Agent::leave_network`] is what frees the address and the name.
///
/// Refuses while an agent owns the state directory, and refuses a directory
/// with no `state.sqlite` in it — a wipe pointed at the wrong place by a
/// mistyped `--state-dir` would otherwise remove somebody's documents.
pub fn wipe(paths: &StoragePaths) -> Result<WipePlan> {
let plan = wipe_plan(paths)?;
// Held across the removal, so an agent cannot start into a directory
// that is half gone. The lock file is itself removed at the end: on
// Unix the lock lives on the open handle, not on the name.
let lock = DirectoryLock::acquire(paths.lock_file())?;
for entry in plan.entries() {
remove_entry(entry)?;
}
drop(lock);
// Created by acquiring the lock a moment ago, so it is ours to remove
// and nothing is left behind claiming the directory.
let _ = std::fs::remove_file(paths.lock_file());
Ok(plan)
}
/// Every entry of `dir`, once `marker` proves it is one of ours.
///
/// An empty or missing directory is nothing to remove rather than an error:
/// wiping twice must be as ordinary as wiping once.
fn removable(dir: &Path, marker: &Path) -> Result<Vec<std::path::PathBuf>> {
if !dir.exists() {
return Ok(Vec::new());
}
let mut entries = Vec::new();
let read = std::fs::read_dir(dir).map_err(|source| Error::Io {
path: dir.to_path_buf(),
source,
})?;
for entry in read {
let entry = entry.map_err(|source| Error::Io {
path: dir.to_path_buf(),
source,
})?;
entries.push(entry.path());
}
if entries.is_empty() {
return Ok(entries);
}
// The marker is what tells a state directory from a directory that
// merely got named as one. The lock file counts too: an agent that
// crashed before writing anything still leaves that.
let ours = marker.exists() || dir.join("state.lock").exists();
if !ours {
return Err(Error::Storage(format!(
"{} does not look like a tsunagi directory: no {} in it. Nothing was removed.",
dir.display(),
marker.file_name().map_or_else(
|| marker.display().to_string(),
|name| name.to_string_lossy().into_owned()
)
)));
}
entries.sort();
Ok(entries)
}
fn remove_entry(path: &Path) -> Result<()> {
let metadata = match std::fs::symlink_metadata(path) {
Ok(metadata) => metadata,
// Gone between the plan and the removal is the outcome asked for.
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(source) => {
return Err(Error::Io {
path: path.to_path_buf(),
source,
});
}
};
let removed = if metadata.is_dir() {
std::fs::remove_dir_all(path)
} else {
std::fs::remove_file(path)
};
removed.map_err(|source| Error::Io {
path: path.to_path_buf(),
source,
})
}
+954
View File
@@ -0,0 +1,954 @@
//! The mandatory state store, `state.sqlite`.
//!
//! Holds the persistent device identity, configured networks (including their
//! shared secrets, which are needed to re-derive keys after a restart), the
//! stored hostname and auto-start flags.
//!
//! Corruption is reported, never silently repaired: a damaged state store must
//! not quietly turn into a brand new identity.
//!
//! Future work will add per-author record versions, accepted signed states and
//! revocations here. When that lands, writing an event and bumping the author's
//! own counter must happen in one SQLite transaction *before* the change is
//! published to the network.
use std::path::{Path, PathBuf};
use rusqlite::{Connection, OptionalExtension, params};
use crate::error::{Error, Result};
use crate::identity::{DeviceIdentity, NetworkId, NetworkName, NetworkSecret};
use crate::state::{RecordBody, SignedRecord};
/// Schema version written by this build.
pub const SCHEMA_VERSION: i64 = 6;
/// Key of the stored hostname setting.
const SETTING_HOSTNAME: &str = "hostname";
/// A network as persisted in the state store.
///
/// The secret is held in a [`NetworkSecret`], which redacts itself from `Debug`
/// and zeroizes on drop.
#[derive(Debug, Clone)]
pub struct StoredNetwork {
/// Derived public network identifier.
pub network_id: NetworkId,
/// Network name.
pub name: NetworkName,
/// Shared secret, needed to re-derive keys after restart.
pub secret: NetworkSecret,
/// Whether the network is activated automatically at agent startup.
pub auto_start: bool,
/// Local broadcast participation, enabled unless explicitly disabled.
pub broadcast: bool,
/// This agent offers itself as an exit node here. Off unless turned on.
pub exit_node: bool,
/// The member this device sends all its internet traffic through, if any.
pub exit_via: Option<[u8; 32]>,
}
/// The mandatory state store.
#[derive(Debug)]
pub struct StateStore {
conn: Connection,
path: PathBuf,
}
impl StateStore {
/// Opens (creating if absent) the state store at `path`.
///
/// Returns [`Error::StateCorrupted`] if the file exists but is not a usable
/// database. The file is never deleted or recreated by this function.
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref().to_path_buf();
let existed = path.exists();
// The database file is created on demand, so the directory holding
// it has to be too — with the same restricted permissions the agent
// would have given it, never looser.
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
super::create_dir(parent)?;
}
let conn = Connection::open(&path).map_err(|err| Error::StateCorrupted {
path: path.clone(),
reason: format!("cannot open database: {err}"),
})?;
super::restrict_path_permissions(&path)?;
super::apply_common_pragmas(&conn).map_err(|err| Error::StateCorrupted {
path: path.clone(),
reason: format!("cannot configure database: {err}"),
})?;
if existed {
let integrity: String = conn
.query_row("PRAGMA integrity_check", [], |row| row.get(0))
.map_err(|err| Error::StateCorrupted {
path: path.clone(),
reason: format!("integrity check failed: {err}"),
})?;
if integrity != "ok" {
return Err(Error::StateCorrupted {
path,
reason: format!("integrity check reported: {integrity}"),
});
}
}
let store = Self { conn, path };
store.migrate()?;
Ok(store)
}
/// Path of the underlying file.
pub fn path(&self) -> &Path {
&self.path
}
fn corrupt(&self, reason: impl std::fmt::Display) -> Error {
Error::StateCorrupted {
path: self.path.clone(),
reason: reason.to_string(),
}
}
fn migrate(&self) -> Result<()> {
let found: i64 = self
.conn
.query_row("PRAGMA user_version", [], |row| row.get(0))
.map_err(|err| self.corrupt(format!("cannot read schema version: {err}")))?;
if found > SCHEMA_VERSION {
return Err(Error::UnsupportedSchema {
found,
supported: SCHEMA_VERSION,
});
}
if found == SCHEMA_VERSION {
return self.verify_shape();
}
// Migration 2 -> 3: the record body gained a hostname, which changed
// the signing domain, and the version counter gained an author.
//
// The stored records are discarded rather than carried over. They
// were signed under a domain that no longer verifies, so keeping them
// would mean holding rows that every read has to reject — and one of
// those rejections could be mistaken for corruption. Each member
// re-publishes its claim on the next run, which is the one thing here
// that repairs itself.
if (2..3).contains(&found) {
self.conn
.execute_batch(
"BEGIN;
DROP TABLE IF EXISTS signed_records;
DROP TABLE IF EXISTS own_record_version;
COMMIT;",
)
.map_err(|err| self.corrupt(format!("cannot migrate schema to 3: {err}")))?;
self.conn
.execute_batch(SIGNED_RECORDS_SCHEMA)
.map_err(|err| self.corrupt(format!("cannot migrate schema to 3: {err}")))?;
}
// Migration 1 -> 2: signed records that outlive a session.
if (1..2).contains(&found) {
self.conn
.execute_batch(SIGNED_RECORDS_SCHEMA)
.map_err(|err| self.corrupt(format!("cannot migrate schema to 2: {err}")))?;
}
// Migration 0 -> 1: initial schema.
if found < 1 {
self.conn
.execute_batch(
"BEGIN;
CREATE TABLE device_identity (
id INTEGER PRIMARY KEY CHECK (id = 1),
secret_key BLOB NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE networks (
network_id BLOB PRIMARY KEY,
name TEXT NOT NULL,
secret BLOB NOT NULL,
auto_start INTEGER NOT NULL DEFAULT 1,
created_at INTEGER NOT NULL
);
CREATE TABLE settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
PRAGMA user_version = 1;
COMMIT;",
)
.map_err(|err| self.corrupt(format!("cannot create schema: {err}")))?;
self.conn
.execute_batch(SIGNED_RECORDS_SCHEMA)
.map_err(|err| self.corrupt(format!("cannot create schema: {err}")))?;
}
if found < 4 {
self.conn.execute_batch("BEGIN;
ALTER TABLE networks ADD COLUMN broadcast INTEGER NOT NULL DEFAULT 1 CHECK (broadcast IN (0,1));
PRAGMA user_version = 4;
COMMIT;").map_err(|err| self.corrupt(format!("cannot migrate schema to 4: {err}")))?;
}
if found < 5 {
self.conn
.execute_batch(
"BEGIN;
CREATE TABLE IF NOT EXISTS peer_hostnames (
network_id BLOB NOT NULL,
endpoint_id BLOB NOT NULL,
hostname TEXT NOT NULL,
PRIMARY KEY (network_id, endpoint_id)
);
PRAGMA user_version = 5;
COMMIT;",
)
.map_err(|err| self.corrupt(format!("cannot migrate schema to 5: {err}")))?;
}
if found < 6 {
self.conn
.execute_batch(
"BEGIN;
ALTER TABLE networks ADD COLUMN exit_node INTEGER NOT NULL DEFAULT 0 CHECK (exit_node IN (0,1));
ALTER TABLE networks ADD COLUMN exit_via BLOB;
PRAGMA user_version = 6;
COMMIT;",
)
.map_err(|err| self.corrupt(format!("cannot migrate schema to 6: {err}")))?;
}
Ok(())
}
/// Confirms the expected tables exist, so that a truncated or foreign
/// database is reported rather than used.
fn verify_shape(&self) -> Result<()> {
for table in [
"device_identity",
"networks",
"settings",
"signed_records",
"peer_hostnames",
] {
let present: Option<String> = self
.conn
.query_row(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?1",
params![table],
|row| row.get(0),
)
.optional()
.map_err(|err| self.corrupt(format!("cannot inspect schema: {err}")))?;
if present.is_none() {
return Err(self.corrupt(format!("table `{table}` is missing")));
}
}
Ok(())
}
/// Loads the stored device identity, if there is one.
///
/// Reads and never writes, so asking who this device is does not decide
/// it. A store that has never run an agent has no identity yet, which is
/// `None` rather than an error.
///
/// A stored key of the wrong length is a corruption error, never a reason
/// to silently mint a new identity.
pub fn device_identity(&self) -> Result<Option<DeviceIdentity>> {
let stored: Option<Vec<u8>> = self
.conn
.query_row(
"SELECT secret_key FROM device_identity WHERE id = 1",
[],
|row| row.get(0),
)
.optional()
.map_err(|err| self.corrupt(format!("cannot read device identity: {err}")))?;
let Some(bytes) = stored else {
return Ok(None);
};
let bytes: [u8; 32] = bytes.as_slice().try_into().map_err(|_| {
self.corrupt(format!(
"stored device key has {} bytes, expected 32; refusing to replace it",
bytes.len()
))
})?;
Ok(Some(DeviceIdentity::from_secret_bytes(&bytes)))
}
/// Loads the stored device identity, creating one on first use.
pub fn load_or_create_device_identity(&self) -> Result<DeviceIdentity> {
if let Some(identity) = self.device_identity()? {
return Ok(identity);
}
let identity = DeviceIdentity::generate();
self.conn
.execute(
"INSERT INTO device_identity (id, secret_key, created_at) VALUES (1, ?1, ?2)",
params![identity.secret_bytes().as_slice(), now_unix()],
)
.map_err(|err| self.corrupt(format!("cannot store device identity: {err}")))?;
Ok(identity)
}
/// Replaces the device identity, giving up what the outgoing key held.
///
/// A device key is the author of every record this agent has signed, so
/// replacing it makes this a different member. The addresses and names
/// the old key claimed would otherwise stay reserved to a key nobody
/// holds, and nothing could ever free them — there is no way to sign on
/// another author's behalf, and by design there is no authority that
/// could overrule one.
///
/// So the outgoing key signs a release for every network on its way out.
/// That is the revocation: a positive statement, merged like any other,
/// which frees the address and the name for whoever wants them next.
///
/// All of it commits together. A crash part way through must not leave an
/// identity that has already been replaced beside releases that were
/// never written, because the old key would then be gone and unable to
/// sign them.
///
/// Returns the new identity and the networks a release was signed for.
pub fn rotate_device_identity(&self) -> Result<(DeviceIdentity, Vec<NetworkId>)> {
let outgoing = self.device_identity()?;
let networks = self.list_networks()?;
let replacement = DeviceIdentity::generate();
let transaction = self
.conn
.unchecked_transaction()
.map_err(|err| Error::Storage(format!("cannot begin a transaction: {err}")))?;
let mut released = Vec::new();
if let Some(outgoing) = &outgoing {
let author = outgoing.endpoint_id();
let signing = outgoing.signing_key();
for network in &networks {
let previous: Option<i64> = transaction
.query_row(
"SELECT version FROM own_record_version
WHERE network_id = ?1 AND author = ?2",
params![
network.network_id.as_bytes().as_slice(),
author.as_bytes().as_slice()
],
|row| row.get(0),
)
.optional()
.map_err(|err| Error::Storage(format!("cannot read our version: {err}")))?;
// A key that never published anything has nothing to give up.
let Some(previous) = previous else { continue };
let version = (previous.max(0) as u64).saturating_add(1);
let record =
SignedRecord::sign(&signing, network.network_id, version, RecordBody::Release);
let body = postcard::to_stdvec(&record.body)
.map_err(|err| Error::Storage(format!("cannot encode a record body: {err}")))?;
transaction
.execute(
"INSERT INTO signed_records (network_id, author, version, body, signature)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(network_id, author) DO UPDATE SET
version = excluded.version,
body = excluded.body,
signature = excluded.signature",
params![
record.network.as_slice(),
record.author.as_slice(),
record.version as i64,
body,
record.signature
],
)
.map_err(|err| Error::Storage(format!("cannot store a release: {err}")))?;
transaction
.execute(
"INSERT INTO own_record_version (network_id, author, version)
VALUES (?1, ?2, ?3)
ON CONFLICT(network_id, author) DO UPDATE SET
version = max(version, excluded.version)",
params![
record.network.as_slice(),
record.author.as_slice(),
record.version as i64
],
)
.map_err(|err| Error::Storage(format!("cannot store our version: {err}")))?;
released.push(network.network_id);
}
}
transaction
.execute(
"INSERT INTO device_identity (id, secret_key, created_at) VALUES (1, ?1, ?2)
ON CONFLICT(id) DO UPDATE SET
secret_key = excluded.secret_key,
created_at = excluded.created_at",
params![replacement.secret_bytes().as_slice(), now_unix()],
)
.map_err(|err| Error::Storage(format!("cannot store the new identity: {err}")))?;
transaction
.commit()
.map_err(|err| Error::Storage(format!("cannot commit the new identity: {err}")))?;
Ok((replacement, released))
}
/// Inserts or updates a network configuration.
pub fn upsert_network(
&self,
network_id: NetworkId,
name: &NetworkName,
secret: &NetworkSecret,
auto_start: bool,
) -> Result<()> {
self.conn
.execute(
"INSERT INTO networks (network_id, name, secret, auto_start, created_at)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(network_id) DO UPDATE SET
name = excluded.name,
secret = excluded.secret,
auto_start = excluded.auto_start",
params![
network_id.as_bytes().as_slice(),
name.as_str(),
secret.expose(),
auto_start as i64,
now_unix()
],
)
.map_err(|err| Error::Storage(format!("cannot store network: {err}")))?;
Ok(())
}
/// Sets the auto-start flag of a configured network.
pub fn set_auto_start(&self, network_id: NetworkId, auto_start: bool) -> Result<()> {
self.conn
.execute(
"UPDATE networks SET auto_start = ?2 WHERE network_id = ?1",
params![network_id.as_bytes().as_slice(), auto_start as i64],
)
.map_err(|err| Error::Storage(format!("cannot update network: {err}")))?;
Ok(())
}
/// Saves local participation without changing identity, membership or auto-start.
pub fn set_broadcast(&self, network_id: NetworkId, enabled: bool) -> Result<()> {
let changed = self
.conn
.execute(
"UPDATE networks SET broadcast = ?2 WHERE network_id = ?1",
params![network_id.as_bytes().as_slice(), enabled as i64],
)
.map_err(|err| Error::Storage(format!("cannot update broadcast policy: {err}")))?;
if changed == 0 {
return Err(Error::NetworkUnknown(network_id));
}
Ok(())
}
/// Removes a network configuration entirely.
pub fn remove_network(&self, network_id: NetworkId) -> Result<()> {
self.conn
.execute(
"DELETE FROM networks WHERE network_id = ?1",
params![network_id.as_bytes().as_slice()],
)
.map_err(|err| Error::Storage(format!("cannot remove network: {err}")))?;
Ok(())
}
/// Saves whether this agent offers itself as an exit node in a network.
pub fn set_exit_node(&self, network_id: NetworkId, enabled: bool) -> Result<()> {
let changed = self
.conn
.execute(
"UPDATE networks SET exit_node = ?2 WHERE network_id = ?1",
params![network_id.as_bytes().as_slice(), enabled as i64],
)
.map_err(|err| Error::Storage(format!("cannot update exit node offer: {err}")))?;
if changed == 0 {
return Err(Error::NetworkUnknown(network_id));
}
Ok(())
}
/// Saves which member this device sends its internet traffic through.
///
/// One device has one default route, so choosing an exit node in one
/// network clears the choice in every other.
pub fn set_exit_via(&self, network_id: NetworkId, via: Option<&[u8; 32]>) -> Result<()> {
let tx = self
.conn
.unchecked_transaction()
.map_err(|err| Error::Storage(format!("cannot update exit node: {err}")))?;
let changed = tx
.execute(
"UPDATE networks SET exit_via = ?2 WHERE network_id = ?1",
params![network_id.as_bytes().as_slice(), via.map(|v| v.as_slice())],
)
.map_err(|err| Error::Storage(format!("cannot update exit node: {err}")))?;
if changed == 0 {
return Err(Error::NetworkUnknown(network_id));
}
if via.is_some() {
tx.execute(
"UPDATE networks SET exit_via = NULL WHERE network_id != ?1",
params![network_id.as_bytes().as_slice()],
)
.map_err(|err| Error::Storage(format!("cannot update exit node: {err}")))?;
}
tx.commit()
.map_err(|err| Error::Storage(format!("cannot update exit node: {err}")))?;
Ok(())
}
/// Lists every configured network.
pub fn list_networks(&self) -> Result<Vec<StoredNetwork>> {
let mut stmt = self
.conn
.prepare("SELECT network_id, name, secret, auto_start, broadcast, exit_node, exit_via FROM networks ORDER BY name")
.map_err(|err| Error::Storage(format!("cannot list networks: {err}")))?;
let rows = stmt
.query_map([], |row| {
let id: Vec<u8> = row.get(0)?;
let name: String = row.get(1)?;
let secret: Vec<u8> = row.get(2)?;
let auto_start: i64 = row.get(3)?;
Ok((
id,
name,
secret,
auto_start != 0,
row.get::<_, bool>(4)?,
row.get::<_, bool>(5)?,
row.get::<_, Option<Vec<u8>>>(6)?,
))
})
.map_err(|err| Error::Storage(format!("cannot list networks: {err}")))?;
let mut out = Vec::new();
for row in rows {
let (id, name, secret, auto_start, broadcast, exit_node, exit_via) =
row.map_err(|err| Error::Storage(format!("cannot read network row: {err}")))?;
let id: [u8; 32] = id
.as_slice()
.try_into()
.map_err(|_| self.corrupt("stored network id is not 32 bytes"))?;
out.push(StoredNetwork {
network_id: NetworkId::from_bytes(id),
name: NetworkName::new(name)?,
secret: NetworkSecret::from_bytes(secret)?,
auto_start,
broadcast,
exit_node,
exit_via: exit_via.and_then(|bytes| <[u8; 32]>::try_from(bytes.as_slice()).ok()),
});
}
Ok(out)
}
/// Reads the stored hostname, if any.
pub fn hostname(&self) -> Result<Option<String>> {
self.get_setting(SETTING_HOSTNAME)
}
/// Stores the hostname.
///
/// Today this is a local setting. In the future a rename must be a signed
/// record that revokes the specific old binding and announces the new one,
/// ideally atomically in one record.
pub fn set_hostname(&self, hostname: &str) -> Result<()> {
self.set_setting(SETTING_HOSTNAME, hostname)
}
/// Loads every signed record known for a network.
///
/// Records are returned as stored; the caller verifies them, because the
/// database is not a trust boundary — a restored backup or a copied file
/// could contain anything.
pub fn signed_records(&self, network_id: NetworkId) -> Result<Vec<SignedRecord>> {
let mut stmt = self
.conn
.prepare(
"SELECT author, version, body, signature FROM signed_records
WHERE network_id = ?1",
)
.map_err(|err| Error::Storage(format!("cannot read signed records: {err}")))?;
let rows = stmt
.query_map(params![network_id.as_bytes().as_slice()], |row| {
let author: Vec<u8> = row.get(0)?;
let version: i64 = row.get(1)?;
let body: Vec<u8> = row.get(2)?;
let signature: Vec<u8> = row.get(3)?;
Ok((author, version, body, signature))
})
.map_err(|err| Error::Storage(format!("cannot read signed records: {err}")))?;
let mut out = Vec::new();
for row in rows {
let (author, version, body, signature) =
row.map_err(|err| Error::Storage(format!("cannot read a record row: {err}")))?;
let Ok(author) = <[u8; 32]>::try_from(author.as_slice()) else {
continue;
};
let Ok(body) = postcard::from_bytes(&body) else {
continue;
};
out.push(SignedRecord {
author,
network: *network_id.as_bytes(),
version: version as u64,
body,
signature,
});
}
Ok(out)
}
/// Stores a record received from somebody else.
pub fn put_signed_record(&self, record: &SignedRecord) -> Result<()> {
let body = postcard::to_stdvec(&record.body)
.map_err(|err| Error::Storage(format!("cannot encode a record body: {err}")))?;
self.conn
.execute(
"INSERT INTO signed_records (network_id, author, version, body, signature)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(network_id, author) DO UPDATE SET
version = excluded.version,
body = excluded.body,
signature = excluded.signature",
params![
record.network.as_slice(),
record.author.as_slice(),
record.version as i64,
body,
record.signature
],
)
.map_err(|err| Error::Storage(format!("cannot store a signed record: {err}")))?;
Ok(())
}
/// Stores one of **our own** records and bumps our counter, atomically.
///
/// The model requires that a record and the author's own version counter
/// are committed together, and **before** the record is published, so a
/// crash can never leave us able to reuse a version number we already put
/// on the wire.
pub fn publish_own_record(&self, record: &SignedRecord) -> Result<()> {
let body = postcard::to_stdvec(&record.body)
.map_err(|err| Error::Storage(format!("cannot encode a record body: {err}")))?;
let transaction = self
.conn
.unchecked_transaction()
.map_err(|err| Error::Storage(format!("cannot begin a transaction: {err}")))?;
transaction
.execute(
"INSERT INTO signed_records (network_id, author, version, body, signature)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(network_id, author) DO UPDATE SET
version = excluded.version,
body = excluded.body,
signature = excluded.signature",
params![
record.network.as_slice(),
record.author.as_slice(),
record.version as i64,
body,
record.signature
],
)
.map_err(|err| Error::Storage(format!("cannot store our record: {err}")))?;
transaction
.execute(
"INSERT INTO own_record_version (network_id, author, version)
VALUES (?1, ?2, ?3)
ON CONFLICT(network_id, author) DO UPDATE SET
version = max(version, excluded.version)",
params![
record.network.as_slice(),
record.author.as_slice(),
record.version as i64
],
)
.map_err(|err| Error::Storage(format!("cannot store our version: {err}")))?;
transaction
.commit()
.map_err(|err| Error::Storage(format!("cannot commit our record: {err}")))
}
/// The highest version an author has ever published for a network.
///
/// Monotonic even if the record is later replaced by a conflicting one,
/// so a number is never reused. Keyed by author as well: a replaced
/// device key is a different author and starts its own sequence, while
/// the outgoing one keeps its place so the release it signs on the way
/// out cannot collide with something it already published.
pub fn own_record_version(
&self,
network_id: NetworkId,
author: iroh::EndpointId,
) -> Result<u64> {
let version: Option<i64> = self
.conn
.query_row(
"SELECT version FROM own_record_version WHERE network_id = ?1 AND author = ?2",
params![
network_id.as_bytes().as_slice(),
author.as_bytes().as_slice()
],
|row| row.get(0),
)
.optional()
.map_err(|err| Error::Storage(format!("cannot read our version: {err}")))?;
Ok(version.unwrap_or(0).max(0) as u64)
}
/// Forgets every record of a network.
pub fn forget_signed_records(&self, network_id: NetworkId) -> Result<()> {
self.conn
.execute(
"DELETE FROM signed_records WHERE network_id = ?1",
params![network_id.as_bytes().as_slice()],
)
.map_err(|err| Error::Storage(format!("cannot clear signed records: {err}")))?;
Ok(())
}
/// Remembers the hostname a member last announced.
pub fn remember_peer_hostname(
&self,
network_id: NetworkId,
endpoint_id: &[u8; 32],
hostname: &str,
) -> Result<()> {
self.conn
.execute(
"INSERT INTO peer_hostnames (network_id, endpoint_id, hostname)
VALUES (?1, ?2, ?3)
ON CONFLICT(network_id, endpoint_id) DO UPDATE SET hostname = excluded.hostname",
params![
network_id.as_bytes().as_slice(),
endpoint_id.as_slice(),
hostname
],
)
.map_err(|err| Error::Storage(format!("cannot remember a hostname: {err}")))?;
Ok(())
}
/// The last hostname every member of a network announced.
pub fn peer_hostnames(&self, network_id: NetworkId) -> Result<Vec<([u8; 32], String)>> {
let mut stmt = self
.conn
.prepare("SELECT endpoint_id, hostname FROM peer_hostnames WHERE network_id = ?1")
.map_err(|err| Error::Storage(format!("cannot read hostnames: {err}")))?;
let rows = stmt
.query_map(params![network_id.as_bytes().as_slice()], |row| {
Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, String>(1)?))
})
.map_err(|err| Error::Storage(format!("cannot read hostnames: {err}")))?;
let mut out = Vec::new();
for row in rows {
let (id, hostname) =
row.map_err(|err| Error::Storage(format!("cannot read hostnames: {err}")))?;
if let Ok(id) = <[u8; 32]>::try_from(id.as_slice()) {
out.push((id, hostname));
}
}
Ok(out)
}
/// Forgets every remembered hostname of a network.
pub fn forget_peer_hostnames(&self, network_id: NetworkId) -> Result<()> {
self.conn
.execute(
"DELETE FROM peer_hostnames WHERE network_id = ?1",
params![network_id.as_bytes().as_slice()],
)
.map_err(|err| Error::Storage(format!("cannot forget hostnames: {err}")))?;
Ok(())
}
/// Reads an arbitrary setting.
pub fn get_setting(&self, key: &str) -> Result<Option<String>> {
self.conn
.query_row(
"SELECT value FROM settings WHERE key = ?1",
params![key],
|row| row.get(0),
)
.optional()
.map_err(|err| Error::Storage(format!("cannot read setting `{key}`: {err}")))
}
/// Writes an arbitrary setting.
pub fn set_setting(&self, key: &str, value: &str) -> Result<()> {
self.conn
.execute(
"INSERT INTO settings (key, value) VALUES (?1, ?2)
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
params![key, value],
)
.map_err(|err| Error::Storage(format!("cannot write setting `{key}`: {err}")))?;
Ok(())
}
}
/// Schema for the signed records described in [`crate::state`].
///
/// The version counter is keyed by author as well as network. A device key
/// can be replaced, and the replacement is a different author: it must start
/// its own sequence rather than inherit one, and the outgoing author's last
/// version has to survive so its release record cannot collide with
/// something it already published.
const SIGNED_RECORDS_SCHEMA: &str = "BEGIN;
CREATE TABLE IF NOT EXISTS signed_records (
network_id BLOB NOT NULL,
author BLOB NOT NULL,
version INTEGER NOT NULL,
body BLOB NOT NULL,
signature BLOB NOT NULL,
PRIMARY KEY (network_id, author)
);
CREATE TABLE IF NOT EXISTS own_record_version (
network_id BLOB NOT NULL,
author BLOB NOT NULL,
version INTEGER NOT NULL,
PRIMARY KEY (network_id, author)
);
PRAGMA user_version = 3;
COMMIT;";
/// Seconds since the Unix epoch, saturating at 0 before it.
pub(crate) fn now_unix() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
#[cfg(test)]
mod broadcast_storage_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::identity::NetworkKeys;
#[test]
fn exit_node_choices_are_off_by_default_and_one_choice_clears_the_others() {
let dir = tempfile::tempdir().unwrap();
let store = StateStore::open(dir.path().join("state.sqlite")).unwrap();
let secret = NetworkSecret::from_bytes([5; 32]).unwrap();
let make = |name: &str| {
let name = NetworkName::new(name).unwrap();
let id = NetworkKeys::derive(&name, &secret).network_id();
store.upsert_network(id, &name, &secret, true).unwrap();
id
};
let (a, b) = (make("exit-a"), make("exit-b"));
let listed = || store.list_networks().unwrap();
assert!(
listed()
.iter()
.all(|n| !n.exit_node && n.exit_via.is_none())
);
store.set_exit_node(a, true).unwrap();
store.set_exit_via(a, Some(&[7; 32])).unwrap();
store.set_exit_via(b, Some(&[8; 32])).unwrap();
let by_id = |id: NetworkId| listed().into_iter().find(|n| n.network_id == id).unwrap();
assert!(by_id(a).exit_node, "offering is per network and kept");
assert_eq!(by_id(a).exit_via, None, "one default route: a was cleared");
assert_eq!(by_id(b).exit_via, Some([8; 32]));
store.set_exit_via(b, None).unwrap();
assert_eq!(by_id(b).exit_via, None);
}
#[test]
fn a_remembered_hostname_survives_a_restart_and_follows_the_latest_name() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("state.sqlite");
let name = NetworkName::new("hostnames").unwrap();
let secret = NetworkSecret::from_bytes([9; 32]).unwrap();
let id = NetworkKeys::derive(&name, &secret).network_id();
let peer = [3u8; 32];
let store = StateStore::open(&path).unwrap();
store.remember_peer_hostname(id, &peer, "laptop").unwrap();
store.remember_peer_hostname(id, &peer, "laptop-2").unwrap();
drop(store);
let store = StateStore::open(&path).unwrap();
assert_eq!(
store.peer_hostnames(id).unwrap(),
vec![(peer, "laptop-2".to_string())]
);
store.forget_peer_hostnames(id).unwrap();
assert!(store.peer_hostnames(id).unwrap().is_empty());
}
#[test]
fn v3_migration_preserves_identity_and_networks_and_defaults_broadcast_on() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("state.sqlite");
let name = NetworkName::new("migration-broadcast").unwrap();
let secret = NetworkSecret::from_bytes([7; 32]).unwrap();
let id = NetworkKeys::derive(&name, &secret).network_id();
let store = StateStore::open(&path).unwrap();
let identity = store
.load_or_create_device_identity()
.unwrap()
.endpoint_id();
store.upsert_network(id, &name, &secret, false).unwrap();
store.set_hostname("old-host").unwrap();
store
.conn
.execute_batch(
"ALTER TABLE networks DROP COLUMN broadcast; \
ALTER TABLE networks DROP COLUMN exit_node; \
ALTER TABLE networks DROP COLUMN exit_via; \
PRAGMA user_version=3;",
)
.unwrap();
drop(store);
let store = StateStore::open(&path).unwrap();
let networks = store.list_networks().unwrap();
assert_eq!(networks.len(), 1);
assert_eq!(networks[0].network_id, id);
assert!(!networks[0].auto_start);
assert!(networks[0].broadcast);
assert_eq!(store.hostname().unwrap().as_deref(), Some("old-host"));
assert_eq!(
store
.load_or_create_device_identity()
.unwrap()
.endpoint_id(),
identity
);
store.set_broadcast(id, false).unwrap();
store.upsert_network(id, &name, &secret, true).unwrap();
drop(store);
let store = StateStore::open(&path).unwrap();
assert!(
!store.list_networks().unwrap()[0].broadcast,
"rejoin and restart preserve opt-out"
);
store.remove_network(id).unwrap();
store.upsert_network(id, &name, &secret, true).unwrap();
assert!(
store.list_networks().unwrap()[0].broadcast,
"forgotten network gets defaults"
);
}
}
+74
View File
@@ -0,0 +1,74 @@
//! Stopping the tasks an agent started.
//!
//! Shutdown is cooperative first and bounded always: every loop watches a
//! cancellation token, and anything still running when its grace period ends
//! is aborted. Nothing an agent owns may outlive the agent, and no peer may
//! hold shutdown open by refusing to read.
//!
//! Public because a protocol crate spawns tasks of its own and is held to
//! the same rule: the agent gives each plugin a grace period, but abandoning
//! the future it is awaiting does not stop the task behind it.
use std::time::Duration;
use tokio::task::JoinHandle;
/// How long a task gets to notice cancellation before it is cut off.
///
/// Winding down is cooperative first: a task in the middle of a write to a
/// peer gets a moment to finish it. After that it is aborted, because a peer
/// that stops reading must not be able to hold shutdown open.
pub const TASK_GRACE: Duration = Duration::from_secs(5);
/// Waits for `task` to finish, aborting it if it outlasts `grace`.
///
/// Returns whether it finished on its own. `what` names it in the warning,
/// which is the only signal that something was cut off rather than stopped.
pub async fn wind_down(task: JoinHandle<()>, grace: Duration, what: &str) -> bool {
let abort = task.abort_handle();
if tokio::time::timeout(grace, task).await.is_err() {
tracing::warn!(
task = what,
grace_secs = grace.as_secs_f32(),
"task did not wind down in time; aborting it"
);
abort.abort();
return false;
}
true
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[tokio::test]
async fn a_task_that_stops_on_its_own_is_not_aborted() {
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
let task = tokio::spawn(async move {
tx.send(()).await.ok();
});
assert!(wind_down(task, Duration::from_secs(5), "cooperative").await);
assert_eq!(rx.recv().await, Some(()), "it ran to completion");
}
#[tokio::test]
async fn a_task_that_ignores_the_grace_is_aborted() {
// A stand-in for a task blocked writing to a peer that stopped
// reading: it will never finish by itself.
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
let task = tokio::spawn(async move {
let _held = tx;
std::future::pending::<()>().await;
});
let finished = wind_down(task, Duration::from_millis(50), "wedged").await;
assert!(!finished, "it had to be cut off");
// Aborted for real: the task is gone, so what it held is dropped.
let closed = tokio::time::timeout(Duration::from_secs(5), rx.recv()).await;
assert_eq!(closed, Ok(None), "the aborted task dropped its sender");
}
}
+217
View File
@@ -0,0 +1,217 @@
//! A harness for testing against a real agent.
//!
//! Behind the `testing` feature, because it is only of use to a test and
//! pulls in `tempfile` and a tracing subscriber. It lives here rather than
//! being copied into each crate's `tests/` directory: a protocol plugin in
//! its own crate needs exactly the same harness, and two copies of it would
//! drift.
//!
//! Everything here builds agents that touch nothing outside a temporary
//! directory: loopback only, no relay, no address publication.
// A harness fails a test loudly and at once; that is the whole job. The
// crate forbids these elsewhere for the opposite reason — nothing off the
// network may panic — and this module never sees anything off the network.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::agent::Event;
use crate::config::{AgentConfig, StoragePaths, TransportPolicy};
use crate::discovery::SharedMemoryDiscovery;
use crate::identity::{NetworkName, NetworkSecret};
use crate::{Agent, Result};
use tempfile::TempDir;
use tokio::sync::broadcast::error::RecvError;
/// Installs a tracing subscriber when `TSUNAGI_TEST_LOG` is set.
///
/// The library never installs a global subscriber itself; tests opt in.
pub fn init_tracing() {
use std::sync::Once;
static ONCE: Once = Once::new();
ONCE.call_once(|| {
if let Ok(filter) = std::env::var("TSUNAGI_TEST_LOG") {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::new(filter))
.with_writer(std::io::stderr)
.try_init();
}
});
}
/// Overall deadline for anything a test waits on.
pub const DEADLINE: Duration = Duration::from_secs(30);
/// How often condition polling re-checks. Never used as the primary sync.
const POLL_INTERVAL: Duration = Duration::from_millis(25);
/// Builds a fully local configuration rooted at `dir`.
///
/// Loopback binding with a dynamic port, no relay, no address lookup and no
/// port mapping, so the suite needs neither the internet nor privileges.
/// Timeouts are shortened so that failure paths finish quickly.
pub fn local_config(dir: &Path) -> AgentConfig {
let limits = crate::Limits {
dial_timeout: Duration::from_millis(1500),
handshake_timeout: Duration::from_secs(5),
..Default::default()
};
AgentConfig::new(StoragePaths::under(dir))
.with_transport(TransportPolicy::LocalOnly)
.with_loopback_bind()
.with_discovery_interval(Duration::from_millis(150))
.with_limits(limits)
}
/// Builds a local configuration wired to a shared in-memory discovery table.
pub fn config_with(dir: &Path, discovery: &SharedMemoryDiscovery) -> AgentConfig {
local_config(dir).with_discovery(Arc::new(discovery.clone()))
}
/// A temporary directory plus the agent running on it.
pub struct TestAgent {
/// The directory it keeps its state in, removed when this is dropped.
pub dir: TempDir,
/// The agent itself.
pub agent: Agent,
}
impl TestAgent {
/// Starts an agent on a fresh temporary directory.
pub async fn spawn(discovery: &SharedMemoryDiscovery) -> Result<Self> {
init_tracing();
let dir = TempDir::new().expect("temp dir");
let agent = Agent::spawn(config_with(dir.path(), discovery)).await?;
Ok(Self { dir, agent })
}
/// Starts an agent with a caller-supplied configuration on a fresh dir.
pub async fn spawn_with(
build: impl FnOnce(AgentConfig) -> AgentConfig,
discovery: &SharedMemoryDiscovery,
) -> Result<Self> {
init_tracing();
let dir = TempDir::new().expect("temp dir");
let agent = Agent::spawn(build(config_with(dir.path(), discovery))).await?;
Ok(Self { dir, agent })
}
/// Stops the agent and returns the directory so it can be reopened.
pub async fn stop(self) -> TempDir {
self.agent.shutdown().await;
self.dir
}
}
/// A network name and a fresh high-entropy secret.
pub fn network(name: &str) -> (NetworkName, NetworkSecret) {
(
NetworkName::new(name).expect("valid network name"),
NetworkSecret::generate(),
)
}
/// Waits for an event matching `predicate`, under the global deadline.
pub async fn wait_event<T>(
rx: &mut tokio::sync::broadcast::Receiver<Event>,
predicate: impl Fn(&Event) -> Option<T>,
) -> T {
let deadline = Instant::now() + DEADLINE;
let mut seen: Vec<String> = Vec::new();
loop {
let remaining = deadline
.checked_duration_since(Instant::now())
.unwrap_or_default();
assert!(!remaining.is_zero(), "timed out waiting for an event");
match tokio::time::timeout(remaining, rx.recv()).await {
Ok(Ok(event)) => {
if let Some(value) = predicate(&event) {
return value;
}
if seen.len() < 12 {
let label = format!("{event:?}");
seen.push(label.chars().take(70).collect());
}
}
Ok(Err(RecvError::Lagged(skipped))) => {
panic!("event subscriber lagged, missed {skipped} events; first seen: {seen:#?}");
}
Ok(Err(RecvError::Closed)) => panic!("event channel closed while waiting"),
Err(_) => panic!("timed out waiting for an event"),
}
}
}
/// Polls an async condition until it returns `Some`, under the global deadline.
pub async fn wait_until<T, F, Fut>(what: &str, mut probe: F) -> T
where
F: FnMut() -> Fut,
Fut: Future<Output = Option<T>>,
{
let deadline = Instant::now() + DEADLINE;
loop {
let remaining = deadline
.checked_duration_since(Instant::now())
.unwrap_or_default();
assert!(
!remaining.is_zero(),
"timed out waiting for condition: {what}"
);
// The probe is bounded too. A deadline checked only between probes is
// no deadline at all: one call into a wedged agent that never answers
// would hang the test process for ever instead of failing it.
match tokio::time::timeout(remaining, probe()).await {
Ok(Some(value)) => return value,
Ok(None) => tokio::time::sleep(POLL_INTERVAL).await,
Err(_) => panic!("timed out waiting for condition: {what}"),
}
}
}
/// Lets a few discovery rounds pass.
///
/// Only ever used before asserting that something did **not** happen; waiting
/// for success always goes through [`wait_event`] or [`wait_until`].
pub async fn settle() {
tokio::time::sleep(Duration::from_millis(900)).await;
}
/// Waits until `agent` has `count` authenticated peers in `network`.
pub async fn wait_for_peers(
agent: &Agent,
network: crate::NetworkId,
count: usize,
) -> Vec<iroh::EndpointId> {
wait_until(&format!("{count} peers in {network}"), || async move {
let status = agent.network_status(network).await.ok()?;
if status.peers.len() >= count {
Some(status.connected_peers())
} else {
None
}
})
.await
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(start_paused = true)]
#[should_panic(expected = "timed out waiting for condition")]
async fn a_probe_that_never_answers_fails_the_deadline_instead_of_hanging() {
// Virtual time, so the deadline arrives at once rather than in thirty
// real seconds — and the deadline does arrive, which is the point: a
// probe that never returns used to hang the process instead.
wait_until::<(), _, _>("an answer that never comes", || async {
std::future::pending::<Option<()>>().await
})
.await;
}
}