UltradesuandClaude Sonnet 5.5
7c176e0b74
Choosing an exit node on Windows did nothing: the using side was refused, because Windows has no per-user routing and the agent's own traffic would loop into the tunnel it carries. Now the overlay interface gets 0.0.0.0/1 and 128.0.0.0/1 routes (and ::/1, 8000::/1 as an IPv6 blackhole), and every underlay address the agent talks to directly gets a /32 route through the physical default gateway, marked with route metric 7373 so stale ones are found and removed. Bypass routes are written before the halves and removed after them; a failed step removes what was written. All routes are in the active store. The addresses come from the runtime: each network runtime hands the interface the verified direct IPv4 paths of its control connections and the relay hosts in use (Interface::set_underlay, refreshed every dial round); relay names are resolved when the rules are applied. ExitHostRules::needs_bypass (true only on Windows) gates it, so Linux and macOS plans never carry or re-apply on path changes. ExitHostPlan gains `bypass` and `bypass_hosts`; networks leaving forget their entry. Limits: a path that appears after the last apply goes through the tunnel until the next apply; data-plane-only addresses are not collected separately (the data links ride the same iroh paths). Nothing was run on Windows. Checks: cargo fmt --check, cargo clippy --locked --workspace --all-targets -D warnings and cargo test --locked --workspace --all-targets passed on macOS; scripts and ordering are unit-tested with an injected runner and resolver, and Interface::set_underlay with the mock host. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2524 lines
95 KiB
Rust
2524 lines
95 KiB
Rust
//! The per-network runtime.
|
|
//!
|
|
//! One of these runs for every locally active network. It owns that network's
|
|
//! sessions, its dial loop and its counters. Everything it touches carries an
|
|
//! explicit [`NetworkId`], so deactivating or breaking one network cannot
|
|
//! disturb another and cannot stop the agent.
|
|
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use iroh::EndpointId;
|
|
use iroh::endpoint::{Connection, RecvStream, SendStream};
|
|
use tokio::sync::{broadcast, mpsc, oneshot};
|
|
use tokio::task::JoinHandle;
|
|
|
|
use crate::config::{DiscoveryPolicy, Limits, ReconnectPolicy};
|
|
use crate::dataplane::transport::{InboundLink, PacketLink, PacketTransport, SharedLink};
|
|
use crate::dataplane::{PluginCapability, SharedPlugin};
|
|
use crate::discovery::worker::DiscoveryWorker;
|
|
use crate::discovery::{Candidate, CandidateSource, NetworkDiscovery};
|
|
use crate::error::{Error, Result};
|
|
use crate::identity::{NetworkId, NetworkKeys};
|
|
use crate::net::{EndpointAdapter, PathAddr, snapshot_connection};
|
|
use crate::proto::handshake::{self, HandshakeOutcome, Role};
|
|
use crate::proto::message::{Announcement, ControlMessage, Envelope, encode, kind};
|
|
use crate::state::allocator::allocate;
|
|
use crate::state::{Ipv4Range, Merged, RecordBody, SignedRecord, StateSet};
|
|
use crate::storage::Storage;
|
|
|
|
use super::events::Event;
|
|
use super::session::{self, Session, SessionEvent};
|
|
use super::shutdown::Shutdown;
|
|
use super::status::{
|
|
CandidateStatus, MemberStatus, NetworkMetrics, NetworkState, NetworkStatus, PeerStatus,
|
|
};
|
|
|
|
/// An inbound connection that already passed the handshake.
|
|
#[derive(Debug)]
|
|
pub(crate) struct InboundSession {
|
|
pub(crate) conn: Connection,
|
|
pub(crate) send: SendStream,
|
|
pub(crate) recv: RecvStream,
|
|
pub(crate) outcome: HandshakeOutcome,
|
|
}
|
|
|
|
/// Commands accepted by a network runtime.
|
|
pub(crate) enum NetCommand {
|
|
Inbound(Box<InboundSession>),
|
|
Send {
|
|
peer: EndpointId,
|
|
message: ControlMessage,
|
|
reply: oneshot::Sender<Result<()>>,
|
|
},
|
|
Broadcast {
|
|
message: ControlMessage,
|
|
reply: oneshot::Sender<usize>,
|
|
},
|
|
/// A peer opened a data plane link towards us.
|
|
InboundLink(Box<InboundLink>),
|
|
Status {
|
|
reply: oneshot::Sender<Box<NetworkStatus>>,
|
|
},
|
|
Recheck,
|
|
SetBroadcast {
|
|
enabled: bool,
|
|
reply: oneshot::Sender<()>,
|
|
},
|
|
/// Offer this agent as an exit node in this network, or stop.
|
|
SetExitOffer {
|
|
enabled: bool,
|
|
reply: oneshot::Sender<()>,
|
|
},
|
|
/// Send this device's internet traffic through a member, or stop.
|
|
SetExitVia {
|
|
via: Option<EndpointId>,
|
|
reply: oneshot::Sender<std::result::Result<(), String>>,
|
|
},
|
|
/// Resend this agent's announcement to every peer of this network.
|
|
Reannounce,
|
|
/// Answer to a different name from now on.
|
|
SetHostname(String),
|
|
/// Give up everything this agent claimed here, and tell whoever is
|
|
/// listening.
|
|
///
|
|
/// The reply is the number of peers the tombstone was queued to. They
|
|
/// pass it on, so a member that is away learns of it from them rather
|
|
/// than from an agent that has already gone.
|
|
Release {
|
|
reply: oneshot::Sender<usize>,
|
|
},
|
|
/// An IP plugin reported an error from one of its own tasks.
|
|
PluginError {
|
|
/// Plugin protocol id.
|
|
protocol: String,
|
|
/// Human readable reason, free of secrets.
|
|
reason: String,
|
|
},
|
|
/// An IP plugin reported that data plane connectivity or a tunnel for a peer failed.
|
|
ProtocolFailure {
|
|
peer: EndpointId,
|
|
protocol: String,
|
|
reason: String,
|
|
},
|
|
}
|
|
|
|
impl std::fmt::Debug for NetCommand {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
match self {
|
|
NetCommand::Inbound(_) => f.write_str("Inbound"),
|
|
NetCommand::InboundLink(link) => write!(f, "InboundLink({})", link.protocol),
|
|
NetCommand::Send { peer, message, .. } => {
|
|
write!(f, "Send({}, {})", peer.fmt_short(), kind(message))
|
|
}
|
|
NetCommand::Broadcast { message, .. } => write!(f, "Broadcast({})", kind(message)),
|
|
NetCommand::Status { .. } => f.write_str("Status"),
|
|
NetCommand::Recheck => f.write_str("Recheck"),
|
|
NetCommand::SetBroadcast { enabled, .. } => write!(f, "SetBroadcast({enabled})"),
|
|
NetCommand::SetExitOffer { enabled, .. } => write!(f, "SetExitOffer({enabled})"),
|
|
NetCommand::SetExitVia { via, .. } => write!(f, "SetExitVia({})", via.is_some()),
|
|
NetCommand::Reannounce => f.write_str("Reannounce"),
|
|
NetCommand::SetHostname(_) => f.write_str("SetHostname"),
|
|
NetCommand::Release { .. } => f.write_str("Release"),
|
|
NetCommand::PluginError { protocol, .. } => write!(f, "PluginError({protocol})"),
|
|
NetCommand::ProtocolFailure { peer, protocol, .. } => {
|
|
write!(f, "ProtocolFailure({}, {protocol})", peer.fmt_short())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Handle to a running network runtime.
|
|
#[derive(Debug)]
|
|
pub(crate) struct NetworkHandle {
|
|
pub(crate) keys: NetworkKeys,
|
|
pub(crate) commands: mpsc::Sender<NetCommand>,
|
|
shutdown: Shutdown,
|
|
task: JoinHandle<()>,
|
|
}
|
|
|
|
impl NetworkHandle {
|
|
/// Stops the runtime and waits for its task to finish.
|
|
pub(crate) async fn stop(self) {
|
|
self.shutdown.trigger();
|
|
crate::task::wind_down(self.task, crate::task::TASK_GRACE, "network runtime").await;
|
|
}
|
|
}
|
|
|
|
/// Everything a network runtime needs to run.
|
|
pub(crate) struct RuntimeParams {
|
|
pub(crate) broadcast: bool,
|
|
/// This agent offers itself as an exit node here.
|
|
pub(crate) exit_offer: bool,
|
|
/// The member this device's internet traffic goes through.
|
|
pub(crate) exit_via: Option<EndpointId>,
|
|
pub(crate) keys: NetworkKeys,
|
|
pub(crate) adapter: EndpointAdapter,
|
|
pub(crate) storage: Storage,
|
|
pub(crate) events: broadcast::Sender<Event>,
|
|
pub(crate) limits: Arc<Limits>,
|
|
pub(crate) reconnect: ReconnectPolicy,
|
|
pub(crate) discovery: Option<Arc<dyn NetworkDiscovery>>,
|
|
pub(crate) discovery_interval: Duration,
|
|
pub(crate) discovery_policy: DiscoveryPolicy,
|
|
pub(crate) plugins: Vec<SharedPlugin>,
|
|
/// Who holds which overlay address, shared with every other network.
|
|
pub(crate) routes: Arc<crate::overlay::RoutingTable>,
|
|
/// The one interface, when the agent has one.
|
|
pub(crate) interface: Option<Arc<crate::overlay::Interface>>,
|
|
pub(crate) hostname: String,
|
|
/// Signing key for this agent's own records.
|
|
pub(crate) device_secret: iroh::SecretKey,
|
|
/// Peers with no direct data path, for tests only.
|
|
#[cfg(feature = "testing")]
|
|
pub(crate) unreachable_data_peers: Arc<std::sync::Mutex<std::collections::HashSet<EndpointId>>>,
|
|
/// The IPv4 overlay range this agent would use, if the network has not
|
|
/// already settled on another one.
|
|
pub(crate) ipv4_range: Option<Ipv4Range>,
|
|
/// The range it was configured with but cannot have, because another
|
|
/// network on this agent holds it.
|
|
pub(crate) range_conflict: Option<Ipv4Range>,
|
|
/// The range derived from this network's own id, proposed when the
|
|
/// configured one belongs to another network here.
|
|
///
|
|
/// Held back at first: a network that already exists has a range of its
|
|
/// own, and a member that proposed before hearing anything would be
|
|
/// arguing with it instead of adopting it. Every member derives the
|
|
/// same one, so once the wait is over there is nothing to argue about.
|
|
pub(crate) ipv4_fallback: Option<Ipv4Range>,
|
|
/// How data plane links are opened. `None` disables the data plane.
|
|
pub(crate) transport: Option<Arc<dyn PacketTransport>>,
|
|
}
|
|
|
|
/// Outcome of one attempt to open a data plane link.
|
|
struct LinkOutcome {
|
|
peer: EndpointId,
|
|
protocol: String,
|
|
result: Result<SharedLink, String>,
|
|
}
|
|
|
|
/// Outcome of one outbound dial.
|
|
enum DialOutcome {
|
|
Established(Box<InboundSession>),
|
|
Failed {
|
|
peer: EndpointId,
|
|
reason: String,
|
|
during_handshake: bool,
|
|
},
|
|
}
|
|
|
|
/// Backoff bookkeeping for one candidate.
|
|
#[derive(Debug)]
|
|
struct DialState {
|
|
consecutive_failures: u32,
|
|
next_attempt: Instant,
|
|
in_flight: bool,
|
|
source: CandidateSource,
|
|
}
|
|
|
|
impl DialState {
|
|
fn new(source: CandidateSource) -> Self {
|
|
Self {
|
|
consecutive_failures: 0,
|
|
next_attempt: Instant::now(),
|
|
in_flight: false,
|
|
source,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Starts a network runtime.
|
|
pub(crate) fn spawn(params: RuntimeParams) -> NetworkHandle {
|
|
let keys = params.keys.clone();
|
|
let shutdown = Shutdown::new();
|
|
let (commands_tx, commands_rx) = mpsc::channel(64);
|
|
|
|
let runtime_shutdown = shutdown.clone();
|
|
let task = tokio::spawn(async move {
|
|
let mut runtime = Runtime::new(params, runtime_shutdown);
|
|
runtime.run(commands_rx).await;
|
|
});
|
|
|
|
NetworkHandle {
|
|
keys,
|
|
commands: commands_tx,
|
|
shutdown,
|
|
task,
|
|
}
|
|
}
|
|
|
|
/// How long a peer's account of what it can reach is believed.
|
|
///
|
|
/// Soft state: it is repeated while it holds, and simply stops being
|
|
/// repeated when the peer goes. Nothing is revoked, so nothing can be
|
|
/// revoked wrongly — a relay that disappears stops being chosen by
|
|
/// itself, a few announcements later.
|
|
const REACH_EXPIRY: Duration = Duration::from_secs(90);
|
|
|
|
/// How long a network waits to be told its range before proposing the one
|
|
/// derived from its id.
|
|
///
|
|
/// Only for that fallback: the configured range is proposed at once, as it
|
|
/// always was. This is the window in which an existing network's records
|
|
/// can arrive and be adopted instead.
|
|
const RANGE_PROPOSAL_GRACE: Duration = Duration::from_secs(3);
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ProtocolDialState {
|
|
failures: u32,
|
|
cooldown_until: std::time::Instant,
|
|
}
|
|
|
|
const DATA_LINK_BASE_BACKOFF: Duration = Duration::from_secs(3);
|
|
const DATA_LINK_MAX_BACKOFF: Duration = Duration::from_secs(60);
|
|
|
|
fn data_link_backoff(failures: u32) -> Duration {
|
|
if failures == 0 {
|
|
return Duration::ZERO;
|
|
}
|
|
let shift = (failures - 1).min(5);
|
|
let secs = (DATA_LINK_BASE_BACKOFF.as_secs() << shift).min(DATA_LINK_MAX_BACKOFF.as_secs());
|
|
Duration::from_secs(secs)
|
|
}
|
|
|
|
struct Runtime {
|
|
discovery_worker: Option<DiscoveryWorker>,
|
|
discovery_candidates: mpsc::Receiver<Candidate>,
|
|
params: RuntimeParams,
|
|
/// When this runtime started, for the fallback range's grace period.
|
|
activated: std::time::Instant,
|
|
network_id: NetworkId,
|
|
local_id: EndpointId,
|
|
shutdown: Shutdown,
|
|
sessions: HashMap<EndpointId, Session>,
|
|
dial_states: HashMap<EndpointId, DialState>,
|
|
candidate_addrs: HashMap<EndpointId, iroh::EndpointAddr>,
|
|
metrics: NetworkMetrics,
|
|
session_events_tx: mpsc::Sender<SessionEvent>,
|
|
session_events_rx: mpsc::Receiver<SessionEvent>,
|
|
dial_results_tx: mpsc::Sender<DialOutcome>,
|
|
dial_results_rx: mpsc::Receiver<DialOutcome>,
|
|
/// Live data plane links, keyed by peer and plugin protocol.
|
|
///
|
|
/// The direct ones. What a protocol holds is the hub's link for that
|
|
/// peer, which outlives any particular path.
|
|
links: HashMap<(EndpointId, String), SharedLink>,
|
|
/// Every data link of this network, and what is reached through what.
|
|
hub: Arc<crate::dataplane::relay::RelayHub>,
|
|
/// Which peers each peer says it has a live link with, and when it
|
|
/// last said so.
|
|
///
|
|
/// First-hand only: a peer speaks for itself and never for anybody
|
|
/// else, so there is no hearsay to weigh and nothing to poison. Soft
|
|
/// state — an entry that stops being refreshed expires, which is how a
|
|
/// relay that has gone stops being chosen without anybody revoking it.
|
|
reachable: HashMap<
|
|
EndpointId,
|
|
(
|
|
HashSet<crate::proto::message::ReachableLink>,
|
|
std::time::Instant,
|
|
),
|
|
>,
|
|
/// What this agent last told its peers it could reach, so it is only
|
|
/// said again when it changed.
|
|
announced_reach: HashSet<crate::proto::message::ReachableLink>,
|
|
/// Links currently being opened, so we do not start two.
|
|
opening: HashSet<(EndpointId, String)>,
|
|
link_results_tx: mpsc::Sender<LinkOutcome>,
|
|
link_results_rx: mpsc::Receiver<LinkOutcome>,
|
|
/// Protocol dial failure count and backoff cooldown per peer and protocol.
|
|
protocol_dial_states: HashMap<(EndpointId, String), ProtocolDialState>,
|
|
/// Peers already told about a protocol version that cannot match, so it
|
|
/// is said once rather than on every announcement.
|
|
reported_mismatch: HashSet<(EndpointId, String)>,
|
|
/// Set once this agent has given everything up here.
|
|
///
|
|
/// A release is a statement on the way out, and anything that would
|
|
/// publish a claim afterwards — the periodic check, a peer's records
|
|
/// arriving — would silently take it back.
|
|
released: bool,
|
|
/// The address last reported as absent from the interface, so it is said
|
|
/// once rather than for ever.
|
|
reported_missing: Option<std::net::Ipv4Addr>,
|
|
/// Signed records, merged from every replica we have talked to.
|
|
state: StateSet,
|
|
/// Snapshots received while dispatching, handled on the next loop pass.
|
|
pending_state: Vec<(EndpointId, Vec<SignedRecord>)>,
|
|
/// The highest version this agent has ever published for this network.
|
|
own_version: u64,
|
|
/// Whether the exit node in use was reachable when last looked at, so
|
|
/// that losing it is said once and getting it back is too.
|
|
exit_via_was_online: bool,
|
|
/// What the last announcement said about being an exit node.
|
|
announced_exit: bool,
|
|
/// The last hostname each peer announced, kept across restarts so a
|
|
/// member that is away can still be called by name.
|
|
known_hostnames: HashMap<EndpointId, String>,
|
|
}
|
|
|
|
impl Runtime {
|
|
fn new(params: RuntimeParams, shutdown: Shutdown) -> Self {
|
|
let network_id = params.keys.network_id();
|
|
let local_id = params.adapter.endpoint_id();
|
|
let (session_events_tx, session_events_rx) = mpsc::channel(256);
|
|
let (dial_results_tx, dial_results_rx) = mpsc::channel(64);
|
|
let (link_results_tx, link_results_rx) = mpsc::channel(64);
|
|
let (candidate_tx, discovery_candidates) =
|
|
mpsc::channel(params.limits.max_discovery_candidates.max(1));
|
|
let discovery_worker = params.discovery.as_ref().map(|backend| {
|
|
DiscoveryWorker::spawn(
|
|
backend.clone(),
|
|
params.keys.discovery_key(),
|
|
params.adapter.clone(),
|
|
params.discovery_policy.clone(),
|
|
params.discovery_interval,
|
|
candidate_tx,
|
|
)
|
|
});
|
|
Self {
|
|
discovery_worker,
|
|
discovery_candidates,
|
|
params,
|
|
activated: std::time::Instant::now(),
|
|
network_id,
|
|
local_id,
|
|
shutdown,
|
|
sessions: HashMap::new(),
|
|
dial_states: HashMap::new(),
|
|
candidate_addrs: HashMap::new(),
|
|
metrics: NetworkMetrics::default(),
|
|
session_events_tx,
|
|
session_events_rx,
|
|
dial_results_tx,
|
|
dial_results_rx,
|
|
links: HashMap::new(),
|
|
hub: crate::dataplane::relay::RelayHub::new(network_id, local_id),
|
|
reachable: HashMap::new(),
|
|
announced_reach: HashSet::new(),
|
|
opening: HashSet::new(),
|
|
link_results_tx,
|
|
link_results_rx,
|
|
protocol_dial_states: HashMap::new(),
|
|
reported_mismatch: HashSet::new(),
|
|
released: false,
|
|
reported_missing: None,
|
|
state: StateSet::new(),
|
|
pending_state: Vec::new(),
|
|
own_version: 0,
|
|
exit_via_was_online: false,
|
|
announced_exit: false,
|
|
known_hostnames: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
fn emit(&self, event: Event) {
|
|
// A broadcast with no subscribers is not an error.
|
|
let _ = self.params.events.send(event);
|
|
}
|
|
|
|
async fn run(&mut self, mut commands: mpsc::Receiver<NetCommand>) {
|
|
// Everything this agent knew before it restarted, including the
|
|
// address it holds.
|
|
self.load_state().await;
|
|
|
|
let mut ticker = tokio::time::interval(self.params.discovery_interval);
|
|
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
|
|
|
loop {
|
|
tokio::select! {
|
|
biased;
|
|
_ = self.shutdown.wait() => break,
|
|
_ = self.hub.changed() => {
|
|
self.ensure_links();
|
|
self.update_paths();
|
|
self.announce_reach(false);
|
|
}
|
|
command = commands.recv() => match command {
|
|
Some(command) => self.handle_command(command).await,
|
|
None => break,
|
|
},
|
|
event = self.session_events_rx.recv() => {
|
|
if let Some(event) = event {
|
|
self.handle_session_event(event).await;
|
|
self.drain_pending_state().await;
|
|
}
|
|
}
|
|
result = self.dial_results_rx.recv() => {
|
|
if let Some(result) = result {
|
|
self.handle_dial_result(result).await;
|
|
}
|
|
}
|
|
result = self.link_results_rx.recv() => {
|
|
if let Some(result) = result {
|
|
self.handle_link_result(result);
|
|
}
|
|
}
|
|
Some(candidate) = self.discovery_candidates.recv() => {
|
|
self.add_candidate(candidate);
|
|
self.start_dials();
|
|
}
|
|
_ = ticker.tick() => {
|
|
self.discovery_round().await;
|
|
self.ensure_links();
|
|
self.update_paths();
|
|
self.announce_reach(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
self.teardown().await;
|
|
}
|
|
|
|
async fn teardown(&mut self) {
|
|
if let Some(worker) = self.discovery_worker.take() {
|
|
worker.stop().await;
|
|
}
|
|
// Stop accepting session events first: nothing is going to act on them
|
|
// any more, and a sender blocked on a full queue would stall shutdown.
|
|
self.session_events_rx.close();
|
|
self.hub.close();
|
|
self.links.clear();
|
|
self.protocol_dial_states.clear();
|
|
let peers: Vec<EndpointId> = self.sessions.keys().copied().collect();
|
|
for peer in peers {
|
|
if let Some(session) = self.sessions.remove(&peer) {
|
|
session.stop().await;
|
|
}
|
|
}
|
|
self.emit(Event::NetworkDeactivated {
|
|
network: self.network_id,
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------- commands
|
|
|
|
async fn handle_command(&mut self, command: NetCommand) {
|
|
match command {
|
|
NetCommand::Inbound(inbound) => {
|
|
self.install_session(*inbound).await;
|
|
}
|
|
NetCommand::InboundLink(inbound) => self.install_link(*inbound),
|
|
NetCommand::Send {
|
|
peer,
|
|
message,
|
|
reply,
|
|
} => {
|
|
let _ = reply.send(self.send_to(peer, message));
|
|
}
|
|
NetCommand::Broadcast { message, reply } => {
|
|
let peers: Vec<EndpointId> = self.sessions.keys().copied().collect();
|
|
let mut delivered = 0;
|
|
for peer in peers {
|
|
if self.send_to(peer, message.clone()).is_ok() {
|
|
delivered += 1;
|
|
}
|
|
}
|
|
let _ = reply.send(delivered);
|
|
}
|
|
NetCommand::Status { reply } => {
|
|
let _ = reply.send(Box::new(self.status()));
|
|
}
|
|
NetCommand::SetBroadcast { enabled, reply } => {
|
|
self.params.broadcast = enabled;
|
|
self.update_broadcast();
|
|
if let Some(interface) = self.params.interface.clone() {
|
|
interface.sync_broadcast_rules().await;
|
|
}
|
|
self.reannounce();
|
|
let _ = reply.send(());
|
|
}
|
|
NetCommand::SetExitOffer { enabled, reply } => {
|
|
self.params.exit_offer = enabled;
|
|
self.apply_exit().await;
|
|
self.reannounce();
|
|
let _ = reply.send(());
|
|
}
|
|
NetCommand::SetExitVia { via, reply } => {
|
|
let outcome = match via {
|
|
Some(peer) if !self.exit_node_online(peer) => Err(
|
|
"that member is not connected or does not offer to be an exit node"
|
|
.to_string(),
|
|
),
|
|
_ => Ok(()),
|
|
};
|
|
if outcome.is_ok() {
|
|
self.params.exit_via = via;
|
|
self.exit_via_was_online = via.is_some();
|
|
self.apply_exit().await;
|
|
}
|
|
let _ = reply.send(outcome);
|
|
}
|
|
NetCommand::Recheck => self.discovery_round().await,
|
|
NetCommand::Reannounce => self.reannounce(),
|
|
NetCommand::Release { reply } => {
|
|
// A tombstone, so the address and the name this agent held
|
|
// are freed for somebody else instead of staying reserved
|
|
// to a member that has gone.
|
|
self.publish_record(RecordBody::Release).await;
|
|
self.released = true;
|
|
let _ = reply.send(self.sessions.len());
|
|
}
|
|
NetCommand::SetHostname(hostname) => {
|
|
if self.params.hostname != hostname {
|
|
self.params.hostname = hostname;
|
|
// Publishing the claim is the revocation: one record per
|
|
// author, so the new version replaces the old name
|
|
// rather than sitting beside it.
|
|
self.ensure_own_claim().await;
|
|
self.broadcast_state();
|
|
}
|
|
// Told to peers regardless, so a session that missed the
|
|
// earlier announcement is not left with a stale name.
|
|
self.reannounce();
|
|
}
|
|
NetCommand::PluginError { protocol, reason } => {
|
|
// Counted here so that the per-network metric and the event
|
|
// always agree, wherever the error came from.
|
|
self.metrics.plugin_errors += 1;
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol,
|
|
reason,
|
|
});
|
|
}
|
|
NetCommand::ProtocolFailure {
|
|
peer,
|
|
protocol,
|
|
reason,
|
|
} => {
|
|
if self.links.remove(&(peer, protocol.clone())).is_some() {
|
|
self.hub.clear_direct(peer, &protocol);
|
|
}
|
|
self.metrics.data_link_failures += 1;
|
|
self.record_protocol_failure(peer, &protocol);
|
|
self.emit(Event::DataLinkDown {
|
|
network: self.network_id,
|
|
peer,
|
|
protocol,
|
|
reason,
|
|
});
|
|
self.ensure_links();
|
|
self.update_paths();
|
|
self.announce_reach(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Rebuilds this agent's announcement and pushes it to every session.
|
|
///
|
|
/// Used when a plugin's capability changed, so peers do not have to wait
|
|
/// for a reconnect to learn about it.
|
|
fn reannounce(&mut self) {
|
|
let announcement = ControlMessage::Announce(self.local_announcement());
|
|
let peers: Vec<EndpointId> = self.sessions.keys().copied().collect();
|
|
for peer in peers {
|
|
if let Err(err) = self.send_to(peer, announcement.clone()) {
|
|
tracing::debug!(%err, "could not queue re-announcement");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Queues a message without blocking the runtime loop.
|
|
///
|
|
/// The envelope is encoded here so that the exact number of control bytes
|
|
/// handed to the transport is known and can be reported honestly.
|
|
///
|
|
/// A full queue is backpressure: the send fails rather than stalling every
|
|
/// other peer in this network.
|
|
fn send_to(&mut self, peer: EndpointId, message: ControlMessage) -> Result<()> {
|
|
let network = self.network_id;
|
|
let envelope = Envelope {
|
|
network_id: *network.as_bytes(),
|
|
message,
|
|
};
|
|
let encoded = encode(&envelope)?;
|
|
let bytes = encoded.len() as u64;
|
|
|
|
let session = self.sessions.get_mut(&peer).ok_or(Error::NoSuchPeer {
|
|
network,
|
|
peer: peer.fmt_short().to_string(),
|
|
})?;
|
|
|
|
match session.outbound.try_send(encoded) {
|
|
Ok(()) => {
|
|
session.messages_sent += 1;
|
|
session.bytes_sent += bytes;
|
|
self.metrics.control_messages_sent += 1;
|
|
self.metrics.control_bytes_sent += bytes;
|
|
Ok(())
|
|
}
|
|
Err(mpsc::error::TrySendError::Full(_)) => Err(Error::Storage(format!(
|
|
"outbound queue for peer {} is full",
|
|
peer.fmt_short()
|
|
))),
|
|
Err(mpsc::error::TrySendError::Closed(_)) => Err(Error::NoSuchPeer {
|
|
network,
|
|
peer: peer.fmt_short().to_string(),
|
|
}),
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------- discovery
|
|
|
|
async fn discovery_round(&mut self) {
|
|
if self.shutdown.is_triggered() {
|
|
return;
|
|
}
|
|
// The fallback range becomes available with the passage of time
|
|
// alone, so something has to look again; this runs on a timer and
|
|
// the check is a comparison when nothing has changed.
|
|
self.ensure_own_claim().await;
|
|
// The same timer repairs the relay picture: reachability is soft
|
|
// state, repeated while it holds and forgotten when it stops.
|
|
self.update_paths();
|
|
self.announce_reach(true);
|
|
|
|
let mut candidates: Vec<Candidate> = Vec::new();
|
|
|
|
// Everybody the signed state says belongs here. Their records
|
|
// reached us through somebody, so we know they exist and who they
|
|
// are, even having never spoken to them; an id with no address is
|
|
// still dialable when the endpoint's own discovery can resolve
|
|
// one. This is what makes a network a mesh rather than a star
|
|
// around whoever was named on a command line.
|
|
for record in self.state.records() {
|
|
if let Ok(author) = record.author_id()
|
|
&& author != self.local_id
|
|
{
|
|
candidates.push(Candidate::new(
|
|
iroh::EndpointAddr::new(author),
|
|
CandidateSource::Member,
|
|
));
|
|
}
|
|
}
|
|
|
|
// A stale or missing cache only changes which candidates we try first.
|
|
// It never bypasses authentication.
|
|
for hint in self.params.storage.hints_for_network(self.network_id).await {
|
|
let Ok(endpoint_id) = EndpointId::from_bytes(&hint.endpoint_id) else {
|
|
continue;
|
|
};
|
|
let Some(addr) = decode_hint(endpoint_id, &hint.addr) else {
|
|
continue;
|
|
};
|
|
candidates.push(Candidate::new(addr, CandidateSource::Cache));
|
|
}
|
|
|
|
for candidate in candidates {
|
|
self.add_candidate(candidate);
|
|
}
|
|
|
|
self.start_dials();
|
|
// Last, so it carries what this round learned.
|
|
self.introduce_peers();
|
|
self.refresh_underlay().await;
|
|
}
|
|
|
|
/// Hands the interface the underlay destinations this network depends
|
|
/// on: the verified direct IPv4 paths of every control connection, and
|
|
/// the relays in use. Only hosts that cannot exempt the agent from an
|
|
/// exit node's default route read it.
|
|
async fn refresh_underlay(&self) {
|
|
let Some(interface) = self.params.interface.clone() else {
|
|
return;
|
|
};
|
|
let mut addresses = Vec::new();
|
|
let mut hosts = Vec::new();
|
|
for session in self.sessions.values() {
|
|
for path in snapshot_connection(&session.conn).paths {
|
|
match path.remote {
|
|
PathAddr::Ip(std::net::SocketAddr::V4(socket)) => addresses.push(*socket.ip()),
|
|
PathAddr::Relay(url) => hosts.extend(relay_host(&url)),
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
for url in self.params.adapter.snapshot().relay_urls {
|
|
hosts.extend(relay_host(&url));
|
|
}
|
|
interface
|
|
.set_underlay(self.network_id, addresses, hosts)
|
|
.await;
|
|
}
|
|
|
|
fn add_candidate(&mut self, candidate: Candidate) {
|
|
let peer = candidate.endpoint_id();
|
|
if peer == self.local_id {
|
|
return;
|
|
}
|
|
if candidate.source == CandidateSource::Discovery && !self.dial_states.contains_key(&peer) {
|
|
let count = self
|
|
.dial_states
|
|
.values()
|
|
.filter(|s| s.source == CandidateSource::Discovery)
|
|
.count();
|
|
if count >= self.params.limits.max_discovery_candidates {
|
|
let replace = self
|
|
.dial_states
|
|
.iter()
|
|
.filter(|(id, s)| {
|
|
s.source == CandidateSource::Discovery
|
|
&& !s.in_flight
|
|
&& !self.sessions.contains_key(*id)
|
|
})
|
|
.max_by_key(|(_, s)| s.consecutive_failures)
|
|
.map(|(id, _)| *id);
|
|
let Some(replace) = replace else {
|
|
return;
|
|
};
|
|
self.dial_states.remove(&replace);
|
|
self.candidate_addrs.remove(&replace);
|
|
}
|
|
}
|
|
self.candidate_addrs
|
|
.entry(peer)
|
|
.and_modify(|existing| {
|
|
if candidate.source == CandidateSource::Discovery {
|
|
*existing = candidate.addr.clone();
|
|
} else {
|
|
merge_addr(existing, &candidate.addr);
|
|
}
|
|
})
|
|
.or_insert_with(|| candidate.addr.clone());
|
|
self.dial_states
|
|
.entry(peer)
|
|
.or_insert_with(|| DialState::new(candidate.source));
|
|
}
|
|
|
|
fn start_dials(&mut self) {
|
|
let now = Instant::now();
|
|
let in_flight = self
|
|
.dial_states
|
|
.values()
|
|
.filter(|state| state.in_flight)
|
|
.count();
|
|
let mut budget = self
|
|
.params
|
|
.limits
|
|
.max_concurrent_dials
|
|
.saturating_sub(in_flight);
|
|
if budget == 0 || self.sessions.len() >= self.params.limits.max_sessions_per_network {
|
|
return;
|
|
}
|
|
|
|
let ready: Vec<EndpointId> = self
|
|
.dial_states
|
|
.iter()
|
|
.filter(|(peer, state)| {
|
|
!state.in_flight && state.next_attempt <= now && !self.sessions.contains_key(*peer)
|
|
})
|
|
.map(|(peer, _)| *peer)
|
|
.collect();
|
|
|
|
for peer in ready {
|
|
if budget == 0 {
|
|
break;
|
|
}
|
|
let Some(addr) = self.candidate_addrs.get(&peer).cloned() else {
|
|
continue;
|
|
};
|
|
if let Some(state) = self.dial_states.get_mut(&peer) {
|
|
state.in_flight = true;
|
|
}
|
|
budget -= 1;
|
|
self.metrics.dial_attempts += 1;
|
|
|
|
let adapter = self.params.adapter.clone();
|
|
let keys = self.params.keys.clone();
|
|
let limits = Arc::clone(&self.params.limits);
|
|
let results = self.dial_results_tx.clone();
|
|
let local_id = self.local_id;
|
|
let shutdown = self.shutdown.clone();
|
|
|
|
tokio::spawn(async move {
|
|
let outcome = tokio::select! {
|
|
biased;
|
|
_ = shutdown.wait() => DialOutcome::Failed {
|
|
peer,
|
|
reason: "network deactivated".into(),
|
|
during_handshake: false,
|
|
},
|
|
outcome = dial(adapter, addr, keys, limits, local_id, peer) => outcome,
|
|
};
|
|
let _ = results.send(outcome).await;
|
|
});
|
|
}
|
|
}
|
|
|
|
async fn handle_dial_result(&mut self, result: DialOutcome) {
|
|
match result {
|
|
DialOutcome::Established(inbound) => {
|
|
let peer = inbound.outcome.peer;
|
|
if let Some(state) = self.dial_states.get_mut(&peer) {
|
|
state.in_flight = false;
|
|
state.consecutive_failures = 0;
|
|
state.next_attempt = Instant::now();
|
|
}
|
|
self.install_session(*inbound).await;
|
|
}
|
|
DialOutcome::Failed {
|
|
peer,
|
|
reason,
|
|
during_handshake,
|
|
} => {
|
|
self.metrics.dial_failures += 1;
|
|
if during_handshake {
|
|
self.metrics.handshake_failures += 1;
|
|
}
|
|
let give_up = {
|
|
let policy = &self.params.reconnect;
|
|
let state = self
|
|
.dial_states
|
|
.entry(peer)
|
|
.or_insert_with(|| DialState::new(CandidateSource::Discovery));
|
|
state.in_flight = false;
|
|
state.consecutive_failures = state.consecutive_failures.saturating_add(1);
|
|
let delay = policy.delay_for(state.consecutive_failures);
|
|
state.next_attempt = Instant::now() + delay;
|
|
policy
|
|
.max_consecutive_failures
|
|
.is_some_and(|max| state.consecutive_failures >= max)
|
|
};
|
|
if give_up {
|
|
// Keep the backoff entry but push it far out; discovery
|
|
// seeing the peer again resets it.
|
|
if let Some(state) = self.dial_states.get_mut(&peer) {
|
|
state.next_attempt = Instant::now() + self.params.reconnect.max_delay;
|
|
}
|
|
}
|
|
self.emit(Event::DialFailed {
|
|
network: self.network_id,
|
|
peer,
|
|
reason,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------- agreed state
|
|
|
|
/// Reads back what this agent already knew before it restarted.
|
|
///
|
|
/// Records are verified again on load: the database is not a trust
|
|
/// boundary, because a restored backup or a copied file could hold
|
|
/// anything.
|
|
async fn load_state(&mut self) {
|
|
let stored = self
|
|
.params
|
|
.storage
|
|
.signed_records(self.network_id)
|
|
.await
|
|
.unwrap_or_default();
|
|
let (_, errors) = self.state.merge_all(self.network_id, stored);
|
|
for err in errors {
|
|
tracing::warn!(%err, "discarding an unusable stored record");
|
|
}
|
|
self.own_version = self
|
|
.params
|
|
.storage
|
|
.own_record_version(self.network_id, self.local_id)
|
|
.await
|
|
.unwrap_or(0);
|
|
for (id, hostname) in self
|
|
.params
|
|
.storage
|
|
.peer_hostnames(self.network_id)
|
|
.await
|
|
.unwrap_or_default()
|
|
{
|
|
if let Ok(id) = EndpointId::from_bytes(&id) {
|
|
self.known_hostnames.insert(id, hostname);
|
|
}
|
|
}
|
|
|
|
self.ensure_own_claim().await;
|
|
self.publish_allocations().await;
|
|
}
|
|
|
|
/// The range this network uses: whatever it has already settled on, else
|
|
/// what this agent was configured with.
|
|
///
|
|
/// Adopting the agreed one is what lets a participant join without being
|
|
/// told the range out of band.
|
|
fn effective_range(&self) -> Option<Ipv4Range> {
|
|
if let Some(agreed) = self.state.agreed_range() {
|
|
return Some(agreed);
|
|
}
|
|
// Nobody has settled one yet, so this agent would be proposing its
|
|
// own. It must not propose a range it cannot route: one agent has
|
|
// one interface, and claiming an address in a range another network
|
|
// already owns would spread the collision rather than contain it —
|
|
// "the lowest author's range wins" would carry it to everybody.
|
|
if let Some(wanted) = self.params.ipv4_range
|
|
&& self
|
|
.params
|
|
.routes
|
|
.would_overlap(self.network_id, wanted)
|
|
.is_none()
|
|
{
|
|
return Some(wanted);
|
|
}
|
|
|
|
// The configured range is another network's here, so this one falls
|
|
// back to the range derived from its id — after a moment's wait, so
|
|
// that a network which already exists gets to say what it uses
|
|
// first. Every member derives the same one, so members that reach
|
|
// this point agree without negotiating.
|
|
let fallback = self.params.ipv4_fallback?;
|
|
if self.activated.elapsed() < RANGE_PROPOSAL_GRACE {
|
|
return None;
|
|
}
|
|
self.params
|
|
.routes
|
|
.would_overlap(self.network_id, fallback)
|
|
.is_none()
|
|
.then_some(fallback)
|
|
}
|
|
|
|
/// Makes sure this agent holds an address, claiming one if it does not.
|
|
///
|
|
/// Called after anything that could change the picture: startup, and
|
|
/// every time another replica's records arrive.
|
|
async fn ensure_own_claim(&mut self) {
|
|
// On the way out. Claiming again here is how a goodbye becomes a
|
|
// hello nobody asked for.
|
|
if self.released {
|
|
return;
|
|
}
|
|
let wanted_hostname = {
|
|
let hostname = crate::state::sanitise_hostname(&self.params.hostname);
|
|
(!hostname.is_empty()).then_some(hostname)
|
|
};
|
|
let range = self.effective_range();
|
|
|
|
let wanted_address = match range {
|
|
None => None,
|
|
Some(range) => {
|
|
let holders = self.state.address_holders();
|
|
// An address we still hold is kept; this is what makes a
|
|
// returning participant get its old address back.
|
|
match self.state.address_of(&self.local_id) {
|
|
Some(mine) if range.contains(mine) => Some(mine),
|
|
_ => {
|
|
let taken: std::collections::HashSet<std::net::Ipv4Addr> = holders
|
|
.iter()
|
|
.filter(|(_, holder)| **holder != self.local_id)
|
|
.map(|(address, _)| *address)
|
|
.collect();
|
|
match allocate(
|
|
self.network_id,
|
|
self.local_id,
|
|
range,
|
|
&taken,
|
|
self.state
|
|
.get(&self.local_id)
|
|
.and_then(|record| record.body.claimed_address()),
|
|
) {
|
|
Ok(address) => Some(address),
|
|
Err(err) => {
|
|
self.metrics.plugin_errors += 1;
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol: "overlay".into(),
|
|
reason: err.to_string(),
|
|
});
|
|
None
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
let body = RecordBody::Claim {
|
|
address: wanted_address,
|
|
// The range only travels with an address, so a member of an
|
|
// IPv6-only network does not assert one.
|
|
range: wanted_address.and(range),
|
|
hostname: wanted_hostname,
|
|
};
|
|
|
|
// Nothing to say is not the same as saying nothing changed: a member
|
|
// that claims neither an address nor a name has no reason to occupy a
|
|
// record at all.
|
|
if matches!(
|
|
&body,
|
|
RecordBody::Claim {
|
|
address: None,
|
|
hostname: None,
|
|
..
|
|
}
|
|
) {
|
|
return;
|
|
}
|
|
|
|
// Republishing an unchanged claim would bump the version for no
|
|
// reason and make every replica store it again.
|
|
if self
|
|
.state
|
|
.get(&self.local_id)
|
|
.is_some_and(|record| record.body == body)
|
|
{
|
|
return;
|
|
}
|
|
|
|
self.publish_record(body).await;
|
|
}
|
|
|
|
/// Signs, stores and announces one of this agent's own records.
|
|
///
|
|
/// Stored before it is announced, in one transaction with the version
|
|
/// counter, so a crash can never let us reuse a version we already put on
|
|
/// the wire.
|
|
async fn publish_record(&mut self, body: RecordBody) {
|
|
let version = self.own_version.saturating_add(1);
|
|
let record = SignedRecord::sign(&self.params.device_secret, self.network_id, version, body);
|
|
|
|
if let Err(err) = self.params.storage.publish_own_record(record.clone()).await {
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol: "overlay".into(),
|
|
reason: format!("cannot store our own record: {err}"),
|
|
});
|
|
return;
|
|
}
|
|
self.own_version = version;
|
|
|
|
match self.state.merge(self.network_id, record) {
|
|
Ok(_) => {}
|
|
Err(err) => {
|
|
tracing::error!(%err, "our own record did not verify");
|
|
return;
|
|
}
|
|
}
|
|
self.broadcast_state();
|
|
}
|
|
|
|
/// Handles snapshots collected while dispatching messages.
|
|
async fn drain_pending_state(&mut self) {
|
|
for (peer, records) in std::mem::take(&mut self.pending_state) {
|
|
self.receive_state(peer, records).await;
|
|
}
|
|
}
|
|
|
|
/// Sends everything we know to every peer.
|
|
fn broadcast_state(&mut self) {
|
|
let mut records = self.state.records();
|
|
records.truncate(self.params.limits.max_state_records);
|
|
if records.is_empty() {
|
|
return;
|
|
}
|
|
let message = ControlMessage::State { records };
|
|
let peers: Vec<EndpointId> = self.sessions.keys().copied().collect();
|
|
for peer in peers {
|
|
if let Err(err) = self.send_to(peer, message.clone()) {
|
|
tracing::debug!(%err, "could not queue a state snapshot");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Merges a snapshot from a peer.
|
|
async fn receive_state(&mut self, peer: EndpointId, records: Vec<SignedRecord>) {
|
|
let before = self.state.records();
|
|
let (outcomes, errors) = self.state.merge_all(self.network_id, records);
|
|
|
|
for err in errors {
|
|
self.metrics.protocol_violations += 1;
|
|
self.emit(Event::ProtocolViolation {
|
|
network: Some(self.network_id),
|
|
peer: Some(peer),
|
|
reason: format!("unusable signed record: {err}"),
|
|
});
|
|
}
|
|
for outcome in &outcomes {
|
|
if *outcome == Merged::Conflicted {
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol: "overlay".into(),
|
|
reason: "two different records from one author at the same version; \
|
|
a device key appears to be in use in two places"
|
|
.into(),
|
|
});
|
|
}
|
|
}
|
|
|
|
let changed = outcomes.iter().any(|outcome| {
|
|
matches!(
|
|
outcome,
|
|
Merged::Added | Merged::Updated | Merged::Conflicted
|
|
)
|
|
});
|
|
if !changed {
|
|
return;
|
|
}
|
|
|
|
for record in self.state.records() {
|
|
if record.author == *self.local_id.as_bytes() {
|
|
continue;
|
|
}
|
|
if let Err(err) = self.params.storage.put_signed_record(record).await {
|
|
tracing::debug!(%err, "cannot persist a record");
|
|
}
|
|
}
|
|
|
|
// Somebody may have taken the address we were using.
|
|
self.ensure_own_claim().await;
|
|
self.publish_allocations().await;
|
|
if self.state.records() != before {
|
|
self.broadcast_state();
|
|
}
|
|
}
|
|
|
|
fn broadcast_policy(&self) -> crate::overlay::broadcast::BroadcastPolicy {
|
|
crate::overlay::broadcast::BroadcastPolicy {
|
|
enabled: self.params.broadcast,
|
|
peers: self
|
|
.sessions
|
|
.values()
|
|
.filter(|session| session.broadcast)
|
|
.map(|session| session.peer)
|
|
.collect(),
|
|
}
|
|
}
|
|
|
|
fn exit_policy(&self) -> crate::overlay::ExitPolicy {
|
|
crate::overlay::ExitPolicy {
|
|
offer: self.params.exit_offer,
|
|
via: self.params.exit_via,
|
|
}
|
|
}
|
|
|
|
/// Whether this agent offers to be an exit node and can really be one.
|
|
fn exit_ready(&self) -> bool {
|
|
self.params.exit_offer
|
|
&& self
|
|
.params
|
|
.interface
|
|
.as_ref()
|
|
.is_some_and(|interface| interface.exit_offer_ready(self.network_id))
|
|
}
|
|
|
|
/// Whether a member is connected and says it is an exit node.
|
|
fn exit_node_online(&self, peer: EndpointId) -> bool {
|
|
self.sessions
|
|
.get(&peer)
|
|
.is_some_and(|session| session.exit_node)
|
|
}
|
|
|
|
/// Puts the exit settings into the routing table and the host rules.
|
|
///
|
|
/// The routes and the rules stay as they are when the exit node in use
|
|
/// goes away: dropping them would send the traffic out the old way, in
|
|
/// the clear, without anyone having asked.
|
|
async fn apply_exit(&mut self) {
|
|
self.params
|
|
.routes
|
|
.set_exit(self.network_id, self.exit_policy());
|
|
if let Some(interface) = self.params.interface.clone() {
|
|
interface.sync_exit_rules().await;
|
|
}
|
|
// Said only once it is true: the rules may not have taken, or may
|
|
// only now be possible.
|
|
if self.exit_ready() != self.announced_exit {
|
|
self.reannounce();
|
|
}
|
|
}
|
|
|
|
/// Says so, once, when the exit node in use is lost or comes back.
|
|
fn watch_exit_via(&mut self) {
|
|
let Some(via) = self.params.exit_via else {
|
|
return;
|
|
};
|
|
let online = self.exit_node_online(via);
|
|
if online == self.exit_via_was_online {
|
|
return;
|
|
}
|
|
self.exit_via_was_online = online;
|
|
if online {
|
|
tracing::info!(
|
|
network = %self.network_id,
|
|
exit_node = %via.fmt_short(),
|
|
"the exit node is back"
|
|
);
|
|
} else {
|
|
tracing::warn!(
|
|
network = %self.network_id,
|
|
exit_node = %via.fmt_short(),
|
|
"the exit node is gone or no longer offers to be one; internet traffic stays \
|
|
routed through it and is dropped until it returns or `tsng exit-node off`"
|
|
);
|
|
}
|
|
}
|
|
|
|
fn update_broadcast(&self) {
|
|
self.params
|
|
.routes
|
|
.set_broadcast(self.network_id, self.broadcast_policy());
|
|
}
|
|
|
|
/// Tells the plugins who holds which overlay address.
|
|
async fn publish_allocations(&mut self) {
|
|
let Some(range) = self.effective_range() else {
|
|
return;
|
|
};
|
|
let mut allocations: Vec<(EndpointId, std::net::Ipv4Addr)> = self
|
|
.state
|
|
.address_holders()
|
|
.into_iter()
|
|
.map(|(address, holder)| (holder, address))
|
|
.collect();
|
|
allocations.sort_by_key(|(holder, _)| *holder.as_bytes());
|
|
|
|
for plugin in &self.params.plugins {
|
|
plugin.on_address_allocation(self.network_id, range, &allocations);
|
|
}
|
|
|
|
// And the system level's own view, which is what decides whose
|
|
// packet is whose. The local address is kept out of the peer list:
|
|
// a packet for ourselves does not go over a tunnel.
|
|
let local = self.state.address_of(&self.local_id);
|
|
let routes = crate::overlay::NetworkRoutes {
|
|
exit: self.exit_policy(),
|
|
broadcast: self.broadcast_policy(),
|
|
range: Some(range),
|
|
local,
|
|
peers: allocations
|
|
.iter()
|
|
.filter(|(holder, _)| *holder != self.local_id)
|
|
.map(|(holder, address)| (*address, *holder))
|
|
.collect(),
|
|
};
|
|
if let Err(err) = self.params.routes.set_network(self.network_id, routes) {
|
|
// Reported once per change rather than swallowed: two networks
|
|
// wanting the same addresses is a thing the user has to settle.
|
|
self.metrics.plugin_errors += 1;
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol: "overlay".into(),
|
|
reason: err.to_string(),
|
|
});
|
|
return;
|
|
}
|
|
let Some(interface) = self.params.interface.clone() else {
|
|
return;
|
|
};
|
|
if let Err(err) = interface.sync_addresses().await {
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol: "overlay".into(),
|
|
reason: err.to_string(),
|
|
});
|
|
return;
|
|
}
|
|
|
|
// The agent assigns this address itself, so finding it absent means
|
|
// the assignment did not take — something outside removed it, or the
|
|
// provisioner reported a success it did not achieve. Left unsaid it
|
|
// looks like a broken network: packets would leave with the wrong
|
|
// source and every peer would drop them. Checked rather than
|
|
// assumed, because the assumption is exactly the kind that has been
|
|
// wrong here before. Said once per address, not every round.
|
|
// An in-memory interface has no addresses to be missing from: with
|
|
// `--no-tun` this is the arrangement asked for, not a fault.
|
|
let is_missing = if let Some(address) = local {
|
|
if interface.on_host() {
|
|
let mut local_found =
|
|
crate::overlay::address_is_local(std::net::IpAddr::V4(address));
|
|
// On Windows, duplicate address detection (DAD) and netsh/IP helper updates
|
|
// may take a short moment before a newly assigned address is ready for binding.
|
|
if !local_found && cfg!(target_os = "windows") {
|
|
for _ in 0..10 {
|
|
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
|
|
if crate::overlay::address_is_local(std::net::IpAddr::V4(address)) {
|
|
local_found = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
!local_found
|
|
} else {
|
|
false
|
|
}
|
|
} else {
|
|
false
|
|
};
|
|
|
|
let missing = if is_missing {
|
|
if let Some(address) = local {
|
|
let already = self.reported_missing == Some(address);
|
|
self.reported_missing = Some(address);
|
|
(!already).then_some(address)
|
|
} else {
|
|
None
|
|
}
|
|
} else {
|
|
self.reported_missing = None;
|
|
None
|
|
};
|
|
if let Some(address) = missing {
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol: "overlay".into(),
|
|
reason: format!(
|
|
"this agent was allocated {address}/{} but the address is not on any \
|
|
interface, so the overlay cannot work: packets would leave with the \
|
|
wrong source and every peer would drop them. It should have been \
|
|
assigned to `{}` automatically; check whether something else removed \
|
|
it.",
|
|
range.prefix_len,
|
|
interface.name()
|
|
),
|
|
});
|
|
}
|
|
|
|
// The broadcast route names this address as its source, so it goes in
|
|
// only once the address is really on the interface.
|
|
if !is_missing {
|
|
interface.sync_broadcast_rules().await;
|
|
self.apply_exit().await;
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------ data plane
|
|
|
|
/// The protocols this agent speaks, with the version it speaks them at.
|
|
fn served_protocols(&self) -> Vec<(String, u16)> {
|
|
self.params
|
|
.plugins
|
|
.iter()
|
|
.map(|plugin| (plugin.protocol_id().to_string(), plugin.protocol_version()))
|
|
.collect()
|
|
}
|
|
|
|
fn record_protocol_failure(&mut self, peer: EndpointId, protocol: &str) {
|
|
let key = (peer, protocol.to_string());
|
|
let entry = self
|
|
.protocol_dial_states
|
|
.entry(key)
|
|
.or_insert(ProtocolDialState {
|
|
failures: 0,
|
|
cooldown_until: std::time::Instant::now(),
|
|
});
|
|
entry.failures = entry.failures.saturating_add(1);
|
|
let backoff = data_link_backoff(entry.failures);
|
|
entry.cooldown_until = std::time::Instant::now() + backoff;
|
|
}
|
|
|
|
fn record_protocol_success(&mut self, peer: EndpointId, protocol: &str) {
|
|
self.protocol_dial_states
|
|
.remove(&(peer, protocol.to_string()));
|
|
}
|
|
|
|
/// Opens whatever data plane links are missing, and forgets dead ones.
|
|
///
|
|
/// Only one side dials, chosen by a rule both sides compute the same way,
|
|
/// so two agents never open two links for the same thing.
|
|
fn ensure_links(&mut self) {
|
|
let Some(transport) = self.params.transport.clone() else {
|
|
return;
|
|
};
|
|
let served = self.served_protocols();
|
|
if served.is_empty() {
|
|
return;
|
|
}
|
|
|
|
let mut dead: Vec<(EndpointId, String)> = Vec::new();
|
|
for (key, link) in &self.links {
|
|
if link.is_closed() || self.no_direct_data(key.0) {
|
|
dead.push(key.clone());
|
|
}
|
|
}
|
|
for (peer, protocol) in dead {
|
|
self.links.remove(&(peer, protocol.clone()));
|
|
// Only the direct path is gone. Whether the peer is still
|
|
// reachable through somebody is decided below; the protocol's
|
|
// link stays either way.
|
|
self.hub.clear_direct(peer, &protocol);
|
|
self.record_protocol_failure(peer, &protocol);
|
|
self.emit(Event::DataLinkDown {
|
|
network: self.network_id,
|
|
peer,
|
|
protocol,
|
|
reason: "link closed".into(),
|
|
});
|
|
}
|
|
|
|
// Choose the highest priority mutually supported protocol for each peer.
|
|
// If a direct data link is already open or opening, do not dial another.
|
|
// If a protocol recently failed, fall back to other mutually supported
|
|
// protocols according to backoff cooldown.
|
|
let now = std::time::Instant::now();
|
|
let mut wanted: Vec<(EndpointId, String)> = Vec::new();
|
|
for session in self.sessions.values() {
|
|
let peer = session.peer;
|
|
if self.links.keys().any(|(p, _)| *p == peer)
|
|
|| self.opening.iter().any(|(p, _)| *p == peer)
|
|
{
|
|
continue;
|
|
}
|
|
if self.no_direct_data(peer) {
|
|
continue;
|
|
}
|
|
// The smaller endpoint id dials; the other side accepts. Both
|
|
// compute this identically, so exactly one link is created.
|
|
if self.local_id.as_bytes() >= peer.as_bytes() {
|
|
continue;
|
|
}
|
|
|
|
let supported: Vec<String> = served
|
|
.iter()
|
|
.filter(|(name, ours)| {
|
|
session
|
|
.capabilities
|
|
.iter()
|
|
.any(|c| c.enabled && c.protocol == *name && c.version == *ours)
|
|
})
|
|
.map(|(name, _)| name.clone())
|
|
.collect();
|
|
|
|
if supported.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
let chosen = supported.iter().find(|proto| {
|
|
match self.protocol_dial_states.get(&(peer, (*proto).clone())) {
|
|
Some(state) => state.cooldown_until <= now,
|
|
None => true,
|
|
}
|
|
});
|
|
|
|
if let Some(proto) = chosen {
|
|
wanted.push((peer, proto.clone()));
|
|
}
|
|
}
|
|
|
|
for (peer, protocol) in wanted {
|
|
let key = (peer, protocol.clone());
|
|
if self.links.contains_key(&key) || self.opening.contains(&key) {
|
|
continue;
|
|
}
|
|
self.opening.insert(key);
|
|
|
|
let results = self.link_results_tx.clone();
|
|
let transport = Arc::clone(&transport);
|
|
let network = self.network_id;
|
|
let shutdown = self.shutdown.clone();
|
|
tokio::spawn(async move {
|
|
let result = tokio::select! {
|
|
biased;
|
|
_ = shutdown.wait() => Err("network deactivated".to_string()),
|
|
result = transport.open(network, peer, &protocol) => {
|
|
result.map_err(|err| err.to_string())
|
|
}
|
|
};
|
|
let _ = results
|
|
.send(LinkOutcome {
|
|
peer,
|
|
protocol,
|
|
result,
|
|
})
|
|
.await;
|
|
});
|
|
}
|
|
}
|
|
|
|
/// Takes in what another member knows about the rest.
|
|
///
|
|
/// Returns how many of them were new here, which is the only reason to
|
|
/// start dialling again straight away.
|
|
fn learn_peers(&mut self, hints: &[crate::proto::message::PeerHint]) -> usize {
|
|
let mut learned = 0;
|
|
for hint in hints {
|
|
let Ok(peer) = EndpointId::from_bytes(&hint.endpoint) else {
|
|
continue;
|
|
};
|
|
if peer == self.local_id || self.sessions.contains_key(&peer) {
|
|
continue;
|
|
}
|
|
let mut addr = iroh::EndpointAddr::new(peer);
|
|
for text in &hint.addrs {
|
|
// The same decoder as the cache's hints: one spelling, one
|
|
// place it is read, nothing to drift.
|
|
if let Some(decoded) = decode_hint(peer, text) {
|
|
merge_addr(&mut addr, &decoded);
|
|
}
|
|
}
|
|
|
|
let fresh = !self.candidate_addrs.contains_key(&peer);
|
|
self.candidate_addrs
|
|
.entry(peer)
|
|
.and_modify(|existing| merge_addr(existing, &addr))
|
|
.or_insert(addr);
|
|
self.dial_states
|
|
.entry(peer)
|
|
.or_insert_with(|| DialState::new(CandidateSource::Introduced));
|
|
if fresh {
|
|
learned += 1;
|
|
}
|
|
}
|
|
learned
|
|
}
|
|
|
|
/// Tells the peers it is talking to about the peers it knows.
|
|
///
|
|
/// A device told about one member ends up talking to all of them: a
|
|
/// network is a mesh, and which member happened to be named on a
|
|
/// command line should not decide who anybody talks to. Sent on every
|
|
/// round, because it is cheap, bounded and self-repairing.
|
|
fn introduce_peers(&mut self) {
|
|
let mut hints: Vec<crate::proto::message::PeerHint> = Vec::new();
|
|
|
|
// What this agent can see for itself: where each peer it is
|
|
// talking to is right now. An agent that only ever accepts has no
|
|
// candidates of its own, and it is exactly the one everybody else
|
|
// was pointed at — so without this, introductions would come from
|
|
// nobody.
|
|
let mut observed: HashMap<EndpointId, iroh::EndpointAddr> = HashMap::new();
|
|
for session in self.sessions.values() {
|
|
let snapshot = crate::net::snapshot_connection(&session.conn);
|
|
let mut addr = iroh::EndpointAddr::new(session.peer);
|
|
for path in &snapshot.paths {
|
|
match &path.remote {
|
|
crate::net::PathAddr::Ip(socket) => addr = addr.with_ip_addr(*socket),
|
|
crate::net::PathAddr::Relay(url) => {
|
|
if let Ok(url) = url.parse() {
|
|
addr = addr.with_relay_url(url);
|
|
}
|
|
}
|
|
crate::net::PathAddr::Other(_) => {}
|
|
}
|
|
}
|
|
observed.insert(session.peer, addr);
|
|
}
|
|
for (peer, addr) in &self.candidate_addrs {
|
|
observed
|
|
.entry(*peer)
|
|
.and_modify(|existing| merge_addr(existing, addr))
|
|
.or_insert_with(|| addr.clone());
|
|
}
|
|
|
|
for (peer, addr) in &observed {
|
|
if *peer == self.local_id {
|
|
continue;
|
|
}
|
|
// An id with no address at all is still worth passing on: the
|
|
// other side's own discovery may be able to resolve it.
|
|
hints.push(crate::proto::message::PeerHint {
|
|
endpoint: *peer.as_bytes(),
|
|
addrs: hint_addrs(addr),
|
|
});
|
|
}
|
|
// This agent, too: it is the one member the others cannot be told
|
|
// about by anybody else.
|
|
hints.push(crate::proto::message::PeerHint {
|
|
endpoint: *self.local_id.as_bytes(),
|
|
addrs: hint_addrs(&self.params.adapter.addr()),
|
|
});
|
|
hints.truncate(self.params.limits.max_state_records);
|
|
if hints.is_empty() {
|
|
return;
|
|
}
|
|
|
|
let message = ControlMessage::Peers { peers: hints };
|
|
let peers: Vec<EndpointId> = self.sessions.keys().copied().collect();
|
|
for peer in peers {
|
|
if let Err(err) = self.send_to(peer, message.clone()) {
|
|
tracing::debug!(%err, "could not queue an introduction");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Publishes only locally observed, protocol-specific transport edges.
|
|
fn announce_reach(&mut self, force: bool) {
|
|
let live: HashSet<_> = self
|
|
.links
|
|
.iter()
|
|
.filter(|(_, link)| !link.is_closed())
|
|
.map(
|
|
|((peer, protocol), _)| crate::proto::message::ReachableLink {
|
|
peer: *peer.as_bytes(),
|
|
protocol: protocol.clone(),
|
|
},
|
|
)
|
|
.collect();
|
|
if !force && live == self.announced_reach {
|
|
return;
|
|
}
|
|
self.announced_reach = live.clone();
|
|
let mut links: Vec<_> = live.into_iter().collect();
|
|
links.sort_by(|a, b| (&a.protocol, a.peer).cmp(&(&b.protocol, b.peer)));
|
|
links.truncate(self.params.limits.max_state_records);
|
|
let message = ControlMessage::Reachable { links };
|
|
let peers: Vec<_> = self.sessions.keys().copied().collect();
|
|
for peer in peers {
|
|
if let Err(err) = self.send_to(peer, message.clone()) {
|
|
tracing::debug!(%err, "could not queue a reachability announcement");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Builds the graph off the packet path. Every remote row comes from that
|
|
/// member's authenticated control session; stale or incompatible rows never
|
|
/// enter the graph. The hub replaces an immutable table only on changes.
|
|
fn update_paths(&mut self) {
|
|
self.reachable.retain(|peer, (_, heard)| {
|
|
heard.elapsed() < REACH_EXPIRY && self.sessions.contains_key(peer)
|
|
});
|
|
for (protocol, version) in self.served_protocols() {
|
|
let peers: Vec<_> = self
|
|
.sessions
|
|
.values()
|
|
.filter(|session| {
|
|
session.capabilities.iter().any(|capability| {
|
|
capability.enabled
|
|
&& capability.protocol == protocol
|
|
&& capability.version == version
|
|
})
|
|
})
|
|
.map(|session| session.peer)
|
|
.collect();
|
|
let mut members: HashSet<_> = peers.iter().map(|peer| *peer.as_bytes()).collect();
|
|
members.insert(*self.local_id.as_bytes());
|
|
let graph = self
|
|
.reachable
|
|
.iter()
|
|
.filter(|(peer, _)| members.contains(peer.as_bytes()))
|
|
.map(|(peer, (links, _))| {
|
|
let mut neighbors: Vec<_> = links
|
|
.iter()
|
|
.filter(|link| {
|
|
link.protocol == protocol
|
|
&& members.contains(&link.peer)
|
|
&& link.peer != *peer.as_bytes()
|
|
})
|
|
.map(|link| link.peer)
|
|
.collect();
|
|
neighbors.sort_unstable();
|
|
(*peer.as_bytes(), neighbors)
|
|
})
|
|
.collect();
|
|
self.hub.set_topology(&protocol, graph, members);
|
|
for peer in peers {
|
|
if self.hub.has_link(peer, &protocol) || !self.hub.reachable(peer, &protocol) {
|
|
continue;
|
|
}
|
|
let Some(plugin) = self
|
|
.params
|
|
.plugins
|
|
.iter()
|
|
.find(|plugin| plugin.protocol_id() == protocol)
|
|
.cloned()
|
|
else {
|
|
continue;
|
|
};
|
|
let link = self.hub.link(peer, &protocol);
|
|
let path = link.path_description();
|
|
let max_datagram = link.max_datagram_size();
|
|
plugin.on_peer_link(self.network_id, peer, link);
|
|
self.emit(Event::DataLinkUp {
|
|
network: self.network_id,
|
|
peer,
|
|
protocol: protocol.clone(),
|
|
path,
|
|
max_datagram,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
fn handle_link_result(&mut self, outcome: LinkOutcome) {
|
|
let key = (outcome.peer, outcome.protocol.clone());
|
|
self.opening.remove(&key);
|
|
match outcome.result {
|
|
Ok(link) => {
|
|
self.record_protocol_success(outcome.peer, &outcome.protocol);
|
|
self.adopt_link(outcome.peer, outcome.protocol, link);
|
|
// A new link is both a path for us and one we can offer
|
|
// others, so it changes what we route and what we say.
|
|
self.update_paths();
|
|
self.announce_reach(false);
|
|
}
|
|
Err(reason) => {
|
|
// A data plane that cannot be set up is reported, never fatal.
|
|
self.metrics.data_link_failures += 1;
|
|
self.record_protocol_failure(outcome.peer, &outcome.protocol);
|
|
self.emit(Event::DataLinkDown {
|
|
network: self.network_id,
|
|
peer: outcome.peer,
|
|
protocol: outcome.protocol,
|
|
reason,
|
|
});
|
|
// Fall back to other mutually supported protocols immediately.
|
|
self.ensure_links();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Whether this agent refuses a direct data path to a peer.
|
|
///
|
|
/// Always false outside a test build: see
|
|
/// [`crate::config::AgentConfig::with_unreachable_data_peers`].
|
|
fn no_direct_data(&self, peer: EndpointId) -> bool {
|
|
#[cfg(feature = "testing")]
|
|
{
|
|
match self.params.unreachable_data_peers.lock() {
|
|
Ok(guard) => guard.contains(&peer),
|
|
Err(poisoned) => poisoned.into_inner().contains(&peer),
|
|
}
|
|
}
|
|
#[cfg(not(feature = "testing"))]
|
|
{
|
|
let _ = peer;
|
|
false
|
|
}
|
|
}
|
|
|
|
fn install_link(&mut self, inbound: InboundLink) {
|
|
if self.no_direct_data(inbound.peer) {
|
|
return;
|
|
}
|
|
self.record_protocol_success(inbound.peer, &inbound.protocol);
|
|
self.adopt_link(inbound.peer, inbound.protocol, inbound.link);
|
|
self.update_paths();
|
|
self.announce_reach(false);
|
|
}
|
|
|
|
/// Hands a link to the plugin that owns its protocol.
|
|
fn adopt_link(&mut self, peer: EndpointId, protocol: String, link: SharedLink) {
|
|
let Some(plugin) = self
|
|
.params
|
|
.plugins
|
|
.iter()
|
|
.find(|plugin| plugin.protocol_id() == protocol)
|
|
.cloned()
|
|
else {
|
|
return;
|
|
};
|
|
|
|
let path = link.path_description();
|
|
self.links
|
|
.insert((peer, protocol.clone()), Arc::clone(&link));
|
|
// The protocol is handed the hub's link, not this one: the same
|
|
// object for as long as the peer is a peer, whatever happens to
|
|
// the path under it. A tunnel is not torn down because a direct
|
|
// link came or went.
|
|
let first_time = !self.hub.has_link(peer, &protocol);
|
|
self.hub.set_direct(peer, &protocol, link);
|
|
let shared = self.hub.link(peer, &protocol);
|
|
let max_datagram = shared.max_datagram_size();
|
|
if first_time {
|
|
plugin.on_peer_link(self.network_id, peer, shared);
|
|
}
|
|
self.metrics.data_links_established += 1;
|
|
self.emit(Event::DataLinkUp {
|
|
network: self.network_id,
|
|
peer,
|
|
protocol,
|
|
path,
|
|
max_datagram,
|
|
});
|
|
}
|
|
|
|
/// Drops every link to a peer.
|
|
fn drop_links_for(&mut self, peer: EndpointId) {
|
|
// The peer itself is going, not just a path to it, so the
|
|
// protocol's link goes with it.
|
|
self.hub.remove_peer(peer);
|
|
self.reachable.remove(&peer);
|
|
self.protocol_dial_states.retain(|(p, _), _| *p != peer);
|
|
let keys: Vec<(EndpointId, String)> = self
|
|
.links
|
|
.keys()
|
|
.filter(|(id, _)| *id == peer)
|
|
.cloned()
|
|
.collect();
|
|
for key in keys {
|
|
self.links.remove(&key);
|
|
self.emit(Event::DataLinkDown {
|
|
network: self.network_id,
|
|
peer,
|
|
protocol: key.1,
|
|
reason: "peer session ended".into(),
|
|
});
|
|
}
|
|
}
|
|
|
|
// --------------------------------------------------------------- sessions
|
|
|
|
async fn install_session(&mut self, inbound: InboundSession) {
|
|
let InboundSession {
|
|
conn,
|
|
send,
|
|
recv,
|
|
outcome,
|
|
} = inbound;
|
|
let peer = outcome.peer;
|
|
|
|
if self.sessions.len() >= self.params.limits.max_sessions_per_network
|
|
&& !self.sessions.contains_key(&peer)
|
|
{
|
|
conn.close(1u32.into(), b"session limit reached");
|
|
self.emit(Event::ProtocolViolation {
|
|
network: Some(self.network_id),
|
|
peer: Some(peer),
|
|
reason: "session limit for this network reached".into(),
|
|
});
|
|
return;
|
|
}
|
|
|
|
// Two agents may dial each other at the same time. Both sides apply the
|
|
// same deterministic rule, so they converge on the same session.
|
|
if let Some(existing) = self.sessions.get(&peer) {
|
|
let existing_initiator = initiator_of(existing.role, self.local_id, peer);
|
|
let new_initiator = initiator_of(outcome.role, self.local_id, peer);
|
|
if existing_initiator.as_bytes() <= new_initiator.as_bytes() {
|
|
conn.close(0u32.into(), b"duplicate session");
|
|
return;
|
|
}
|
|
if let Some(old) = self.sessions.remove(&peer) {
|
|
old.abort();
|
|
old.conn
|
|
.close(0u32.into(), b"replaced by preferred session");
|
|
}
|
|
}
|
|
|
|
self.record_hints(&conn).await;
|
|
|
|
let session = session::spawn(
|
|
self.network_id,
|
|
peer,
|
|
outcome.role,
|
|
conn.clone(),
|
|
send,
|
|
recv,
|
|
Arc::clone(&self.params.limits),
|
|
self.session_events_tx.clone(),
|
|
self.shutdown.clone(),
|
|
);
|
|
|
|
let snapshot = snapshot_connection(&conn);
|
|
self.sessions.insert(peer, session);
|
|
if let Some(worker) = &self.discovery_worker {
|
|
worker.set_connected(true);
|
|
}
|
|
self.metrics.sessions_established += 1;
|
|
|
|
// Announce ourselves straight away so the peer learns our hostname and
|
|
// capabilities without another round of discovery.
|
|
let announcement = ControlMessage::Announce(self.local_announcement());
|
|
if let Err(err) = self.send_to(peer, announcement) {
|
|
tracing::debug!(%err, "could not queue initial announcement");
|
|
}
|
|
|
|
self.broadcast_state();
|
|
|
|
self.emit(Event::PeerConnected {
|
|
network: self.network_id,
|
|
peer,
|
|
role: outcome.role,
|
|
transport: snapshot.transport,
|
|
rtt: snapshot.rtt,
|
|
});
|
|
}
|
|
|
|
fn local_announcement(&mut self) -> Announcement {
|
|
let mut capabilities = Vec::new();
|
|
let mut errors = Vec::new();
|
|
for plugin in &self.params.plugins {
|
|
match plugin.local_capability(self.network_id) {
|
|
Ok(Some(capability)) => capabilities.push(capability),
|
|
Ok(None) => {}
|
|
Err(err) => errors.push((plugin.protocol_id().to_string(), err.to_string())),
|
|
}
|
|
}
|
|
for (protocol, reason) in errors {
|
|
self.metrics.plugin_errors += 1;
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol,
|
|
reason,
|
|
});
|
|
}
|
|
capabilities.truncate(self.params.limits.max_capabilities);
|
|
let exit_node = self.exit_ready();
|
|
self.announced_exit = exit_node;
|
|
Announcement {
|
|
broadcast: self.params.broadcast,
|
|
exit_node,
|
|
hostname: self.params.hostname.clone(),
|
|
capabilities,
|
|
}
|
|
}
|
|
|
|
async fn record_hints(&self, conn: &Connection) {
|
|
let snapshot = snapshot_connection(conn);
|
|
let peer_bytes = *snapshot.remote_id.as_bytes();
|
|
for path in &snapshot.paths {
|
|
if let Some(encoded) = encode_hint(&path.remote) {
|
|
self.params
|
|
.storage
|
|
.record_hint(
|
|
self.network_id,
|
|
peer_bytes,
|
|
encoded,
|
|
self.params.limits.max_hints_per_peer,
|
|
)
|
|
.await;
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn handle_session_event(&mut self, event: SessionEvent) {
|
|
match event {
|
|
SessionEvent::Message {
|
|
session_id,
|
|
peer,
|
|
message,
|
|
bytes,
|
|
} => {
|
|
let current = self.sessions.get(&peer).map(|session| session.id);
|
|
if current != Some(session_id) {
|
|
return;
|
|
}
|
|
self.metrics.control_messages_received += 1;
|
|
self.metrics.control_bytes_received += bytes as u64;
|
|
if let Some(session) = self.sessions.get_mut(&peer) {
|
|
session.messages_received += 1;
|
|
session.bytes_received += bytes as u64;
|
|
}
|
|
self.dispatch_message(peer, message);
|
|
}
|
|
SessionEvent::Violation {
|
|
session_id,
|
|
peer,
|
|
error,
|
|
} => {
|
|
let current = self.sessions.get(&peer).map(|session| session.id);
|
|
if current != Some(session_id) {
|
|
return;
|
|
}
|
|
self.metrics.protocol_violations += 1;
|
|
self.emit(Event::ProtocolViolation {
|
|
network: Some(self.network_id),
|
|
peer: Some(peer),
|
|
reason: error.to_string(),
|
|
});
|
|
}
|
|
SessionEvent::Closed {
|
|
session_id,
|
|
peer,
|
|
reason,
|
|
} => {
|
|
let current = self.sessions.get(&peer).map(|session| session.id);
|
|
if current != Some(session_id) {
|
|
return;
|
|
}
|
|
if let Some(session) = self.sessions.remove(&peer) {
|
|
session.abort();
|
|
session.conn.close(0u32.into(), b"session ended");
|
|
}
|
|
if let Some(worker) = &self.discovery_worker {
|
|
worker.set_connected(!self.sessions.is_empty());
|
|
}
|
|
self.metrics.disconnects += 1;
|
|
self.drop_links_for(peer);
|
|
self.update_broadcast();
|
|
self.watch_exit_via();
|
|
self.update_paths();
|
|
self.announce_reach(false);
|
|
for plugin in &self.params.plugins {
|
|
plugin.on_peer_gone(self.network_id, peer);
|
|
}
|
|
// Retry promptly, then back off if it keeps failing.
|
|
let policy = &self.params.reconnect;
|
|
let state = self
|
|
.dial_states
|
|
.entry(peer)
|
|
.or_insert_with(|| DialState::new(CandidateSource::Discovery));
|
|
state.in_flight = false;
|
|
state.next_attempt = Instant::now() + policy.initial_delay;
|
|
self.emit(Event::PeerDisconnected {
|
|
network: self.network_id,
|
|
peer,
|
|
reason,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Keeps the name a peer announced, on disk too when it changed.
|
|
///
|
|
/// Written off the loop: a name that is not stored this time is stored
|
|
/// the next time the peer announces it.
|
|
fn remember_hostname(&mut self, peer: EndpointId, hostname: &str) {
|
|
if hostname.is_empty()
|
|
|| self
|
|
.known_hostnames
|
|
.get(&peer)
|
|
.is_some_and(|h| h == hostname)
|
|
{
|
|
return;
|
|
}
|
|
self.known_hostnames.insert(peer, hostname.to_string());
|
|
let storage = self.params.storage.clone();
|
|
let network_id = self.network_id;
|
|
let hostname = hostname.to_string();
|
|
tokio::spawn(async move {
|
|
if let Err(err) = storage
|
|
.remember_peer_hostname(network_id, *peer.as_bytes(), hostname)
|
|
.await
|
|
{
|
|
tracing::debug!(%err, "could not remember a peer's hostname");
|
|
}
|
|
});
|
|
}
|
|
|
|
fn dispatch_message(&mut self, peer: EndpointId, message: ControlMessage) {
|
|
match &message {
|
|
ControlMessage::Announce(announcement) => {
|
|
let capabilities = announcement.capabilities.clone();
|
|
if let Some(session) = self.sessions.get_mut(&peer) {
|
|
session.hostname = Some(announcement.hostname.clone());
|
|
session.broadcast = announcement.broadcast;
|
|
session.exit_node = announcement.exit_node;
|
|
session.capabilities = capabilities.clone();
|
|
}
|
|
self.remember_hostname(peer, &announcement.hostname);
|
|
self.update_broadcast();
|
|
self.watch_exit_via();
|
|
self.dispatch_capabilities(peer, &capabilities);
|
|
self.ensure_links();
|
|
self.update_paths();
|
|
}
|
|
ControlMessage::Ping { seq, payload } => {
|
|
let pong = ControlMessage::Pong {
|
|
seq: *seq,
|
|
payload: payload.clone(),
|
|
};
|
|
if let Err(err) = self.send_to(peer, pong) {
|
|
tracing::debug!(%err, "could not queue pong");
|
|
}
|
|
}
|
|
ControlMessage::State { records } => {
|
|
self.pending_state.push((peer, records.clone()));
|
|
}
|
|
// Members somebody else knows of. Candidates, nothing more:
|
|
// each still has to pass the handshake, and each is tried the
|
|
// same way as one from any other source.
|
|
ControlMessage::Peers { peers } => {
|
|
let learned = self.learn_peers(peers);
|
|
if learned > 0 {
|
|
self.start_dials();
|
|
}
|
|
}
|
|
// What that peer can reach, from that peer. Kept with the time
|
|
// it was heard, so it can go stale on its own.
|
|
ControlMessage::Reachable { links } => {
|
|
let heard = links.iter().cloned().collect();
|
|
self.reachable
|
|
.insert(peer, (heard, std::time::Instant::now()));
|
|
self.update_paths();
|
|
}
|
|
ControlMessage::Pong { .. } | ControlMessage::Bye { .. } => {}
|
|
}
|
|
|
|
self.emit(Event::MessageReceived {
|
|
network: self.network_id,
|
|
peer,
|
|
message,
|
|
});
|
|
}
|
|
|
|
fn dispatch_capabilities(&mut self, peer: EndpointId, capabilities: &[PluginCapability]) {
|
|
let mut errors = Vec::new();
|
|
for capability in capabilities {
|
|
for plugin in &self.params.plugins {
|
|
if plugin.protocol_id() != capability.protocol {
|
|
continue;
|
|
}
|
|
if plugin.protocol_version() != capability.version {
|
|
// Said once per peer and protocol. A version that will
|
|
// not match this build will not match it on the next
|
|
// announcement either, and repeating it every round
|
|
// would bury everything else.
|
|
let key = (peer, capability.protocol.clone());
|
|
if self.reported_mismatch.insert(key) {
|
|
errors.push((
|
|
capability.protocol.clone(),
|
|
format!(
|
|
"peer speaks {} version {} and this build speaks {}, so there \
|
|
is no data plane with it; the control plane is unaffected",
|
|
capability.protocol,
|
|
capability.version,
|
|
plugin.protocol_version()
|
|
),
|
|
));
|
|
}
|
|
continue;
|
|
}
|
|
// The core hands the opaque payload over without interpreting it.
|
|
if let Err(err) = plugin.on_peer_capability(self.network_id, peer, capability) {
|
|
errors.push((plugin.protocol_id().to_string(), err.to_string()));
|
|
}
|
|
}
|
|
}
|
|
for (protocol, reason) in errors {
|
|
self.metrics.plugin_errors += 1;
|
|
self.emit(Event::PluginError {
|
|
network: self.network_id,
|
|
protocol,
|
|
reason,
|
|
});
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------- status
|
|
|
|
fn status(&self) -> NetworkStatus {
|
|
let served = self.served_protocols();
|
|
let mut peers: Vec<PeerStatus> = self
|
|
.sessions
|
|
.values()
|
|
.map(|session| {
|
|
let snapshot = snapshot_connection(&session.conn);
|
|
PeerStatus {
|
|
broadcast: session.broadcast,
|
|
exit_node: session.exit_node,
|
|
endpoint_id: session.peer,
|
|
role: session.role,
|
|
hostname: session
|
|
.hostname
|
|
.clone()
|
|
.or_else(|| self.known_hostnames.get(&session.peer).cloned()),
|
|
protocols: session
|
|
.capabilities
|
|
.iter()
|
|
.filter(|capability| capability.enabled)
|
|
.filter(|capability| {
|
|
served.iter().any(|(name, ours)| {
|
|
*name == capability.protocol && *ours == capability.version
|
|
})
|
|
})
|
|
.map(|capability| capability.protocol.clone())
|
|
.collect(),
|
|
capabilities: session.capabilities.clone(),
|
|
connected_for: session.established.elapsed(),
|
|
paths: snapshot.paths,
|
|
transport: snapshot.transport,
|
|
rtt: snapshot.rtt,
|
|
connection: snapshot.counters,
|
|
control_messages_sent: session.messages_sent,
|
|
control_messages_received: session.messages_received,
|
|
control_bytes_sent: session.bytes_sent,
|
|
control_bytes_received: session.bytes_received,
|
|
}
|
|
})
|
|
.collect();
|
|
peers.sort_by(|a, b| a.endpoint_id.as_bytes().cmp(b.endpoint_id.as_bytes()));
|
|
|
|
let mut candidates: Vec<CandidateStatus> = self
|
|
.dial_states
|
|
.iter()
|
|
.map(|(peer, state)| CandidateStatus {
|
|
endpoint_id: *peer,
|
|
source: state.source,
|
|
consecutive_failures: state.consecutive_failures,
|
|
})
|
|
.collect();
|
|
candidates.sort_by(|a, b| a.endpoint_id.as_bytes().cmp(b.endpoint_id.as_bytes()));
|
|
|
|
// The durable roster. Every author of a signed record is a member,
|
|
// including this agent and including members that are not here —
|
|
// except one that has released everything. That record is a
|
|
// tombstone kept so the release cannot be undone by a replica that
|
|
// has not heard of it; the author is not a member any more, and
|
|
// listing it as one turns leaving into a peer that looks broken.
|
|
let mut members: Vec<MemberStatus> = self
|
|
.state
|
|
.records()
|
|
.into_iter()
|
|
.filter(|record| !matches!(record.body, RecordBody::Release))
|
|
.filter_map(|record| {
|
|
let endpoint_id = record.author_id().ok()?;
|
|
Some(MemberStatus {
|
|
endpoint_id,
|
|
// Only uncontested claims are reported as held: two
|
|
// members may have claimed the same thing, and saying
|
|
// both hold it would be untrue on one of them.
|
|
overlay_address_v4: self.state.address_of(&endpoint_id),
|
|
hostname: self.state.hostname_of(&endpoint_id).map(str::to_string),
|
|
last_hostname: self.known_hostnames.get(&endpoint_id).cloned(),
|
|
})
|
|
})
|
|
.collect();
|
|
members.sort_by(|a, b| a.endpoint_id.as_bytes().cmp(b.endpoint_id.as_bytes()));
|
|
members.dedup_by(|a, b| a.endpoint_id == b.endpoint_id);
|
|
|
|
NetworkStatus {
|
|
broadcast: self.params.broadcast,
|
|
exit_offer: self.params.exit_offer,
|
|
exit_via: self.params.exit_via,
|
|
exit_via_online: self
|
|
.params
|
|
.exit_via
|
|
.is_some_and(|via| self.exit_node_online(via)),
|
|
exit_rules: self
|
|
.params
|
|
.interface
|
|
.as_ref()
|
|
.map(|interface| interface.exit_rules(self.network_id))
|
|
.unwrap_or_default(),
|
|
descriptor: self.params.keys.descriptor(),
|
|
name: self.params.keys.name().clone(),
|
|
network_id: self.network_id,
|
|
state: NetworkState::Active,
|
|
peers,
|
|
candidates,
|
|
members,
|
|
range: self.effective_range(),
|
|
// Only worth reporting while it is actually stuck: with a
|
|
// fallback in play the configured range being another
|
|
// network's is how it is meant to work, not a fault.
|
|
range_conflict: self
|
|
.params
|
|
.range_conflict
|
|
.filter(|_| self.effective_range().is_none()),
|
|
metrics: self.metrics.clone(),
|
|
relay: self.hub.counters(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Which side dialled, given a role and the two identities.
|
|
fn initiator_of(role: Role, local: EndpointId, peer: EndpointId) -> EndpointId {
|
|
match role {
|
|
Role::Initiator => local,
|
|
Role::Responder => peer,
|
|
}
|
|
}
|
|
|
|
/// Performs one dial and handshake.
|
|
async fn dial(
|
|
adapter: EndpointAdapter,
|
|
addr: iroh::EndpointAddr,
|
|
keys: NetworkKeys,
|
|
limits: Arc<Limits>,
|
|
local_id: EndpointId,
|
|
peer: EndpointId,
|
|
) -> DialOutcome {
|
|
let connect = tokio::time::timeout(limits.dial_timeout, adapter.connect(addr)).await;
|
|
let (conn, mut send, mut recv) = match connect {
|
|
Ok(Ok(parts)) => parts,
|
|
Ok(Err(err)) => {
|
|
return DialOutcome::Failed {
|
|
peer,
|
|
reason: err.to_string(),
|
|
during_handshake: false,
|
|
};
|
|
}
|
|
Err(_) => {
|
|
return DialOutcome::Failed {
|
|
peer,
|
|
reason: "dial timed out".into(),
|
|
during_handshake: false,
|
|
};
|
|
}
|
|
};
|
|
|
|
match handshake::initiate(&conn, &mut send, &mut recv, local_id, &keys, &limits).await {
|
|
Ok(outcome) => DialOutcome::Established(Box::new(InboundSession {
|
|
conn,
|
|
send,
|
|
recv,
|
|
outcome,
|
|
})),
|
|
Err(err) => {
|
|
conn.close(2u32.into(), b"handshake failed");
|
|
DialOutcome::Failed {
|
|
peer,
|
|
reason: err.to_string(),
|
|
during_handshake: true,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Encodes a path address as a cache hint.
|
|
fn encode_hint(addr: &PathAddr) -> Option<String> {
|
|
match addr {
|
|
PathAddr::Ip(socket) => Some(format!("ip:{socket}")),
|
|
PathAddr::Relay(url) => Some(format!("relay:{url}")),
|
|
PathAddr::Other(_) => None,
|
|
}
|
|
}
|
|
|
|
/// The addresses of a candidate, in the spelling hints use.
|
|
///
|
|
/// Bounded and sorted, so what goes on the wire is the same for the same
|
|
/// address whatever order the transport happened to report it in.
|
|
fn hint_addrs(addr: &iroh::EndpointAddr) -> Vec<String> {
|
|
let mut out: Vec<String> = addr
|
|
.addrs
|
|
.iter()
|
|
.map(|transport| transport.to_string())
|
|
.filter(|text| text.len() <= crate::proto::message::MAX_HINT_ADDR_LEN)
|
|
.collect();
|
|
out.sort();
|
|
out.dedup();
|
|
out.truncate(crate::proto::message::MAX_HINT_ADDRS);
|
|
out
|
|
}
|
|
|
|
/// Decodes a cache hint back into an address. Malformed hints are ignored.
|
|
fn decode_hint(endpoint_id: EndpointId, hint: &str) -> Option<iroh::EndpointAddr> {
|
|
if let Some(rest) = hint.strip_prefix("ip:") {
|
|
let socket: std::net::SocketAddr = rest.parse().ok()?;
|
|
return Some(iroh::EndpointAddr::new(endpoint_id).with_ip_addr(socket));
|
|
}
|
|
if let Some(rest) = hint.strip_prefix("relay:") {
|
|
let url: iroh::RelayUrl = rest.parse().ok()?;
|
|
return Some(iroh::EndpointAddr::new(endpoint_id).with_relay_url(url));
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Folds newly learned addresses into a known candidate address.
|
|
fn merge_addr(existing: &mut iroh::EndpointAddr, incoming: &iroh::EndpointAddr) {
|
|
if existing.id != incoming.id {
|
|
*existing = incoming.clone();
|
|
return;
|
|
}
|
|
for addr in &incoming.addrs {
|
|
existing.addrs.insert(addr.clone());
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_data_link_backoff_progression() {
|
|
assert_eq!(data_link_backoff(0), Duration::ZERO);
|
|
assert_eq!(data_link_backoff(1), Duration::from_secs(3));
|
|
assert_eq!(data_link_backoff(2), Duration::from_secs(6));
|
|
assert_eq!(data_link_backoff(3), Duration::from_secs(12));
|
|
assert_eq!(data_link_backoff(4), Duration::from_secs(24));
|
|
assert_eq!(data_link_backoff(5), Duration::from_secs(48));
|
|
assert_eq!(data_link_backoff(6), Duration::from_secs(60));
|
|
assert_eq!(data_link_backoff(10), Duration::from_secs(60));
|
|
}
|
|
|
|
#[test]
|
|
fn test_protocol_dial_state_cooldown_and_fallback_selection() {
|
|
let peer = crate::identity::DeviceIdentity::generate().endpoint_id();
|
|
let supported = ["wg", "tcp-tls", "wg-quic"];
|
|
let mut states: HashMap<(EndpointId, String), ProtocolDialState> = HashMap::new();
|
|
|
|
let now = std::time::Instant::now();
|
|
|
|
// 1. Without cooldown, the highest priority protocol "wg" is chosen
|
|
let chosen =
|
|
supported
|
|
.iter()
|
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
|
Some(state) => state.cooldown_until <= now,
|
|
None => true,
|
|
});
|
|
assert_eq!(chosen.copied(), Some("wg"));
|
|
|
|
// 2. Record failure for "wg" putting it on cooldown
|
|
states.insert(
|
|
(peer, "wg".to_string()),
|
|
ProtocolDialState {
|
|
failures: 1,
|
|
cooldown_until: now + Duration::from_secs(3),
|
|
},
|
|
);
|
|
|
|
// With "wg" on cooldown, selection must fall back to next priority "tcp-tls"
|
|
let chosen_after_failure =
|
|
supported
|
|
.iter()
|
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
|
Some(state) => state.cooldown_until <= now,
|
|
None => true,
|
|
});
|
|
assert_eq!(chosen_after_failure.copied(), Some("tcp-tls"));
|
|
|
|
// 3. If "tcp-tls" also fails, selection falls back to "wg-quic"
|
|
states.insert(
|
|
(peer, "tcp-tls".to_string()),
|
|
ProtocolDialState {
|
|
failures: 1,
|
|
cooldown_until: now + Duration::from_secs(3),
|
|
},
|
|
);
|
|
let chosen_quic =
|
|
supported
|
|
.iter()
|
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
|
Some(state) => state.cooldown_until <= now,
|
|
None => true,
|
|
});
|
|
assert_eq!(chosen_quic.copied(), Some("wg-quic"));
|
|
|
|
// 4. Once success is recorded (entry removed), "wg" is preferred again
|
|
states.remove(&(peer, "wg".to_string()));
|
|
let chosen_restored =
|
|
supported
|
|
.iter()
|
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
|
Some(state) => state.cooldown_until <= now,
|
|
None => true,
|
|
});
|
|
assert_eq!(chosen_restored.copied(), Some("wg"));
|
|
}
|
|
}
|
|
|
|
/// The host name of a relay URL such as `https://relay.example:443/`.
|
|
fn relay_host(url: &str) -> Option<String> {
|
|
let rest = url.split_once("://").map_or(url, |(_, rest)| rest);
|
|
let authority = rest.split(['/', '?', '#']).next()?;
|
|
let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
|
|
let host = if host.starts_with('[') {
|
|
host.split_once(']')?.0.trim_start_matches('[')
|
|
} else {
|
|
host.split(':').next()?
|
|
};
|
|
(!host.is_empty()
|
|
&& host.len() <= 253
|
|
&& host
|
|
.chars()
|
|
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == ':'))
|
|
.then(|| host.to_ascii_lowercase())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod relay_host_tests {
|
|
use super::relay_host;
|
|
|
|
#[test]
|
|
fn the_host_is_taken_out_of_a_relay_url() {
|
|
assert_eq!(
|
|
relay_host("https://relay.example.com./").as_deref(),
|
|
Some("relay.example.com.")
|
|
);
|
|
assert_eq!(
|
|
relay_host("https://Euw1-1.relay.iroh.network:443/x").as_deref(),
|
|
Some("euw1-1.relay.iroh.network")
|
|
);
|
|
assert_eq!(
|
|
relay_host("http://192.0.2.7:3340").as_deref(),
|
|
Some("192.0.2.7")
|
|
);
|
|
assert_eq!(
|
|
relay_host("https://[2001:db8::1]:443/").as_deref(),
|
|
Some("2001:db8::1")
|
|
);
|
|
assert_eq!(relay_host("https:///").as_deref(), None);
|
|
assert_eq!(relay_host("https://bad host/").as_deref(), None);
|
|
}
|
|
}
|