fix federation recovery after network suspension
CI / check (push) Successful in 1m13s

This commit is contained in:
Ultradesu committed 2026-09-02 15:09:16 +01:00
1 parent 02ef0095f5
commit 95bcc9bac4
12 files changed
+342 -63

No files matched your search

+199 -25
View File
@@ -4,7 +4,7 @@ use std::collections::HashMap;
use std::fmt;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak};
use std::time::Duration;
use std::time::{Duration, Instant};
use iroh::endpoint::{Connection, RecvStream, SendStream, VarInt, presets};
use iroh::protocol::{AcceptError, ProtocolHandler, Router};
@@ -19,6 +19,7 @@ use tracing::{debug, info, warn};
use crate::config::NetworkConfig;
use crate::error::{NetworkError, Result};
use crate::event::{ConnectionDirection, NetworkEvent, NetworkEventReceiver};
use crate::health::{NetworkHealthSnapshot, NetworkHealthState};
use crate::identity;
use crate::protocol::{
ALPN, Handshake, HandshakeAck, HandshakeErrorCode, MAX_HANDSHAKE_FRAME_SIZE,
@@ -50,6 +51,12 @@ const REJECT_LINGER: Duration = Duration::from_secs(3);
const SHUTDOWN_TASK_GRACE: Duration = Duration::from_secs(5);
/// Upper bound of the rendezvous round delay while no peer is connected yet.
const RENDEZVOUS_LONELY_INTERVAL: Duration = Duration::from_secs(15);
/// A stuck Mainline-DHT operation must not stall discovery forever.
const RENDEZVOUS_ROUND_TIMEOUT: Duration = Duration::from_secs(30);
/// Recreate the Mainline-DHT client after this many consecutive failures.
const RENDEZVOUS_REBUILD_AFTER_FAILURES: u32 = 2;
/// Recommend an application-owned full restart after sustained discovery failure.
const RENDEZVOUS_RESTART_AFTER_FAILURES: u32 = 4;
/// Capacity of the queue of accepted-but-not-yet-consumed incoming byte
/// streams, per stream protocol. A full queue delays the handshake ack of
/// further incoming streams (natural backpressure).
@@ -260,6 +267,27 @@ struct StreamAcceptorSlot {
receiver: Option<mpsc::Receiver<ByteStream>>,
}
#[derive(Debug)]
struct RendezvousHealth {
last_success: Option<Instant>,
consecutive_failures: u32,
consecutive_peer_dial_failures: u32,
client_restarts: u64,
last_error: Option<String>,
}
impl RendezvousHealth {
fn new() -> Self {
Self {
last_success: None,
consecutive_failures: 0,
consecutive_peer_dial_failures: 0,
client_restarts: 0,
last_error: None,
}
}
}
fn alpn_display(alpn: &[u8]) -> String {
String::from_utf8_lossy(alpn).into_owned()
}
@@ -276,6 +304,7 @@ struct Shared<M> {
tasks: Mutex<JoinSet<()>>,
next_generation: AtomicU64,
shutting_down: AtomicBool,
rendezvous_health: Mutex<Option<RendezvousHealth>>,
/// Broadcasts the start of the shutdown to long-running background
/// loops so they can stop promptly instead of being aborted.
shutdown_signal: watch::Sender<bool>,
@@ -294,6 +323,97 @@ impl<M: Message> Shared<M> {
}
}
fn record_rendezvous_success(&self) {
if let Some(health) = lock(&self.rendezvous_health).as_mut() {
health.last_success = Some(Instant::now());
health.consecutive_failures = 0;
health.last_error = None;
}
}
fn record_rendezvous_failure(&self, error: String) -> u32 {
let mut guard = lock(&self.rendezvous_health);
let Some(health) = guard.as_mut() else {
return 0;
};
health.consecutive_failures = health.consecutive_failures.saturating_add(1);
health.last_error = Some(error);
health.consecutive_failures
}
fn record_rendezvous_restart(&self) {
if let Some(health) = lock(&self.rendezvous_health).as_mut() {
health.client_restarts = health.client_restarts.saturating_add(1);
}
}
fn record_peer_connection_success(&self) {
if let Some(health) = lock(&self.rendezvous_health).as_mut() {
health.consecutive_peer_dial_failures = 0;
}
}
fn record_rendezvous_peer_dials(&self, candidates: usize, attempts: usize, successes: usize) {
let mut guard = lock(&self.rendezvous_health);
let Some(health) = guard.as_mut() else {
return;
};
if candidates == 0 || successes > 0 {
health.consecutive_peer_dial_failures = 0;
} else if attempts > 0 {
health.consecutive_peer_dial_failures =
health.consecutive_peer_dial_failures.saturating_add(1);
health.last_error = Some(format!(
"failed to connect to all {attempts} peer(s) discovered by rendezvous"
));
}
}
fn health(&self) -> NetworkHealthSnapshot {
let connected_peers = lock(&self.peers).len();
let guard = lock(&self.rendezvous_health);
let Some(health) = guard.as_ref() else {
return NetworkHealthSnapshot {
state: if connected_peers == 0 {
NetworkHealthState::Discovering
} else {
NetworkHealthState::Healthy
},
connected_peers,
rendezvous_enabled: false,
consecutive_rendezvous_failures: 0,
consecutive_peer_dial_failures: 0,
rendezvous_restarts: 0,
last_rendezvous_success_ago: None,
last_rendezvous_error: None,
restart_recommended: false,
};
};
let restart_recommended = connected_peers == 0
&& (health.consecutive_failures >= RENDEZVOUS_RESTART_AFTER_FAILURES
|| health.consecutive_peer_dial_failures >= RENDEZVOUS_RESTART_AFTER_FAILURES);
let state = if restart_recommended {
NetworkHealthState::Degraded
} else if health.consecutive_failures > 0 || health.consecutive_peer_dial_failures > 0 {
NetworkHealthState::Recovering
} else if health.last_success.is_some() {
NetworkHealthState::Healthy
} else {
NetworkHealthState::Discovering
};
NetworkHealthSnapshot {
state,
connected_peers,
rendezvous_enabled: true,
consecutive_rendezvous_failures: health.consecutive_failures,
consecutive_peer_dial_failures: health.consecutive_peer_dial_failures,
rendezvous_restarts: health.client_restarts,
last_rendezvous_success_ago: health.last_success.map(|at| at.elapsed()),
last_rendezvous_error: health.last_error.clone(),
restart_recommended,
}
}
/// Delivers an event to the application.
///
/// The channel is bounded; if it is full this awaits until the
@@ -335,6 +455,7 @@ impl<M: Message> Shared<M> {
generation,
};
let replaced = lock(&self.peers).insert(peer_id, state);
self.record_peer_connection_success();
if let Some(old) = replaced {
debug!(peer = %peer_id, "replacing existing connection");
old.connection
@@ -786,39 +907,92 @@ impl<M: Message> Shared<M> {
/// Failures of a single round or dial are logged and retried on the next
/// round; the loop only ends when the engine shuts down (the task is
/// aborted).
async fn rendezvous_loop<M: Message>(
shared: Arc<Shared<M>>,
client: RendezvousClient,
config: RendezvousConfig,
) {
async fn rendezvous_loop<M: Message>(shared: Arc<Shared<M>>, config: RendezvousConfig) {
let mut shutdown = shared.shutdown_signal.subscribe();
// Give the endpoint a moment to learn its relay and direct addresses so
// the very first published record is already dialable.
let _ = timeout(shared.config.request_timeout, shared.endpoint.online()).await;
let self_id = shared.endpoint.id();
let mut client = None;
loop {
if shared.is_shutting_down() {
return;
}
if client.is_none() {
match RendezvousClient::new(shared.config.network_id, &config) {
Ok(new_client) => client = Some(new_client),
Err(err) => {
let failures = shared.record_rendezvous_failure(err.to_string());
warn!(error = %err, failures, "failed to start peer rendezvous; retrying");
}
}
}
let round = async {
let Some(active_client) = client.as_ref() else {
return;
};
let addr = shared.endpoint.addr();
let self_addr = (!addr.is_empty()).then_some(addr);
match client.round(self_addr, now_ms()).await {
Ok(peers) => {
for peer_addr in peers {
if self_addr.is_none() {
let failures = shared.record_rendezvous_failure(
"local endpoint has no dialable address".to_string(),
);
debug!(failures, "rendezvous waiting for a dialable local address");
return;
}
match timeout(
RENDEZVOUS_ROUND_TIMEOUT,
active_client.round(self_addr, now_ms()),
)
.await
{
Ok(Ok(round)) => {
let candidates = round.peers.len();
let mut attempts = 0;
let mut successes = 0;
for peer_addr in round.peers {
let peer = peer_addr.id;
if peer == self_id || lock(&shared.peers).contains_key(&peer) {
continue;
}
debug!(peer = %peer, "rendezvous discovered a peer; connecting");
if let Err(err) = shared.connect_to_addr(peer_addr).await {
// Stale entries (peers that left) fail here; they
// age out of the record by TTL.
debug!(peer = %peer, error = %err, "rendezvous connect attempt failed");
attempts += 1;
match shared.connect_to_addr(peer_addr).await {
Ok(_) => successes += 1,
Err(err) => {
// Stale entries (peers that left) fail here;
// they age out of the record by TTL.
debug!(peer = %peer, error = %err, "rendezvous connect attempt failed");
}
}
}
if let Some(error) = round.publish_error {
let failures = shared.record_rendezvous_failure(error.clone());
debug!(%error, failures, "rendezvous publish failed");
} else {
shared.record_rendezvous_success();
}
shared.record_rendezvous_peer_dials(candidates, attempts, successes);
}
Err(err) => debug!(error = %err, "rendezvous round failed"),
Ok(Err(err)) => {
let failures = shared.record_rendezvous_failure(err.to_string());
debug!(error = %err, failures, "rendezvous round failed");
}
Err(_) => {
let failures = shared.record_rendezvous_failure(format!(
"rendezvous round timed out after {}s",
RENDEZVOUS_ROUND_TIMEOUT.as_secs()
));
warn!(failures, "rendezvous round timed out");
}
}
let failures = lock(&shared.rendezvous_health)
.as_ref()
.map_or(0, |health| health.consecutive_failures);
if failures >= RENDEZVOUS_REBUILD_AFTER_FAILURES {
client = None;
shared.record_rendezvous_restart();
info!(failures, "rebuilding the rendezvous client");
}
};
tokio::select! {
@@ -960,6 +1134,7 @@ impl<M: Message> NetworkEngine<M> {
config: NetworkConfig,
secret_key: SecretKey,
) -> Result<(Self, NetworkEventReceiver<M>)> {
let rendezvous_enabled = config.rendezvous.is_some();
let endpoint = Endpoint::builder(presets::N0)
.secret_key(secret_key)
.bind()
@@ -992,6 +1167,7 @@ impl<M: Message> NetworkEngine<M> {
tasks: Mutex::new(JoinSet::new()),
next_generation: AtomicU64::new(0),
shutting_down: AtomicBool::new(false),
rendezvous_health: Mutex::new(rendezvous_enabled.then(RendezvousHealth::new)),
shutdown_signal: watch::Sender::new(false),
});
let handler = FederationProtocol {
@@ -1015,17 +1191,10 @@ impl<M: Message> NetworkEngine<M> {
let router = router_builder.spawn();
*lock(&shared.router) = Some(router);
if let Some(rendezvous) = shared.config.rendezvous.clone() {
match RendezvousClient::new(shared.config.network_id, &rendezvous) {
Ok(client) => {
let loop_shared = shared.clone();
shared.spawn_task(async move {
rendezvous_loop(loop_shared, client, rendezvous).await;
});
}
// Rendezvous is a convenience; the engine stays usable via
// tickets even when the DHT client cannot start.
Err(err) => warn!(error = %err, "peer rendezvous disabled"),
}
let loop_shared = shared.clone();
shared.spawn_task(async move {
rendezvous_loop(loop_shared, rendezvous).await;
});
}
info!(
endpoint_id = %endpoint_id,
@@ -1257,6 +1426,11 @@ impl<M: Message> NetworkEngine<M> {
lock(&self.shared.peers).keys().copied().collect()
}
/// Returns a point-in-time snapshot of transport and rendezvous health.
pub fn health(&self) -> NetworkHealthSnapshot {
self.shared.health()
}
/// Returns `true` if there is an active connection to `peer`.
pub fn is_connected(&self, peer: EndpointId) -> bool {
lock(&self.shared.peers).contains_key(&peer)