A long-running rc.33 daemon reached OOM with roughly 22 GiB resident and 24 GiB swapped. Live inspection found 40 GiB and 10/20 GiB anonymous container mappings; the active worker stack and tracing metadata identified iroh's scheduled path-open retry loop. In iroh 1.2.0 every failed connection requeues the same path, while each retry visits every connection again. Separate control and data connections multiply the queue geometrically. Vendor the published iroh 1.2.0 source and select it with a workspace Cargo patch. Deduplicate retry admission and cap distinct pending paths at 64, evicting the oldest before inserting a new candidate. Keep the current and next retry batches bounded without combining control/data connections or changing wire protocols, identity derivation, authentication or state. Document source provenance, the live evidence and removal conditions. Upstream PR 4551 merged on October 6, but inspection of the official 1.3.0 crate still shows the unconditional enqueue, so that upgrade alone is not a fix. No version bump, release tag or installed-service change. The default workspace suite includes the exact vendored admission module. Three regressions failed with the original enqueue and pass with the fix: repeated failures across connections, duplicate admission at capacity, and 10,000 distinct candidates with bounded length/capacity and FIFO eviction. Transport integration tests continue using real endpoints and encryption. Checks on Linux in nix develop: - cargo fmt --all -- --check - cargo clippy --locked --workspace --all-targets -- -D warnings - cargo test --locked --workspace --all-targets, with only the inherited TSUNAGI_CONTROL_SOCKET removed from the test environment - git diff --cached --check Initial full-suite runs failed six DNS readiness checks because the inherited system socket path overrode their temporary state directories. Clearing that environment variable resolves those failures without changing application code. Root/network-dependent ignored tests and other operating systems were not run.
160 lines
5.6 KiB
Rust
160 lines
5.6 KiB
Rust
use std::{sync::Arc, time::Duration};
|
|
|
|
use iroh::{
|
|
Endpoint,
|
|
endpoint::{
|
|
AfterHandshakeOutcome, Closed, Connection, EndpointHooks, WeakConnectionHandle, presets,
|
|
},
|
|
};
|
|
use n0_error::{Result, StackResultExt, StdResultExt, ensure_any};
|
|
use n0_future::task::AbortOnDropHandle;
|
|
use tokio::{
|
|
sync::mpsc::{UnboundedReceiver, UnboundedSender},
|
|
task::JoinSet,
|
|
};
|
|
use tracing::{Instrument, info, info_span};
|
|
|
|
const ALPN: &[u8] = b"iroh/test";
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result {
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(
|
|
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
|
|
)
|
|
.init();
|
|
|
|
let monitor = Monitor::new();
|
|
let server = Endpoint::builder(presets::Minimal)
|
|
.alpns(vec![ALPN.to_vec()])
|
|
.hooks(monitor.clone())
|
|
.bind()
|
|
.instrument(info_span!("server"))
|
|
.await?;
|
|
let server_addr = server.addr();
|
|
|
|
let count = 2;
|
|
|
|
let client_task = tokio::spawn(
|
|
async move {
|
|
let client = Endpoint::builder(presets::Minimal)
|
|
.bind()
|
|
.instrument(info_span!("client"))
|
|
.await?;
|
|
for _i in 0..count {
|
|
let conn = client.connect(server_addr.clone(), ALPN).await?;
|
|
let mut s = conn.accept_uni().await.anyerr()?;
|
|
let data = s.read_to_end(2).await.anyerr()?;
|
|
ensure_any!(data == b"hi", "unexpected data");
|
|
conn.close(23u32.into(), b"bye");
|
|
}
|
|
client.close().await;
|
|
n0_error::Ok(client)
|
|
}
|
|
.instrument(info_span!("client")),
|
|
);
|
|
|
|
let server_task = tokio::spawn(
|
|
async move {
|
|
for _i in 0..count {
|
|
let conn = server
|
|
.accept()
|
|
.await
|
|
.context("server endpoint closed")?
|
|
.await?;
|
|
let mut s = conn.open_uni().await.anyerr()?;
|
|
s.write_all(b"hi").await.anyerr()?;
|
|
s.finish().anyerr()?;
|
|
conn.closed().await;
|
|
}
|
|
server.close().await;
|
|
n0_error::Ok(())
|
|
}
|
|
.instrument(info_span!("server")),
|
|
);
|
|
client_task.await.std_context("client")?.context("client")?;
|
|
server_task.await.std_context("server")?.context("server")?;
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
drop(monitor);
|
|
Ok(())
|
|
}
|
|
|
|
/// Our connection monitor impl.
|
|
///
|
|
/// This here only logs connection open and close events via tracing.
|
|
/// It could also maintain a datastructure of all connections, or send the stats to some metrics service.
|
|
#[derive(Clone, Debug)]
|
|
struct Monitor {
|
|
tx: UnboundedSender<MonitoredConnection>,
|
|
_task: Arc<AbortOnDropHandle<()>>,
|
|
}
|
|
|
|
/// Static info captured at handshake time, paired with a weak handle to the connection.
|
|
///
|
|
/// We capture `alpn` and `remote_id` at hook time because [`WeakConnectionHandle`] only
|
|
/// exposes [`upgrade`] and [`closed`], so reading these fields after the connection has
|
|
/// been dropped would otherwise be impossible.
|
|
///
|
|
/// [`upgrade`]: WeakConnectionHandle::upgrade
|
|
/// [`closed`]: WeakConnectionHandle::closed
|
|
#[derive(Debug)]
|
|
struct MonitoredConnection {
|
|
alpn: Vec<u8>,
|
|
remote_id: iroh::EndpointId,
|
|
handle: WeakConnectionHandle,
|
|
}
|
|
|
|
impl EndpointHooks for Monitor {
|
|
async fn after_handshake(&self, conn: &Connection) -> AfterHandshakeOutcome {
|
|
let info = MonitoredConnection {
|
|
alpn: conn.alpn().to_vec(),
|
|
remote_id: conn.remote_id(),
|
|
handle: conn.weak_handle(),
|
|
};
|
|
self.tx.send(info).ok();
|
|
AfterHandshakeOutcome::Accept
|
|
}
|
|
}
|
|
|
|
impl Monitor {
|
|
fn new() -> Self {
|
|
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let task = tokio::spawn(Self::run(rx).instrument(info_span!("watcher")));
|
|
Self {
|
|
tx,
|
|
_task: Arc::new(AbortOnDropHandle::new(task)),
|
|
}
|
|
}
|
|
|
|
async fn run(mut rx: UnboundedReceiver<MonitoredConnection>) {
|
|
let mut tasks = JoinSet::new();
|
|
loop {
|
|
tokio::select! {
|
|
Some(MonitoredConnection { alpn, remote_id, handle }) = rx.recv() => {
|
|
let alpn = String::from_utf8_lossy(&alpn).to_string();
|
|
let remote = remote_id.fmt_short();
|
|
let rtt = handle.upgrade().and_then(|c| c.paths().iter().map(|p| p.rtt()).min());
|
|
info!(%remote, %alpn, ?rtt, "new connection");
|
|
tasks.spawn(async move {
|
|
match handle.closed().await {
|
|
Some(Closed { reason, stats, .. }) => {
|
|
// We have access to the final stats of the connection!
|
|
info!(%remote, %alpn, ?reason, udp_rx=stats.udp_rx.bytes, udp_tx=stats.udp_tx.bytes, "connection closed");
|
|
}
|
|
None => {
|
|
// The connection was closed before we could register our stats-on-close listener.
|
|
info!(%remote, %alpn, "connection closed before tracking started");
|
|
}
|
|
}
|
|
}.instrument(tracing::Span::current()));
|
|
}
|
|
Some(res) = tasks.join_next(), if !tasks.is_empty() => res.expect("conn close task panicked"),
|
|
else => break,
|
|
}
|
|
while let Some(res) = tasks.join_next().await {
|
|
res.expect("conn close task panicked");
|
|
}
|
|
}
|
|
}
|
|
}
|