A long-running rc.33 daemon reached OOM with roughly 22 GiB resident and 24 GiB swapped. Live inspection found 40 GiB and 10/20 GiB anonymous container mappings; the active worker stack and tracing metadata identified iroh's scheduled path-open retry loop. In iroh 1.2.0 every failed connection requeues the same path, while each retry visits every connection again. Separate control and data connections multiply the queue geometrically. Vendor the published iroh 1.2.0 source and select it with a workspace Cargo patch. Deduplicate retry admission and cap distinct pending paths at 64, evicting the oldest before inserting a new candidate. Keep the current and next retry batches bounded without combining control/data connections or changing wire protocols, identity derivation, authentication or state. Document source provenance, the live evidence and removal conditions. Upstream PR 4551 merged on October 6, but inspection of the official 1.3.0 crate still shows the unconditional enqueue, so that upgrade alone is not a fix. No version bump, release tag or installed-service change. The default workspace suite includes the exact vendored admission module. Three regressions failed with the original enqueue and pass with the fix: repeated failures across connections, duplicate admission at capacity, and 10,000 distinct candidates with bounded length/capacity and FIFO eviction. Transport integration tests continue using real endpoints and encryption. Checks on Linux in nix develop: - cargo fmt --all -- --check - cargo clippy --locked --workspace --all-targets -- -D warnings - cargo test --locked --workspace --all-targets, with only the inherited TSUNAGI_CONTROL_SOCKET removed from the test environment - git diff --cached --check Initial full-suite runs failed six DNS readiness checks because the inherited system socket path overrode their temporary state directories. Clearing that environment variable resolves those failures without changing application code. Root/network-dependent ignored tests and other operating systems were not run.
182 lines
6.2 KiB
Rust
182 lines
6.2 KiB
Rust
use std::{env, str::FromStr, time::Instant};
|
|
|
|
use clap::Parser;
|
|
use data_encoding::HEXLOWER;
|
|
use iroh::{
|
|
EndpointId, SecretKey,
|
|
endpoint::{RecvStream, SendStream, ZeroRttStatus, presets},
|
|
};
|
|
use n0_error::{Result, StackResultExt, StdResultExt};
|
|
use n0_future::StreamExt;
|
|
use tracing::{info, trace};
|
|
|
|
const PINGPONG_ALPN: &[u8] = b"0rtt-pingpong";
|
|
|
|
#[derive(Parser)]
|
|
struct Args {
|
|
/// The endpoint id to connect to. If not set, the program will start a server.
|
|
endpoint_id: Option<EndpointId>,
|
|
/// Number of rounds to run.
|
|
#[clap(long, default_value = "100")]
|
|
rounds: u64,
|
|
/// Run without 0-RTT for comparison.
|
|
#[clap(long)]
|
|
disable_0rtt: bool,
|
|
}
|
|
|
|
/// Gets a secret key from the IROH_SECRET environment variable or generates a new random one.
|
|
/// If the environment variable is set, it must be a valid string representation of a secret key.
|
|
pub fn get_or_generate_secret_key() -> Result<SecretKey> {
|
|
if let Ok(secret) = env::var("IROH_SECRET") {
|
|
// Parse the secret key from string
|
|
SecretKey::from_str(&secret).context("Invalid secret key format")
|
|
} else {
|
|
// Generate a new random key
|
|
let secret_key = SecretKey::generate();
|
|
println!(
|
|
"Generated new secret key: {}",
|
|
HEXLOWER.encode(&secret_key.to_bytes())
|
|
);
|
|
println!("To reuse this key, set the IROH_SECRET environment variable to this value");
|
|
Ok(secret_key)
|
|
}
|
|
}
|
|
|
|
/// Do a simple ping-pong with the given connection.
|
|
async fn pingpong(send: SendStream, recv: RecvStream, x: u64) -> Result<()> {
|
|
ping(send, x).await?;
|
|
pong(recv, x).await
|
|
}
|
|
|
|
async fn ping(mut send: SendStream, x: u64) -> Result<()> {
|
|
let data = x.to_be_bytes();
|
|
send.write_all(&data).await.anyerr()?;
|
|
send.finish().anyerr()
|
|
}
|
|
|
|
async fn pong(mut recv: RecvStream, x: u64) -> Result<()> {
|
|
let data = x.to_be_bytes();
|
|
let echo = recv.read_to_end(8).await.anyerr()?;
|
|
assert!(echo == data);
|
|
Ok(())
|
|
}
|
|
|
|
async fn connect(args: Args) -> Result<()> {
|
|
let remote_id = args.endpoint_id.unwrap();
|
|
let endpoint = iroh::Endpoint::builder(presets::N0)
|
|
.relay_mode(iroh::RelayMode::Disabled)
|
|
.keylog(true)
|
|
.bind()
|
|
.await?;
|
|
// ensure we have resolved the remote_id before connecting
|
|
// so we get a more accurate connection timing
|
|
let mut address_lookup_stream = endpoint.address_lookup()?.resolve(remote_id);
|
|
let _item = address_lookup_stream
|
|
.next()
|
|
.await
|
|
.context("failed to lookup remote")?;
|
|
|
|
let t0 = Instant::now();
|
|
for i in 0..args.rounds {
|
|
let t0 = Instant::now();
|
|
let connecting = endpoint
|
|
.connect_with_opts(remote_id, PINGPONG_ALPN, Default::default())
|
|
.await?;
|
|
let connection = if args.disable_0rtt {
|
|
let connection = connecting.await.anyerr()?;
|
|
trace!("connecting without 0-RTT");
|
|
let (send, recv) = connection.open_bi().await.anyerr()?;
|
|
pingpong(send, recv, i).await?;
|
|
connection
|
|
} else {
|
|
match connecting.into_0rtt() {
|
|
Ok(zrtt_connection) => {
|
|
trace!("0-RTT possible from our side");
|
|
let (send, recv) = zrtt_connection.open_bi().await.anyerr()?;
|
|
// before we get the full handshake, attempt to send 0-RTT data
|
|
let zrtt_task = tokio::spawn(ping(send, i));
|
|
match zrtt_connection.handshake_completed().await? {
|
|
ZeroRttStatus::Accepted(conn) => {
|
|
let _ = zrtt_task.await.anyerr()?;
|
|
pong(recv, i).await?;
|
|
conn
|
|
}
|
|
ZeroRttStatus::Rejected(conn) => {
|
|
zrtt_task.abort();
|
|
let (send, recv) = conn.open_bi().await.anyerr()?;
|
|
pingpong(send, recv, i).await?;
|
|
conn
|
|
}
|
|
}
|
|
}
|
|
Err(connecting) => {
|
|
trace!("0-RTT not possible from our side");
|
|
let conn = connecting.await.anyerr()?;
|
|
let (send, recv) = conn.open_bi().await.anyerr()?;
|
|
pingpong(send, recv, i).await?;
|
|
conn
|
|
}
|
|
}
|
|
};
|
|
connection.close(0u8.into(), b"");
|
|
let elapsed = t0.elapsed();
|
|
println!("round {i}: {} us", elapsed.as_micros());
|
|
}
|
|
let elapsed = t0.elapsed();
|
|
println!("total time: {} us", elapsed.as_micros());
|
|
println!(
|
|
"time per round: {} us",
|
|
elapsed.as_micros() / (args.rounds as u128)
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
async fn accept(_args: Args) -> Result<()> {
|
|
let secret_key = get_or_generate_secret_key()?;
|
|
let endpoint = iroh::Endpoint::builder(presets::N0)
|
|
.alpns(vec![PINGPONG_ALPN.to_vec()])
|
|
.secret_key(secret_key)
|
|
.relay_mode(iroh::RelayMode::Disabled)
|
|
.bind()
|
|
.await?;
|
|
println!("endpoint id: {}", endpoint.id());
|
|
|
|
let accept = async move {
|
|
while let Some(incoming) = endpoint.accept().await {
|
|
tokio::spawn(async move {
|
|
let accepting = incoming.accept().anyerr()?;
|
|
let connection = accepting.into_0rtt();
|
|
let (mut send, mut recv) = connection.accept_bi().await.anyerr()?;
|
|
trace!("recv.is_0rtt: {}", recv.is_0rtt());
|
|
let data = recv.read_to_end(8).await.anyerr()?;
|
|
trace!("recv: {}", data.len());
|
|
send.write_all(&data).await.anyerr()?;
|
|
send.finish().anyerr()?;
|
|
connection.closed().await;
|
|
n0_error::Ok(())
|
|
});
|
|
}
|
|
};
|
|
tokio::select! {
|
|
_ = accept => {
|
|
info!("accept finished, shutting down");
|
|
},
|
|
_ = tokio::signal::ctrl_c()=> {
|
|
info!("Ctrl-C received, shutting down");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
tracing_subscriber::fmt::init();
|
|
let args = Args::parse();
|
|
if args.endpoint_id.is_some() {
|
|
connect(args).await?;
|
|
} else {
|
|
accept(args).await?;
|
|
};
|
|
Ok(())
|
|
}
|