Detect WireGuard tunnel rekey failure and trigger automatic fallback
Previously, because UDP is connectionless, when a pure WireGuard tunnel
encountered handshake or rekey failures (e.g. boringtun logging
`WARN HANDSHAKE(REKEY_TIMEOUT)` and `CONNECTION_EXPIRED`), the
`WireguardLink` never closed. It remained active in `agent.links` and
as a direct hop in `RelayHub.raw`. Consequently, outgoing packets
continued to be blackholed over the dead direct path, preventing
`RelayHub` from using working multihop relay routes and preventing
the agent from falling back to alternative protocols (`tcp-tls` or
`wg-quic`).
Key changes:
- In `crates/tsunagi-wireguard/src/device.rs`:
- In `FlowTunnel`, track active handshake and rekey attempts via
`handshake_started_at`. If an active handshake attempt remains
unanswered beyond `HANDSHAKE_TIMEOUT` (15s) or boringtun reports
`ConnectionExpired`, declare the tunnel dead.
- Reset timeout timers upon successful handshake reception in
`decapsulate`.
- In `drive_timers`, when tunnel failure is detected, trigger
`inner.on_tunnel_failed(peer)`, which closes the direct transport
link and notifies the agent via `PluginContext::report_peer_protocol_failure`.
- In `crates/tsunagi-wireguard/src/transport.rs`:
- Add `WireguardLink::close` to transition `is_closed` to true and
notify `closed_notify`.
- In `WireguardLink::recv`, select on `closed_notify` so readers
unblock and return `None` immediately on close.
- Add `WireguardTransport::close_link` to close active links by
network and peer.
- In `crates/tsunagi/src/dataplane/mod.rs`:
- Add `PluginRequest::ProtocolFailure` and public helper
`PluginContext::report_peer_protocol_failure`.
- In `crates/tsunagi/src/agent/mod.rs` and `crates/tsunagi/src/agent/network.rs`:
- Add `NetCommand::ProtocolFailure` to runtime command handling.
- On protocol failure: remove dead direct link from `self.links`,
clear it from `RelayHub.raw` (so `RelayHub` immediately routes
over available multihop relays), record failure on `wg` for that
peer with exponential backoff cooldown, emit `DataLinkDown`, and
immediately call `ensure_links()` to dial the next mutually supported
protocol (`tcp-tls` or `wg-quic`).
- Tests:
- Add `test_flow_tunnel_rekey_timeout_triggers_failure` and
`test_flow_tunnel_reset_on_handshake` unit tests in `device.rs`.
- Add `test_wireguard_link_close_and_transport_close_link` in
`transport_tests.rs`.
- Add `test_protocol_dial_state_cooldown_and_fallback_selection` in
`network.rs`.
Validation on Linux:
- cargo fmt --all -- --check passed.
- cargo clippy --locked --workspace --all-targets -- -D warnings passed.
- cargo test --locked --workspace --all-targets passed (350+ tests passed).
- Built release binary: target/release/tsunagi.
This commit is contained in:
1 parent
35acc4d231
commit
3930dff2dc
7 files changed
+422
-14
No files matched your search
@@ -14,20 +14,28 @@ use bytes::Bytes;
|
|||||||
use iroh::EndpointId;
|
use iroh::EndpointId;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
use tsunagi::dataplane::PacketSink;
|
|
||||||
use tsunagi::dataplane::PluginError;
|
|
||||||
use tsunagi::dataplane::routing::{FlowId, flow::ip_flow};
|
use tsunagi::dataplane::routing::{FlowId, flow::ip_flow};
|
||||||
use tsunagi::dataplane::transport::{SharedLink, TransportError};
|
use tsunagi::dataplane::transport::{SharedLink, TransportError};
|
||||||
|
use tsunagi::dataplane::{PacketSink, PluginContext, PluginError};
|
||||||
use tsunagi::identity::NetworkId;
|
use tsunagi::identity::NetworkId;
|
||||||
|
|
||||||
use crate::keys::{WgPublicKey, WgSecretKey};
|
use crate::keys::{WgPublicKey, WgSecretKey};
|
||||||
|
use crate::transport::WireguardTransport;
|
||||||
|
|
||||||
const TIMER_INTERVAL: Duration = Duration::from_millis(250);
|
const TIMER_INTERVAL: Duration = Duration::from_millis(250);
|
||||||
const SCRATCH: usize = 4096;
|
const SCRATCH: usize = 4096;
|
||||||
|
|
||||||
|
/// Maximum consecutive rekey timeout retransmissions before declaring a tunnel dead.
|
||||||
|
pub const MAX_REKEY_TIMEOUTS: u32 = 3;
|
||||||
|
|
||||||
|
/// Overall maximum duration trying to establish or rekey a handshake before declaring it dead.
|
||||||
|
pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(15);
|
||||||
|
|
||||||
struct FlowTunnel {
|
struct FlowTunnel {
|
||||||
tunn: Tunn,
|
tunn: Tunn,
|
||||||
queued: VecDeque<FlowId>,
|
queued: VecDeque<FlowId>,
|
||||||
|
handshake_started_at: Option<std::time::Instant>,
|
||||||
|
rekey_timeouts: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_data(result: &TunnResult<'_>) -> bool {
|
fn is_data(result: &TunnResult<'_>) -> bool {
|
||||||
@@ -45,6 +53,12 @@ impl FlowTunnel {
|
|||||||
if is_data(&result) {
|
if is_data(&result) {
|
||||||
return (result, flow);
|
return (result, flow);
|
||||||
}
|
}
|
||||||
|
if matches!(result, TunnResult::WriteToNetwork(ref b) if !b.is_empty() && b[0] == 1)
|
||||||
|
&& self.handshake_started_at.is_none()
|
||||||
|
{
|
||||||
|
self.handshake_started_at = Some(std::time::Instant::now());
|
||||||
|
self.rekey_timeouts = 0;
|
||||||
|
}
|
||||||
if self.queued.len() < 256 {
|
if self.queued.len() < 256 {
|
||||||
self.queued.push_back(flow);
|
self.queued.push_back(flow);
|
||||||
}
|
}
|
||||||
@@ -57,6 +71,10 @@ impl FlowTunnel {
|
|||||||
scratch: &'a mut [u8],
|
scratch: &'a mut [u8],
|
||||||
) -> (TunnResult<'a>, FlowId) {
|
) -> (TunnResult<'a>, FlowId) {
|
||||||
let result = self.tunn.decapsulate(None, packet, scratch);
|
let result = self.tunn.decapsulate(None, packet, scratch);
|
||||||
|
if self.tunn.time_since_last_handshake().is_some() {
|
||||||
|
self.handshake_started_at = None;
|
||||||
|
self.rekey_timeouts = 0;
|
||||||
|
}
|
||||||
let mut flow = 0;
|
let mut flow = 0;
|
||||||
if packet.is_empty() {
|
if packet.is_empty() {
|
||||||
if is_data(&result) {
|
if is_data(&result) {
|
||||||
@@ -68,15 +86,37 @@ impl FlowTunnel {
|
|||||||
(result, flow)
|
(result, flow)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn update_timers<'a>(&mut self, scratch: &'a mut [u8]) -> TunnResult<'a> {
|
fn update_timers<'a>(&mut self, scratch: &'a mut [u8]) -> (TunnResult<'a>, bool) {
|
||||||
let result = self.tunn.update_timers(scratch);
|
let result = self.tunn.update_timers(scratch);
|
||||||
if matches!(
|
if matches!(
|
||||||
result,
|
result,
|
||||||
TunnResult::Err(boringtun::noise::errors::WireGuardError::ConnectionExpired)
|
TunnResult::Err(boringtun::noise::errors::WireGuardError::ConnectionExpired)
|
||||||
) {
|
) || self.tunn.is_expired()
|
||||||
|
{
|
||||||
self.queued.clear();
|
self.queued.clear();
|
||||||
|
self.handshake_started_at = None;
|
||||||
|
self.rekey_timeouts = 0;
|
||||||
|
return (result, true);
|
||||||
}
|
}
|
||||||
result
|
|
||||||
|
if let TunnResult::WriteToNetwork(ref bytes) = result
|
||||||
|
&& !bytes.is_empty()
|
||||||
|
&& bytes[0] == 1
|
||||||
|
&& self.handshake_started_at.is_none()
|
||||||
|
{
|
||||||
|
self.handshake_started_at = Some(std::time::Instant::now());
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(started) = self.handshake_started_at
|
||||||
|
&& started.elapsed() >= HANDSHAKE_TIMEOUT
|
||||||
|
{
|
||||||
|
self.queued.clear();
|
||||||
|
self.handshake_started_at = None;
|
||||||
|
self.rekey_timeouts = 0;
|
||||||
|
return (result, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
(result, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,9 +245,23 @@ struct Inner {
|
|||||||
private_key: WgSecretKey,
|
private_key: WgSecretKey,
|
||||||
peers: RwLock<HashMap<WgPublicKey, Arc<Peer>>>,
|
peers: RwLock<HashMap<WgPublicKey, Arc<Peer>>>,
|
||||||
sink: Arc<dyn PacketSink>,
|
sink: Arc<dyn PacketSink>,
|
||||||
|
transport: Arc<WireguardTransport>,
|
||||||
|
context: PluginContext,
|
||||||
next_index: AtomicU32,
|
next_index: AtomicU32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Inner {
|
||||||
|
fn on_tunnel_failed(&self, peer: EndpointId) {
|
||||||
|
self.transport.close_link(self.network, peer);
|
||||||
|
self.context.report_peer_protocol_failure(
|
||||||
|
self.network,
|
||||||
|
peer,
|
||||||
|
crate::announcement::WG_PROTOCOL,
|
||||||
|
"WireGuard rekey timeout",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for Inner {
|
impl std::fmt::Debug for Inner {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("Inner")
|
f.debug_struct("Inner")
|
||||||
@@ -225,12 +279,20 @@ pub struct WireguardDevice {
|
|||||||
|
|
||||||
impl WireguardDevice {
|
impl WireguardDevice {
|
||||||
/// Starts the WireGuard tunnels for one network.
|
/// Starts the WireGuard tunnels for one network.
|
||||||
pub fn start(network: NetworkId, private_key: WgSecretKey, sink: Arc<dyn PacketSink>) -> Self {
|
pub fn start(
|
||||||
|
network: NetworkId,
|
||||||
|
private_key: WgSecretKey,
|
||||||
|
sink: Arc<dyn PacketSink>,
|
||||||
|
transport: Arc<WireguardTransport>,
|
||||||
|
context: PluginContext,
|
||||||
|
) -> Self {
|
||||||
let inner = Arc::new(Inner {
|
let inner = Arc::new(Inner {
|
||||||
network,
|
network,
|
||||||
private_key,
|
private_key,
|
||||||
peers: RwLock::new(HashMap::new()),
|
peers: RwLock::new(HashMap::new()),
|
||||||
sink,
|
sink,
|
||||||
|
transport,
|
||||||
|
context,
|
||||||
next_index: AtomicU32::new(1),
|
next_index: AtomicU32::new(1),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -312,6 +374,8 @@ impl WireguardDevice {
|
|||||||
tunn: Mutex::new(FlowTunnel {
|
tunn: Mutex::new(FlowTunnel {
|
||||||
tunn,
|
tunn,
|
||||||
queued: VecDeque::new(),
|
queued: VecDeque::new(),
|
||||||
|
handshake_started_at: None,
|
||||||
|
rekey_timeouts: 0,
|
||||||
}),
|
}),
|
||||||
link,
|
link,
|
||||||
counters: Arc::new(PeerCounters::default()),
|
counters: Arc::new(PeerCounters::default()),
|
||||||
@@ -405,7 +469,11 @@ fn kick_handshake(peer: &Peer) {
|
|||||||
Err(poisoned) => poisoned.into_inner(),
|
Err(poisoned) => poisoned.into_inner(),
|
||||||
};
|
};
|
||||||
match tunn.tunn.format_handshake_initiation(&mut scratch, false) {
|
match tunn.tunn.format_handshake_initiation(&mut scratch, false) {
|
||||||
TunnResult::WriteToNetwork(out) => Some(out.len()),
|
TunnResult::WriteToNetwork(out) => {
|
||||||
|
tunn.handshake_started_at = Some(std::time::Instant::now());
|
||||||
|
tunn.rekey_timeouts = 0;
|
||||||
|
Some(out.len())
|
||||||
|
}
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -505,23 +573,92 @@ async fn drive_timers(inner: Arc<Inner>) {
|
|||||||
let peers: Vec<Arc<Peer>> = read_lock(&inner.peers).values().cloned().collect();
|
let peers: Vec<Arc<Peer>> = read_lock(&inner.peers).values().cloned().collect();
|
||||||
for peer in peers {
|
for peer in peers {
|
||||||
let mut scratch = vec![0u8; SCRATCH];
|
let mut scratch = vec![0u8; SCRATCH];
|
||||||
let len = {
|
let (len, failed) = {
|
||||||
let mut tunn = match peer.tunn.lock() {
|
let mut tunn = match peer.tunn.lock() {
|
||||||
Ok(guard) => guard,
|
Ok(guard) => guard,
|
||||||
Err(poisoned) => poisoned.into_inner(),
|
Err(poisoned) => poisoned.into_inner(),
|
||||||
};
|
};
|
||||||
match tunn.update_timers(&mut scratch) {
|
let (result, failed) = tunn.update_timers(&mut scratch);
|
||||||
|
let len = match result {
|
||||||
TunnResult::WriteToNetwork(out) => Some(out.len()),
|
TunnResult::WriteToNetwork(out) => Some(out.len()),
|
||||||
TunnResult::Err(err) => {
|
TunnResult::Err(err) => {
|
||||||
tracing::trace!(?err, "WireGuard timer error");
|
tracing::trace!(?err, "WireGuard timer error");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
};
|
||||||
|
(len, failed)
|
||||||
};
|
};
|
||||||
if let Some(len) = len {
|
if let Some(len) = len {
|
||||||
send_to_peer(&peer, &scratch[..len]);
|
send_to_peer(&peer, &scratch[..len]);
|
||||||
}
|
}
|
||||||
|
if failed {
|
||||||
|
tracing::warn!(
|
||||||
|
peer = %peer.endpoint_id.fmt_short(),
|
||||||
|
"WireGuard tunnel failed due to rekey timeout, triggering fallback"
|
||||||
|
);
|
||||||
|
inner.on_tunnel_failed(peer.endpoint_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_flow_tunnel_rekey_timeout_triggers_failure() {
|
||||||
|
let my_secret = WgSecretKey::generate();
|
||||||
|
let their_secret = WgSecretKey::generate();
|
||||||
|
let their_public = their_secret.public();
|
||||||
|
let tunn = Tunn::new(
|
||||||
|
my_secret.to_static_secret(),
|
||||||
|
their_public.into_x25519(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
1,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut tunnel = FlowTunnel {
|
||||||
|
tunn,
|
||||||
|
queued: VecDeque::new(),
|
||||||
|
handshake_started_at: Some(std::time::Instant::now() - Duration::from_secs(16)),
|
||||||
|
rekey_timeouts: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut scratch = vec![0u8; SCRATCH];
|
||||||
|
// When handshake timeout exceeds HANDSHAKE_TIMEOUT (15s), update_timers returns true (failed)
|
||||||
|
let (_, failed) = tunnel.update_timers(&mut scratch);
|
||||||
|
assert!(failed, "handshake timeout should report failure");
|
||||||
|
assert!(tunnel.handshake_started_at.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_flow_tunnel_reset_on_handshake() {
|
||||||
|
let my_secret = WgSecretKey::generate();
|
||||||
|
let their_secret = WgSecretKey::generate();
|
||||||
|
let their_public = their_secret.public();
|
||||||
|
let tunn = Tunn::new(
|
||||||
|
my_secret.to_static_secret(),
|
||||||
|
their_public.into_x25519(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
1,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut tunnel = FlowTunnel {
|
||||||
|
tunn,
|
||||||
|
queued: VecDeque::new(),
|
||||||
|
handshake_started_at: Some(std::time::Instant::now()),
|
||||||
|
rekey_timeouts: 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut scratch = vec![0u8; SCRATCH];
|
||||||
|
let _ = tunnel.decapsulate(&[], &mut scratch);
|
||||||
|
// decapsulate without handshake response doesn't establish session
|
||||||
|
assert_eq!(tunnel.rekey_timeouts, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -576,7 +576,18 @@ impl Worker {
|
|||||||
Some(sink) => Arc::clone(sink),
|
Some(sink) => Arc::clone(sink),
|
||||||
None => Arc::new(tsunagi::dataplane::DiscardPackets) as Arc<dyn PacketSink>,
|
None => Arc::new(tsunagi::dataplane::DiscardPackets) as Arc<dyn PacketSink>,
|
||||||
};
|
};
|
||||||
let device = Arc::new(WireguardDevice::start(network, key, sink));
|
let context = self
|
||||||
|
.context
|
||||||
|
.get()
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(PluginContext::detached);
|
||||||
|
let device = Arc::new(WireguardDevice::start(
|
||||||
|
network,
|
||||||
|
key,
|
||||||
|
sink,
|
||||||
|
Arc::clone(&self.transport),
|
||||||
|
context,
|
||||||
|
));
|
||||||
|
|
||||||
let mut shared = self.lock_shared();
|
let mut shared = self.lock_shared();
|
||||||
if let Some(state) = shared.networks.get_mut(&network) {
|
if let Some(state) = shared.networks.get_mut(&network) {
|
||||||
|
|||||||
@@ -222,6 +222,13 @@ impl WireguardLink {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Explicitly closes this link, notifying any pending readers or waiters.
|
||||||
|
pub fn close(&self) {
|
||||||
|
if !self.is_closed.swap(true, Ordering::AcqRel) {
|
||||||
|
self.closed_notify.notify_waiters();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns the current target address.
|
/// Returns the current target address.
|
||||||
pub fn target_addr(&self) -> SocketAddr {
|
pub fn target_addr(&self) -> SocketAddr {
|
||||||
match self.target_addr.read() {
|
match self.target_addr.read() {
|
||||||
@@ -261,8 +268,17 @@ impl PacketLink for WireguardLink {
|
|||||||
|
|
||||||
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
|
fn recv(&self) -> BoxFuture<'_, Option<Bytes>> {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
let mut guard = self.rx.lock().await;
|
if self.is_closed.load(Ordering::Acquire) {
|
||||||
guard.recv().await
|
return None;
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = self.closed_notify.notified() => None,
|
||||||
|
msg = async {
|
||||||
|
let mut guard = self.rx.lock().await;
|
||||||
|
guard.recv().await
|
||||||
|
} => msg,
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -388,6 +404,20 @@ impl WireguardTransport {
|
|||||||
guard.get(&peer).cloned().unwrap_or_default()
|
guard.get(&peer).cloned().unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Explicitly closes any active direct link for the given network and peer.
|
||||||
|
pub fn close_link(&self, network: NetworkId, peer: EndpointId) {
|
||||||
|
let link = {
|
||||||
|
let guard = match self.links.read() {
|
||||||
|
Ok(g) => g,
|
||||||
|
Err(p) => p.into_inner(),
|
||||||
|
};
|
||||||
|
guard.get(&(network, peer)).cloned()
|
||||||
|
};
|
||||||
|
if let Some(link) = link {
|
||||||
|
link.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Notifies the transport that a peer's capability was announced.
|
/// Notifies the transport that a peer's capability was announced.
|
||||||
///
|
///
|
||||||
/// When `we_accept` is true, sends a hole-punch probe to candidate addresses
|
/// When `we_accept` is true, sends a hole-punch probe to candidate addresses
|
||||||
|
|||||||
@@ -144,3 +144,77 @@ async fn test_wireguard_transport_unreachable_probe_times_out() {
|
|||||||
other => panic!("expected Unreachable error, got {other:?}"),
|
other => panic!("expected Unreachable error, got {other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wireguard_link_close_and_transport_close_link() {
|
||||||
|
let identity_a = DeviceIdentity::generate();
|
||||||
|
let peer_a = identity_a.endpoint_id();
|
||||||
|
let identity_b = DeviceIdentity::generate();
|
||||||
|
let peer_b = identity_b.endpoint_id();
|
||||||
|
let network = NetworkId::from_bytes([55u8; 32]);
|
||||||
|
|
||||||
|
let transport_b = Arc::new(
|
||||||
|
WireguardTransport::bind(&identity_b, Some(0))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let port_b = transport_b.bound_port();
|
||||||
|
|
||||||
|
let transport_a = Arc::new(
|
||||||
|
WireguardTransport::bind(&identity_a, Some(0))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let port_a = transport_a.bound_port();
|
||||||
|
|
||||||
|
let addr_b: SocketAddr = format!("127.0.0.1:{port_b}").parse().unwrap();
|
||||||
|
transport_a.set_peer_addr(peer_b, addr_b);
|
||||||
|
transport_a.set_peer_network(peer_b, network);
|
||||||
|
|
||||||
|
let addr_a: SocketAddr = format!("127.0.0.1:{port_a}").parse().unwrap();
|
||||||
|
transport_b.set_peer_addr(peer_a, addr_a);
|
||||||
|
transport_b.set_peer_network(peer_a, network);
|
||||||
|
|
||||||
|
let transport_a_clone = Arc::clone(&transport_a);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
transport_a_clone.accept_loop(|_| {}).await;
|
||||||
|
});
|
||||||
|
|
||||||
|
let transport_b_clone = Arc::clone(&transport_b);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
transport_b_clone.accept_loop(|_| {}).await;
|
||||||
|
});
|
||||||
|
|
||||||
|
let link_a = transport_a
|
||||||
|
.open(network, peer_b, "wg")
|
||||||
|
.await
|
||||||
|
.expect("open link A");
|
||||||
|
assert!(!link_a.is_closed());
|
||||||
|
|
||||||
|
// Spawn a reader task waiting on recv()
|
||||||
|
let link_a_clone = Arc::clone(&link_a);
|
||||||
|
let recv_task = tokio::spawn(async move { link_a_clone.recv().await });
|
||||||
|
|
||||||
|
// Explicitly closing via transport.close_link()
|
||||||
|
transport_a.close_link(network, peer_b);
|
||||||
|
assert!(link_a.is_closed());
|
||||||
|
|
||||||
|
// Wait on closed() notification
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), link_a.closed())
|
||||||
|
.await
|
||||||
|
.expect("link.closed() should resolve promptly");
|
||||||
|
|
||||||
|
// recv() must have returned None
|
||||||
|
let recv_res = tokio::time::timeout(Duration::from_secs(1), recv_task)
|
||||||
|
.await
|
||||||
|
.expect("recv task should complete")
|
||||||
|
.expect("task join");
|
||||||
|
assert_eq!(recv_res, None);
|
||||||
|
|
||||||
|
// Send after close must return Closed error
|
||||||
|
let send_res = link_a.send(Bytes::from_static(b"after close"));
|
||||||
|
assert!(matches!(
|
||||||
|
send_res,
|
||||||
|
Err(tsunagi::dataplane::transport::TransportError::Closed)
|
||||||
|
));
|
||||||
|
}
|
||||||
@@ -1118,6 +1118,26 @@ async fn plugin_request_loop(weak: Weak<Inner>, mut requests: mpsc::Receiver<Plu
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
PluginRequest::ProtocolFailure {
|
||||||
|
network,
|
||||||
|
peer,
|
||||||
|
protocol,
|
||||||
|
reason,
|
||||||
|
} => {
|
||||||
|
let sender = {
|
||||||
|
let networks = inner.networks.read().await;
|
||||||
|
networks.get(&network).map(|handle| handle.commands.clone())
|
||||||
|
};
|
||||||
|
if let Some(sender) = sender {
|
||||||
|
let _ = sender
|
||||||
|
.send(NetCommand::ProtocolFailure {
|
||||||
|
peer,
|
||||||
|
protocol,
|
||||||
|
reason,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,6 +86,12 @@ pub(crate) enum NetCommand {
|
|||||||
/// Human readable reason, free of secrets.
|
/// Human readable reason, free of secrets.
|
||||||
reason: String,
|
reason: String,
|
||||||
},
|
},
|
||||||
|
/// An IP plugin reported that data plane connectivity or a tunnel for a peer failed.
|
||||||
|
ProtocolFailure {
|
||||||
|
peer: EndpointId,
|
||||||
|
protocol: String,
|
||||||
|
reason: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for NetCommand {
|
impl std::fmt::Debug for NetCommand {
|
||||||
@@ -104,6 +110,9 @@ impl std::fmt::Debug for NetCommand {
|
|||||||
NetCommand::SetHostname(_) => f.write_str("SetHostname"),
|
NetCommand::SetHostname(_) => f.write_str("SetHostname"),
|
||||||
NetCommand::Release { .. } => f.write_str("Release"),
|
NetCommand::Release { .. } => f.write_str("Release"),
|
||||||
NetCommand::PluginError { protocol, .. } => write!(f, "PluginError({protocol})"),
|
NetCommand::PluginError { protocol, .. } => write!(f, "PluginError({protocol})"),
|
||||||
|
NetCommand::ProtocolFailure { peer, protocol, .. } => {
|
||||||
|
write!(f, "ProtocolFailure({}, {protocol})", peer.fmt_short())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -522,6 +531,26 @@ impl Runtime {
|
|||||||
reason,
|
reason,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
NetCommand::ProtocolFailure {
|
||||||
|
peer,
|
||||||
|
protocol,
|
||||||
|
reason,
|
||||||
|
} => {
|
||||||
|
if self.links.remove(&(peer, protocol.clone())).is_some() {
|
||||||
|
self.hub.clear_direct(peer, &protocol);
|
||||||
|
}
|
||||||
|
self.metrics.data_link_failures += 1;
|
||||||
|
self.record_protocol_failure(peer, &protocol);
|
||||||
|
self.emit(Event::DataLinkDown {
|
||||||
|
network: self.network_id,
|
||||||
|
peer,
|
||||||
|
protocol,
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
self.ensure_links();
|
||||||
|
self.update_paths();
|
||||||
|
self.announce_reach(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2162,4 +2191,70 @@ mod tests {
|
|||||||
assert_eq!(data_link_backoff(6), Duration::from_secs(60));
|
assert_eq!(data_link_backoff(6), Duration::from_secs(60));
|
||||||
assert_eq!(data_link_backoff(10), Duration::from_secs(60));
|
assert_eq!(data_link_backoff(10), Duration::from_secs(60));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_protocol_dial_state_cooldown_and_fallback_selection() {
|
||||||
|
let peer = crate::identity::DeviceIdentity::generate().endpoint_id();
|
||||||
|
let supported = ["wg", "tcp-tls", "wg-quic"];
|
||||||
|
let mut states: HashMap<(EndpointId, String), ProtocolDialState> = HashMap::new();
|
||||||
|
|
||||||
|
let now = std::time::Instant::now();
|
||||||
|
|
||||||
|
// 1. Without cooldown, the highest priority protocol "wg" is chosen
|
||||||
|
let chosen =
|
||||||
|
supported
|
||||||
|
.iter()
|
||||||
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
||||||
|
Some(state) => state.cooldown_until <= now,
|
||||||
|
None => true,
|
||||||
|
});
|
||||||
|
assert_eq!(chosen.copied(), Some("wg"));
|
||||||
|
|
||||||
|
// 2. Record failure for "wg" putting it on cooldown
|
||||||
|
states.insert(
|
||||||
|
(peer, "wg".to_string()),
|
||||||
|
ProtocolDialState {
|
||||||
|
failures: 1,
|
||||||
|
cooldown_until: now + Duration::from_secs(3),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// With "wg" on cooldown, selection must fall back to next priority "tcp-tls"
|
||||||
|
let chosen_after_failure =
|
||||||
|
supported
|
||||||
|
.iter()
|
||||||
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
||||||
|
Some(state) => state.cooldown_until <= now,
|
||||||
|
None => true,
|
||||||
|
});
|
||||||
|
assert_eq!(chosen_after_failure.copied(), Some("tcp-tls"));
|
||||||
|
|
||||||
|
// 3. If "tcp-tls" also fails, selection falls back to "wg-quic"
|
||||||
|
states.insert(
|
||||||
|
(peer, "tcp-tls".to_string()),
|
||||||
|
ProtocolDialState {
|
||||||
|
failures: 1,
|
||||||
|
cooldown_until: now + Duration::from_secs(3),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let chosen_quic =
|
||||||
|
supported
|
||||||
|
.iter()
|
||||||
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
||||||
|
Some(state) => state.cooldown_until <= now,
|
||||||
|
None => true,
|
||||||
|
});
|
||||||
|
assert_eq!(chosen_quic.copied(), Some("wg-quic"));
|
||||||
|
|
||||||
|
// 4. Once success is recorded (entry removed), "wg" is preferred again
|
||||||
|
states.remove(&(peer, "wg".to_string()));
|
||||||
|
let chosen_restored =
|
||||||
|
supported
|
||||||
|
.iter()
|
||||||
|
.find(|proto| match states.get(&(peer, (*proto).to_string())) {
|
||||||
|
Some(state) => state.cooldown_until <= now,
|
||||||
|
None => true,
|
||||||
|
});
|
||||||
|
assert_eq!(chosen_restored.copied(), Some("wg"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -119,7 +119,7 @@ pub enum PluginError {
|
|||||||
|
|
||||||
/// A request a plugin makes of the agent that owns it.
|
/// A request a plugin makes of the agent that owns it.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) enum PluginRequest {
|
pub enum PluginRequest {
|
||||||
/// Re-send this agent's announcement to every peer of a network.
|
/// Re-send this agent's announcement to every peer of a network.
|
||||||
Reannounce(NetworkId),
|
Reannounce(NetworkId),
|
||||||
/// Surface a plugin error on the agent's event stream.
|
/// Surface a plugin error on the agent's event stream.
|
||||||
@@ -131,6 +131,17 @@ pub(crate) enum PluginRequest {
|
|||||||
/// Human readable reason, free of secrets.
|
/// Human readable reason, free of secrets.
|
||||||
reason: String,
|
reason: String,
|
||||||
},
|
},
|
||||||
|
/// A plugin reported that a data link or tunnel for a peer failed.
|
||||||
|
ProtocolFailure {
|
||||||
|
/// Network the failure occurred in.
|
||||||
|
network: NetworkId,
|
||||||
|
/// Peer whose protocol failed.
|
||||||
|
peer: EndpointId,
|
||||||
|
/// Protocol that failed.
|
||||||
|
protocol: String,
|
||||||
|
/// Human readable reason.
|
||||||
|
reason: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The agent-side handle a plugin is given when it is attached.
|
/// The agent-side handle a plugin is given when it is attached.
|
||||||
@@ -192,6 +203,20 @@ impl PluginContext {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A context configured with a channel for plugin requests and custom sink, for tests.
|
||||||
|
#[cfg(feature = "testing")]
|
||||||
|
pub fn for_test(
|
||||||
|
sender: mpsc::Sender<PluginRequest>,
|
||||||
|
local: EndpointId,
|
||||||
|
sink: Option<Arc<dyn PacketSink>>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
sender: Some(sender),
|
||||||
|
local: Some(local),
|
||||||
|
sink,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// This agent's own endpoint id, when the context is attached.
|
/// This agent's own endpoint id, when the context is attached.
|
||||||
///
|
///
|
||||||
/// A plugin needs it to find itself in the agreed allocation.
|
/// A plugin needs it to find itself in the agreed allocation.
|
||||||
@@ -233,6 +258,22 @@ impl PluginContext {
|
|||||||
reason: reason.into(),
|
reason: reason.into(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reports that data plane connectivity or a tunnel for a peer has failed.
|
||||||
|
pub fn report_peer_protocol_failure(
|
||||||
|
&self,
|
||||||
|
network: NetworkId,
|
||||||
|
peer: EndpointId,
|
||||||
|
protocol: impl Into<String>,
|
||||||
|
reason: impl Into<String>,
|
||||||
|
) {
|
||||||
|
self.send(PluginRequest::ProtocolFailure {
|
||||||
|
network,
|
||||||
|
peer,
|
||||||
|
protocol: protocol.into(),
|
||||||
|
reason: reason.into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for PluginContext {
|
impl std::fmt::Debug for PluginContext {
|
||||||
|
|||||||
Reference in new issue
Block a user