From c0def74115a205b67d51ce99551ae14d1c65752a Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 18:50:22 +0200 Subject: [PATCH] Evict podcast replicas fairly and trust only verified contacts Replicas no longer hand capacity to whoever stored first: a key holds at most one FindValue response worth of records (128, 32 tombstones), so every admitted record is served, and a full key or cache admits a newcomer by evicting a random record from its younger half. In-place card-to-tombstone conversions count against the tombstone budget. Contacts returned by peers are only lookup and bootstrap candidates until they answer; replica responsibility counts verified contacts that are not backed off, and contacts that keep failing are dropped. Refused store batches fall back to the next closest nodes once. Each lookup response may add at most 64 new owners. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/podcast_dht/mod.rs | 367 +++++++++++++---- crates/music-dht/src/podcast_dht/store.rs | 262 ++++++++++-- crates/music-dht/src/podcast_dht/tests.rs | 480 ++++++++++++++++++---- crates/music-dht/tests/podcast_dht.rs | 90 ++++ 4 files changed, 987 insertions(+), 212 deletions(-) diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs index f71554f..ce8d5ba 100644 --- a/crates/music-dht/src/podcast_dht/mod.rs +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -124,6 +124,11 @@ pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8; /// Maximum distinct `(podcast_id, publisher)` records one lookup keeps; /// records of further owners are ignored. pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512; +/// Most new `(podcast_id, publisher)` owners one response (or the local +/// cache) may add to a lookup: a `1 / K` share of +/// [`MAX_PODCAST_LOOKUP_RECORDS`], so the first responders cannot fill the +/// lookup and crowd out later ones. +pub const MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE: usize = MAX_PODCAST_LOOKUP_RECORDS / K; /// A node stores a placement only when fewer than this many of its known /// overlay contacts are closer to the key than itself (`2 * K`: the `K` /// closest with slack for differing routing views). @@ -141,6 +146,9 @@ const SERVE_TIMEOUT: Duration = Duration::from_secs(10); /// A peer that failed to answer (network error or timeout) is skipped by /// lookups, publication and bootstrap for this long. const FAILED_PEER_BACKOFF: Duration = Duration::from_secs(30); +/// A contact that failed this many times in a row (each after its backoff) +/// is removed from the routing table and the persisted peers. +const MAX_CONSECUTIVE_FAILURES: u32 = 3; const MAX_TICKET_BYTES: usize = 16 * 1024; const MAX_ERROR_BYTES: usize = 1024; @@ -354,6 +362,7 @@ pub struct PodcastDht { routing: Mutex, unsupported: Mutex>, failed: Mutex>, + failures: Mutex>, publish_gate: Semaphore, serve_limit: Arc, inflight: Arc>>, @@ -401,6 +410,7 @@ impl PodcastDht { routing: Mutex::new(routing), unsupported: Mutex::new(HashMap::new()), failed: Mutex::new(HashMap::new()), + failures: Mutex::new(HashMap::new()), publish_gate: Semaphore::new(1), serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), inflight: Arc::new(Mutex::new(HashMap::new())), @@ -481,9 +491,12 @@ impl PodcastDht { /// A peer whose manifest lacks [`PODCAST_DHT_ID`] at /// [`PODCAST_DHT_PROTOCOL_VERSION`] is removed from the overlay routing /// table and not contacted for [`UNSUPPORTED_PEER_BACKOFF`]. Peers only - /// enter the routing table after answering on [`PODCAST_DHT_ALPN`] or - /// being returned by an overlay peer, so a peer without the overlay is - /// never queried for records or asked to store them. + /// enter the routing table after answering a request on + /// [`PODCAST_DHT_ALPN`] or sending one as an authenticated requester, so + /// a peer without the overlay is never asked to store records and never + /// counts for replica responsibility. A contact another peer returns is + /// only a lookup candidate until it answers itself; a candidate that does + /// not answer is backed off like any failed peer. pub async fn note_peer_capabilities( &self, peer: EndpointId, @@ -500,10 +513,11 @@ impl PodcastDht { } /// Discovers overlay peers among the routing table and the music DHT - /// contacts with one `FindNode` each. Music-DHT contacts whose capability - /// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are - /// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become - /// overlay contacts when they answer. + /// contacts with one `FindNode` each, then asks the not yet known contacts + /// they return once more. Music-DHT contacts whose capability manifest + /// lacks the overlay ([`Self::note_peer_capabilities`]) are skipped; + /// others are dialed on [`PODCAST_DHT_ALPN`]. A peer only becomes an + /// overlay contact when it answers. pub async fn bootstrap(&self) -> Result<()> { let known_feature: HashSet = lock(&self.routing) .contacts() @@ -517,25 +531,19 @@ impl PodcastDht { .into_iter() .filter(|contact| !known_feature.contains(&contact.peer_id)), ); - let mut seen = HashSet::new(); + let mut seen: HashSet = known_feature; + seen.insert(self.own_contact.peer_id); + let mut queried = HashSet::new(); candidates.retain(|contact| { contact.peer_id != self.own_contact.peer_id && !self.is_unsupported(&contact.peer_id) && !self.is_backed_off(&contact.peer_id) - && seen.insert(contact.peer_id) + && queried.insert(contact.peer_id) }); + seen.extend(queried); candidates.truncate(MAX_BOOTSTRAP_PEERS); - let target = *self.own_contact.node_id.as_bytes(); - let responses = stream::iter(candidates.into_iter().map(|contact| async move { - let response = self - .exchange(&contact, PodcastDhtOperation::FindNode { target }) - .await; - (contact, response) - })) - .buffer_unordered(BOOTSTRAP_CONCURRENCY) - .collect::>() - .await; - for (contact, response) in responses { + let mut discovered = Vec::new(); + for (contact, response) in self.find_own_node(candidates).await { let response = match response { Ok(response) => response, Err(error) => { @@ -544,11 +552,46 @@ impl PodcastDht { } }; self.learn_contact(contact).await?; - self.learn_response_nodes(response.nodes).await; + for node in response.nodes { + if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)).is_ok() + && !self.is_unsupported(&node.peer_id) + && !self.is_backed_off(&node.peer_id) + && seen.insert(node.peer_id) + { + discovered.push(node); + } + } + } + // Returned contacts are unverified: they join only by answering. + discovered.truncate(MAX_BOOTSTRAP_PEERS); + for (contact, response) in self.find_own_node(discovered).await { + match response { + Ok(_) => self.learn_contact(contact).await?, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT contact did not answer"); + } + } } Ok(()) } + /// One `FindNode` for this node's id to each contact. + async fn find_own_node( + &self, + contacts: Vec, + ) -> Vec<(NodeContact, Result)> { + let target = *self.own_contact.node_id.as_bytes(); + stream::iter(contacts.into_iter().map(|contact| async move { + let response = self + .exchange(&contact, PodcastDhtOperation::FindNode { target }) + .await; + (contact, response) + })) + .buffer_unordered(BOOTSTRAP_CONCURRENCY) + .collect::>() + .await + } + /// Publishes (or refreshes) this device's card of a podcast. /// /// Refused with [`MusicDhtError::PodcastPublication`] unless @@ -892,7 +935,10 @@ impl PodcastDht { } /// Stores records on the `K` closest overlay nodes of each of their keys - /// (including this node when it is among them). + /// (including this node when it is among them). When a remote node + /// refuses a batch (`stored = 0`, for example because it does not count + /// itself responsible for the keys) or fails, its records are offered + /// once to the next closest nodes of the same keys, one per refusal. async fn push(&self, records: Vec) -> Result { let mut stats = PodcastDhtPublishStats { records: records.len(), @@ -902,69 +948,105 @@ impl PodcastDht { return Ok(stats); } let mut local = Vec::new(); - let mut remote: HashMap)> = HashMap::new(); - for record in records { + let mut plan = StorePlan::new(); + let mut fallbacks: HashMap<(usize, PodcastDhtKey), Vec> = HashMap::new(); + for (index, record) in records.iter().enumerate() { let mut local_keys = Vec::new(); - let mut remote_keys: HashMap)> = - HashMap::new(); for key in record.dht_keys() { stats.keys += 1; - let (store_local, contacts) = self.replica_targets(&key); - if store_local { + let targets = self.replica_targets(&key); + if targets.local { local_keys.push(key); } - for contact in contacts { - remote_keys - .entry(contact.peer_id) - .or_insert_with(|| (contact, Vec::new())) - .1 - .push(key); + for contact in targets.primary { + plan.add(contact, index, key); } + fallbacks.insert((index, key), targets.fallback); } if !local_keys.is_empty() { local.push((local_keys, record.clone())); } - for (peer, (contact, keys)) in remote_keys { - remote - .entry(peer) - .or_insert_with(|| (contact, Vec::new())) - .1 - .push(PodcastStoreEntry { - keys, - record: record.clone(), - }); - } } if !local.is_empty() { self.database.store(local, now_ms()).await?; stats.local_replica = true; } - let sends = stream::iter(remote.into_values().map(|(contact, entries)| async move { - let mut accepted = false; - for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { - let response = self - .exchange( - &contact, - PodcastDhtOperation::StoreBatch { - records: batch.to_vec(), + let (mut accepted, refused) = self.send_store_plan(&records, plan).await; + let retry = retry_plan(refused, &mut fallbacks); + if !retry.is_empty() { + accepted.extend(self.send_store_plan(&records, retry).await.0); + } + stats.remote_nodes = accepted.len(); + Ok(stats) + } + + /// Sends a store plan. Returns the nodes that stored at least one + /// placement and the `(record index, key)` placements offered in batches + /// that were refused or failed. + async fn send_store_plan( + &self, + records: &[SignedPodcastRecord], + plan: StorePlan, + ) -> (HashSet, Vec<(usize, PodcastDhtKey)>) { + let sends = stream::iter(plan.targets.into_values().map(|(contact, entries)| async move { + let entries: Vec<(usize, PodcastStoreEntry)> = entries + .into_iter() + .map(|(index, keys)| { + ( + index, + PodcastStoreEntry { + keys, + record: records[index].clone(), }, ) - .await; - match response { - Ok(response) => accepted |= response.stored > 0, - Err(error) => { - debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); - break; + }) + .collect(); + let mut accepted = false; + let mut refused = Vec::new(); + let mut failed = false; + for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + let stored = if failed { + false + } else { + let response = self + .exchange( + &contact, + PodcastDhtOperation::StoreBatch { + records: batch.iter().map(|(_, entry)| entry.clone()).collect(), + }, + ) + .await; + match response { + Ok(response) => response.stored > 0, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); + failed = true; + false + } + } + }; + if stored { + accepted = true; + } else { + for (index, entry) in batch { + refused.extend(entry.keys.iter().map(|key| (*index, *key))); } } } - accepted + (contact.peer_id, accepted, refused) })) .buffer_unordered(PUBLISH_CONCURRENCY) .collect::>() .await; - stats.remote_nodes = sends.into_iter().filter(|accepted| *accepted).count(); - Ok(stats) + let mut accepted = HashSet::new(); + let mut refused = Vec::new(); + for (peer, stored, keys) in sends { + if stored { + accepted.insert(peer); + } + refused.extend(keys); + } + (accepted, refused) } /// Iterative lookup of `key`: queries the `K` closest known overlay @@ -1013,16 +1095,23 @@ impl PodcastDht { filter_tokens: filter_tokens.clone(), }; let mut futures: FuturesUnordered<_> = batch - .iter() + .into_iter() .map(|contact| { let operation = operation.clone(); - async move { self.exchange(contact, operation).await } + async move { + let response = self.exchange(&contact, operation).await; + (contact, response) + } }) .collect(); - while let Ok(Some(response)) = tokio::time::timeout_at(deadline, futures.next()).await { + while let Ok(Some((contact, response))) = + tokio::time::timeout_at(deadline, futures.next()).await + { let Ok(response) = response else { continue; }; + // It answered on the overlay: now it is a verified contact. + self.learn_contact(contact).await?; merge_records( &mut winners, response.records, @@ -1031,12 +1120,12 @@ impl PodcastDht { &filter_tokens, ); for node in response.nodes { + // Returned contacts are only candidates until they answer. if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)) .is_ok() && !self.is_unsupported(&node.peer_id) && known.insert(node.peer_id) { - self.learn_contact(node.clone()).await?; candidates.push(node); } } @@ -1098,9 +1187,11 @@ impl PodcastDht { PodcastDhtOperation::StoreBatch { records } => { let entries = validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?; + // Only verified contacts that are not backed off count. let nodes: Vec = lock(&self.routing) .contacts() .into_iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) .map(|contact| contact.node_id) .collect(); let entries: Vec<_> = entries @@ -1137,10 +1228,21 @@ impl PodcastDht { match &result { Ok(_) => { lock(&self.failed).remove(&contact.peer_id); + lock(&self.failures).remove(&contact.peer_id); } Err(MusicDhtError::Network(_) | MusicDhtError::Timeout) => { let until = now_ms().saturating_add(FAILED_PEER_BACKOFF.as_millis() as u64); lock(&self.failed).insert(contact.peer_id, until); + let failures = { + let mut failures = lock(&self.failures); + let count = failures.entry(contact.peer_id).or_default(); + *count += 1; + *count + }; + if failures >= MAX_CONSECUTIVE_FAILURES { + lock(&self.failures).remove(&contact.peer_id); + self.forget_contact(contact.peer_id).await; + } } Err(_) => {} } @@ -1180,6 +1282,16 @@ impl PodcastDht { Ok(response) } + /// Removes a contact that keeps failing from routing and the cache. + async fn forget_contact(&self, peer: EndpointId) { + if lock(&self.routing).remove(&peer) { + debug!(%peer, "podcast DHT contact dropped after repeated failures"); + } + if let Err(error) = self.database.remove_peer(peer).await { + debug!(%peer, %error, "podcast DHT contact removal failed"); + } + } + fn is_unsupported(&self, peer: &EndpointId) -> bool { still_listed(&self.unsupported, peer) } @@ -1188,21 +1300,15 @@ impl PodcastDht { still_listed(&self.failed, peer) } - fn replica_targets(&self, key: &PodcastDhtKey) -> (bool, Vec) { - let mut contacts: Vec<_> = lock(&self.routing) + /// Replica targets of `key` among the contacts that are not backed off + /// ([`replica_targets`]). + fn replica_targets(&self, key: &PodcastDhtKey) -> ReplicaTargets { + let contacts: Vec<_> = lock(&self.routing) .contacts() .into_iter() .filter(|contact| !self.is_backed_off(&contact.peer_id)) .collect(); - contacts.push(self.fresh_own_contact()); - contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); - contacts.dedup_by_key(|contact| contact.peer_id); - contacts.truncate(K); - let local = contacts - .iter() - .any(|contact| contact.peer_id == self.own_contact.peer_id); - contacts.retain(|contact| contact.peer_id != self.own_contact.peer_id); - (local, contacts) + replica_targets(self.fresh_own_contact(), contacts, key) } fn closest_contacts(&self, target: &[u8; 32], count: usize) -> Vec { @@ -1220,17 +1326,6 @@ impl PodcastDht { contact } - async fn learn_response_nodes(&self, nodes: Vec) { - for contact in nodes { - if self.is_unsupported(&contact.peer_id) { - continue; - } - if let Err(error) = self.learn_contact(contact).await { - debug!(%error, "invalid podcast DHT contact ignored"); - } - } - } - async fn learn_contact(&self, mut contact: NodeContact) -> Result<()> { validate_contact(&contact, &self.network_id, Some(self.own_contact.peer_id))?; contact.node_id = NodeId::from_endpoint(&contact.peer_id); @@ -1309,8 +1404,12 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> { Ok(()) } -/// Adds verified records found under `key` to the per-owner winners, keeping -/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners. +/// Adds verified records of one response (or the local cache) found under +/// `key` to the per-owner winners. A newer record of a known owner always +/// replaces its winner; the response may add at most +/// [`MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE`] new owners, and the lookup +/// keeps at most [`MAX_PODCAST_LOOKUP_RECORDS`]. So the order in which +/// responders answer does not let the first ones take every slot. fn merge_records( winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, records: Vec, @@ -1319,6 +1418,7 @@ fn merge_records( filter_tokens: &[String], ) { let now = now_ms(); + let mut added = 0usize; for record in records { if record.verify_at(network_id, now).is_err() || !record.belongs_under(key) { continue; @@ -1332,14 +1432,99 @@ fn merge_records( let owner = (record.podcast_id().clone(), record.publisher()); match winners.get(&owner) { Some(current) if compare_records(current, &record).is_ge() => {} - None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {} - _ => { + Some(_) => { winners.insert(owner, record); } + None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS + || added >= MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE => {} + None => { + winners.insert(owner, record); + added += 1; + } } } } +/// Replica nodes of one key, from [`replica_targets`]. +#[derive(Debug)] +struct ReplicaTargets { + /// Whether this node is among the `K` closest. + local: bool, + /// The other nodes among the `K` closest. + primary: Vec, + /// The next up to `K` nodes, farthest first (popped closest first). + fallback: Vec, +} + +/// The `K` closest nodes of `key` among `contacts` and `own` (this node), and +/// the next `K` as fallbacks. +fn replica_targets( + own: NodeContact, + mut contacts: Vec, + key: &PodcastDhtKey, +) -> ReplicaTargets { + let own_peer = own.peer_id; + contacts.push(own); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + contacts.dedup_by_key(|contact| contact.peer_id); + let mut fallback = contacts.split_off(K.min(contacts.len())); + fallback.retain(|contact| contact.peer_id != own_peer); + fallback.truncate(K); + fallback.reverse(); + let local = contacts.iter().any(|contact| contact.peer_id == own_peer); + contacts.retain(|contact| contact.peer_id != own_peer); + ReplicaTargets { + local, + primary: contacts, + fallback, + } +} + +/// One fallback node per refused `(record index, key)` placement: the +/// closest one not yet offered that placement. +fn retry_plan( + refused: Vec<(usize, PodcastDhtKey)>, + fallbacks: &mut HashMap<(usize, PodcastDhtKey), Vec>, +) -> StorePlan { + let mut retry = StorePlan::new(); + for (index, key) in refused { + if let Some(contact) = fallbacks + .get_mut(&(index, key)) + .and_then(|fallback| fallback.pop()) + { + retry.add(contact, index, key); + } + } + retry +} + +/// Placements to offer per remote node: record index and keys. +struct StorePlan { + targets: HashMap>)>, +} + +impl StorePlan { + fn new() -> Self { + Self { + targets: HashMap::new(), + } + } + + fn add(&mut self, contact: NodeContact, index: usize, key: PodcastDhtKey) { + self.targets + .entry(contact.peer_id) + .or_insert_with(|| (contact, BTreeMap::new())) + .1 + .entry(index) + .or_default() + .push(key); + } + + fn is_empty(&self) -> bool { + self.targets.is_empty() + } +} + /// One in-flight inbound request of a peer; released on drop. struct PeerSlot { inflight: Arc>>, diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs index acaedbc..1a86e82 100644 --- a/crates/music-dht/src/podcast_dht/store.rs +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -17,33 +17,53 @@ use crate::error::{MusicDhtError, Result}; use crate::podcast::PodcastId; use crate::routing::{NodeContact, NodeId}; +use super::MAX_PODCAST_DHT_RECORDS; use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records}; /// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1 /// (pre-release) stored one payload per key; its replica table is dropped on /// open because the cache is replaceable. Own publications are kept. pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2; -/// Maximum records (cards and tombstones) indexed under one key. A full key -/// admits no new `(podcast_id, publisher)` until a record expires: existing -/// records are never evicted for newcomers, and their owners keep refreshing -/// them in place. -pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; -/// Maximum tombstones newly indexed under one key. A tombstone is admitted -/// under a text key only in place of a record of the same -/// `(podcast_id, publisher)` already indexed there, so this budget applies to -/// the podcast-id key, where tombstones of unknown cards land. -pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32; +/// Maximum records (cards and tombstones) indexed under one key. It equals +/// [`MAX_PODCAST_DHT_RECORDS`], so one `FindValue` response carries every +/// record of a key: nothing admitted is unreachable. +/// +/// A full key admits a newcomer by evicting a random placement from its +/// younger half (by first indexing time). The older half is never evicted +/// for newcomers, so cards that were there before a flood stay, and the +/// younger half stays contestable, so nobody who stored first holds the key +/// for good. +pub const MAX_PODCAST_RECORDS_PER_KEY: usize = MAX_PODCAST_DHT_RECORDS; +/// Maximum tombstones indexed under one key, including tombstones that +/// replaced their owner's card in place. Over budget, a random tombstone of +/// another owner from the younger half of the key's tombstones is evicted. +/// It equals [`MAX_PODCAST_RESPONSE_TOMBSTONES`], so every tombstone of a key +/// is served. +pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = MAX_PODCAST_RESPONSE_TOMBSTONES; /// Maximum tombstones in one `FindValue` response; cards fill the rest. pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32; /// Maximum `(podcast_id, publisher)` records one publisher device keeps in a -/// replica. +/// replica. A publisher over it is refused (nothing is evicted for it). pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256; -/// Maximum `(podcast_id, publisher)` records in a replica cache. +/// Maximum `(podcast_id, publisher)` records in a replica cache. A full cache +/// admits a new record by evicting random records from the younger half of +/// the cache (by first indexing time), like a full key. pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096; -/// Maximum total encoded size of the records in a replica cache. +/// Maximum total encoded size of the records in a replica cache, enforced +/// like [`MAX_PODCAST_CACHE_RECORDS`]. pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024; /// Maximum persisted overlay contacts; the least recently seen are dropped. pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024; +/// Maximum records evicted from the cache to admit one record; when that is +/// not enough room, the record is refused and nothing is evicted. +const MAX_EVICTIONS_PER_RECORD: usize = 16; + +const _: () = assert!( + MAX_PODCAST_RECORDS_PER_KEY <= MAX_PODCAST_DHT_RECORDS + && MAX_PODCAST_TOMBSTONES_PER_KEY <= MAX_PODCAST_RESPONSE_TOMBSTONES + && MAX_PODCAST_RESPONSE_TOMBSTONES <= MAX_PODCAST_DHT_RECORDS, + "every record admitted under a key must fit one FindValue response" +); const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS podcast_dht_records ( @@ -227,7 +247,8 @@ impl PodcastDhtDatabase { /// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale /// replicas elsewhere can be overruled) and cards containing every token /// in `filter_tokens` up to `limit` records in all. Records indexed longest - /// come first. Cards precede tombstones. + /// come first. Cards precede tombstones. With the default limits and + /// `limit` = [`MAX_PODCAST_DHT_RECORDS`], every record of the key fits. pub async fn records_by_key( &self, key: PodcastDhtKey, @@ -475,10 +496,10 @@ fn store_one( }; let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; if replace - && !admits_record( + && !make_room_for_record( conn, limits, - &publisher, + (&podcast_id, &publisher), existing.is_none(), existing_bytes, payload.len(), @@ -536,6 +557,14 @@ fn store_one( clamp_i64(record.expires_at_ms()), ], )?; + // A card converted to a tombstone in place counts against the + // tombstone budget of each key it stays under, like a new one. + if record.is_tombstone() { + for key in &indexed { + let key = PodcastDhtKey::from_bytes(*key); + make_room_under_key(conn, limits, &key, true, (&podcast_id, &publisher), now, 0)?; + } + } } // New placements. A tombstone only takes a new place under the podcast-id @@ -549,7 +578,15 @@ fn store_one( if record.is_tombstone() && *key != id_key { continue; } - if !admits_placement(conn, limits, key, record.is_tombstone(), now)? { + if !make_room_under_key( + conn, + limits, + key, + record.is_tombstone(), + (&podcast_id, &publisher), + now, + 1, + )? { continue; } conn.execute( @@ -596,58 +633,197 @@ fn store_one( Ok(written) } -/// Whether the cache may hold the new version of a record: per-publisher and -/// global record counts for a new owner, and the global byte budget. -fn admits_record( +/// Makes room in the cache for the new version of a record of `owner`. +/// +/// * A publisher at [`StoreLimits::per_publisher`] gets no new record. +/// * When the record count (for a new owner) or the byte budget would be +/// exceeded, random records of other owners from the younger half of the +/// cache (by first indexing time) are evicted, at most +/// [`MAX_EVICTIONS_PER_RECORD`]. If that is not enough, nothing is evicted +/// and the record is refused. +fn make_room_for_record( conn: &Connection, limits: &StoreLimits, - publisher: &str, + owner: (&str, &str), new_owner: bool, previous_bytes: usize, new_bytes: usize, now: i64, ) -> rusqlite::Result { + if new_bytes > limits.bytes { + return Ok(false); + } + if new_owner { + let own: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", + params![owner.1, now], + |row| row.get(0), + )?; + if own.max(0) as usize >= limits.per_publisher { + return Ok(false); + } + } let (records, bytes): (i64, i64) = conn.query_row( "SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records WHERE expires_at_ms > ?1", [now], |row| Ok((row.get(0)?, row.get(1)?)), )?; - let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); - if bytes.saturating_add(new_bytes) > limits.bytes { - return Ok(false); - } - if !new_owner { + let mut records = records.max(0) as usize; + let mut bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); + let fits = |records: usize, bytes: usize| { + bytes.saturating_add(new_bytes) <= limits.bytes && (!new_owner || records < limits.records) + }; + if fits(records, bytes) { return Ok(true); } - if records.max(0) as usize >= limits.records { + + // Candidates: the younger half of the other owners' records, shuffled. + let others = records.saturating_sub(usize::from(!new_owner)); + let mut statement = conn.prepare( + "SELECT podcast_id, publisher, payload_bytes FROM ( + SELECT r.podcast_id, r.publisher, r.payload_bytes, + (SELECT MIN(k.first_stored_at_ms) FROM podcast_dht_keys k + WHERE k.podcast_id = r.podcast_id AND k.publisher = r.publisher) AS since + FROM podcast_dht_records r + WHERE r.expires_at_ms > ?1 AND NOT (r.podcast_id = ?2 AND r.publisher = ?3) + ORDER BY since DESC, random() + LIMIT ?4) + ORDER BY random() LIMIT ?5", + )?; + let candidates = statement + .query_map( + params![ + now, + owner.0, + owner.1, + others.div_ceil(2) as i64, + MAX_EVICTIONS_PER_RECORD as i64 + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + )? + .collect::>>()?; + let mut evict = Vec::new(); + for (podcast_id, publisher, payload_bytes) in candidates { + if fits(records, bytes) { + break; + } + records = records.saturating_sub(1); + bytes = bytes.saturating_sub(payload_bytes.max(0) as usize); + evict.push((podcast_id, publisher)); + } + if !fits(records, bytes) { return Ok(false); } - let own: i64 = conn.query_row( - "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", - params![publisher, now], - |row| row.get(0), - )?; - Ok((own.max(0) as usize) < limits.per_publisher) + for (podcast_id, publisher) in evict { + delete_owner(conn, &podcast_id, &publisher)?; + } + Ok(true) } -/// Whether `key` has room for one more placement. Existing placements are -/// never evicted. -fn admits_placement( +/// Makes room under `key` for `incoming` more placements (0 when an existing +/// placement of `owner` was converted to a tombstone in place) and returns +/// whether the key has room. +/// +/// Over [`StoreLimits::per_key`], or over [`StoreLimits::tombstones_per_key`] +/// for a tombstone, a random placement of another owner from the younger +/// half of the key (by first indexing time; only tombstones for the +/// tombstone budget) is evicted. The older half is never evicted for a +/// newcomer. +fn make_room_under_key( conn: &Connection, limits: &StoreLimits, key: &PodcastDhtKey, tombstone: bool, + owner: (&str, &str), now: i64, + incoming: usize, ) -> rusqlite::Result { - let (total, tombstones): (i64, i64) = conn.query_row( - "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys - WHERE dht_key = ?1 AND expires_at_ms > ?2", - params![key.as_bytes().as_slice(), now], - |row| Ok((row.get(0)?, row.get(1)?)), + for _ in 0..=MAX_EVICTIONS_PER_RECORD { + let (total, tombstones): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys + WHERE dht_key = ?1 AND expires_at_ms > ?2", + params![key.as_bytes().as_slice(), now], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let over_tombstones = + tombstone && tombstones.max(0) as usize + incoming > limits.tombstones_per_key; + let over_total = total.max(0) as usize + incoming > limits.per_key; + if !over_tombstones && !over_total { + return Ok(true); + } + let Some((podcast_id, publisher)) = + young_placement(conn, key, over_tombstones, owner, now)? + else { + return Ok(false); + }; + conn.execute( + "DELETE FROM podcast_dht_keys + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_bytes().as_slice(), podcast_id, publisher], + )?; + let placed: bool = conn.query_row( + "SELECT EXISTS (SELECT 1 FROM podcast_dht_keys + WHERE podcast_id = ?1 AND publisher = ?2)", + params![podcast_id, publisher], + |row| row.get(0), + )?; + if !placed { + delete_owner(conn, &podcast_id, &publisher)?; + } + } + Ok(false) +} + +/// A random unexpired placement of another owner than `owner` from the +/// younger half of `key` (only tombstones with `tombstones_only`). +fn young_placement( + conn: &Connection, + key: &PodcastDhtKey, + tombstones_only: bool, + owner: (&str, &str), + now: i64, +) -> rusqlite::Result> { + let filter = "dht_key = ?1 AND expires_at_ms > ?2 AND (?3 = 0 OR tombstone = 1) + AND NOT (podcast_id = ?4 AND publisher = ?5)"; + let count: i64 = conn.query_row( + &format!("SELECT COUNT(*) FROM podcast_dht_keys WHERE {filter}"), + params![ + key.as_bytes().as_slice(), + now, + tombstones_only, + owner.0, + owner.1 + ], + |row| row.get(0), )?; - Ok((total.max(0) as usize) < limits.per_key - && (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key)) + if count <= 0 { + return Ok(None); + } + conn.query_row( + &format!( + "SELECT podcast_id, publisher FROM ( + SELECT podcast_id, publisher FROM podcast_dht_keys WHERE {filter} + ORDER BY first_stored_at_ms DESC, random() LIMIT ?6) + ORDER BY random() LIMIT 1" + ), + params![ + key.as_bytes().as_slice(), + now, + tombstones_only, + owner.0, + owner.1, + (count as usize).div_ceil(2) as i64 + ], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() } fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> { diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs index 034f6c3..316bbb5 100644 --- a/crates/music-dht/src/podcast_dht/tests.rs +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -1,6 +1,8 @@ //! Unit tests of the podcast-discovery overlay: records, ordering, cache and //! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`. +use std::collections::HashSet; + use federation_net::{NetworkId, SecretKey}; use crate::routing::NodeId; @@ -584,50 +586,239 @@ async fn open_db_with( .unwrap() } +fn rust_card(index: usize, issued: u64) -> SignedPodcastRecord { + card_with( + spec(&format!("show-{index}.xml"), "Rust Radio"), + &numbered_secret(index), + 1, + issued, + ) +} + +async fn store_all( + db: &store::PodcastDhtDatabase, + entries: Vec<(Vec, SignedPodcastRecord)>, + now: u64, +) -> usize { + let mut written = 0; + for chunk in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + written += db.store(chunk.to_vec(), now).await.unwrap(); + } + written +} + +async fn count_records(temp: &tempfile::TempDir) -> i64 { + rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")) + .unwrap() + .query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| { + row.get(0) + }) + .unwrap() +} + #[tokio::test] -async fn cache_full_key_keeps_its_records_and_refuses_newcomers() { +async fn cache_full_key_admits_newcomers_from_its_younger_half() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; - let token_key = PodcastDhtKey::token(&network(), "rust"); - let mut entries = Vec::new(); - for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { - // Later publishers expire later; that buys them nothing. - let record = card_with( - spec("show.xml", "Rust Radio"), - &numbered_secret(index), + let rust = PodcastDhtKey::token(&network(), "rust"); + let half = MAX_PODCAST_RECORDS_PER_KEY / 2; + let older: Vec<_> = (0..half).map(|index| rust_card(index, NOW)).collect(); + let younger: Vec<_> = (half..MAX_PODCAST_RECORDS_PER_KEY) + .map(|index| rust_card(index, NOW)) + .collect(); + let keyed = |records: &[SignedPodcastRecord]| -> Vec<_> { + records + .iter() + .map(|record| (vec![rust], record.clone())) + .collect() + }; + assert_eq!(store_all(&db, keyed(&older), NOW).await, half); + assert_eq!(store_all(&db, keyed(&younger), NOW + 1).await, half); + + // Newcomers keep getting in; each takes a random place of the younger + // half, the older half stays. + let mut newcomers = Vec::new(); + for index in 0..half { + let newcomer = rust_card(1_000 + index, NOW); + assert_eq!( + db.store(vec![(vec![rust], newcomer.clone())], NOW + 2 + index as u64) + .await + .unwrap(), 1, - NOW + index as u64, + "a full key admits a newcomer" ); - entries.push((vec![token_key], record)); + newcomers.push(newcomer); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(older.iter().all(|record| cached.contains(record))); + assert!(cached.contains(newcomers.last().unwrap())); } - let first = entries[0].1.clone(); - let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone(); - // The first publisher arrived first; the rest a moment later. - db.store(entries[..1].to_vec(), NOW).await.unwrap(); - for chunk in entries[1..].chunks(64) { - db.store(chunk.to_vec(), NOW + 1).await.unwrap(); - } - let cached = db - .records_by_key(token_key, Vec::new(), NOW, usize::MAX) - .await - .unwrap(); - assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); - assert!(cached.contains(&first)); - assert!(!cached.contains(&last)); - assert_eq!(cached[0], first, "records indexed longest come first"); - // The first publisher keeps refreshing its place. + // Owners refresh in place without evicting anybody. let refresh = card_with( - spec("show.xml", "Rust Radio"), + spec("show-0.xml", "Rust Radio"), &numbered_secret(0), 1, NOW + 1_000, ); assert_eq!( - db.store(vec![(vec![token_key], refresh)], NOW) + db.store(vec![(vec![rust], refresh.clone())], NOW + 1_000) .await .unwrap(), 1 ); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 1_000, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(cached.contains(&refresh)); +} + +#[tokio::test] +async fn sybils_planted_first_cannot_lock_a_late_card_out_of_a_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let rust = PodcastDhtKey::token(&network(), "rust"); + // 128 throwaway identities, each with its own podcast titled like the + // victim's, fill the key first. + let sybils: Vec<_> = (0..MAX_PODCAST_DHT_RECORDS) + .map(|index| (vec![rust], rust_card(index, NOW))) + .collect(); + store_all(&db, sybils, NOW).await; + let victim = card(&secret(1), 1); + assert_eq!( + db.store(vec![entry(&victim)], NOW + 1).await.unwrap(), + victim.dht_keys().len() + ); + let tokens = vec!["rust".to_string(), "radio".to_string()]; + let served = db + .records_by_key(rust, tokens.clone(), NOW + 1, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(served.len() <= MAX_PODCAST_DHT_RECORDS); + assert!(served.contains(&victim), "everything admitted is served"); + let hits = rank_hits(aggregate_hits(served), "Rust Radio", &tokens); + assert!( + hits.iter() + .any(|hit| hit.podcast_id == *victim.podcast_id()) + ); +} + +#[tokio::test] +async fn a_card_back_after_expiring_gets_into_a_cache_full_of_sybils() { + let limits = store::StoreLimits { + records: 16, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + let victim = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id()); + db.store(vec![entry(&victim)], NOW).await.unwrap(); + let sybils = |issued: u64| -> Vec<_> { + (0..16) + .map(|index| { + let record = rust_card(index, issued); + (record.dht_keys(), record) + }) + .collect() + }; + store_all(&db, sybils(NOW), NOW + 1).await; + assert_eq!(count_records(&temp).await, 16); + + // The victim's device is offline longer than a card lives; the sybils + // keep refreshing theirs in place. + let later = NOW + PODCAST_CARD_TTL.as_millis() as u64 + 3_600_000; + store_all(&db, sybils(later), later).await; + assert!( + db.records_by_key(id_key, Vec::new(), later, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap() + .is_empty() + ); + let back = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later + 1); + assert!(db.store(vec![entry(&back)], later + 1).await.unwrap() > 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), later + 1, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(), + vec![back] + ); + assert_eq!(count_records(&temp).await, 16); +} + +#[tokio::test] +async fn sybil_tombstones_planted_first_cannot_hide_a_revocation() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let victim_card = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim_card.podcast_id()); + let rust = PodcastDhtKey::token(&network(), "rust"); + // 64 sybils publish a card of the victim's podcast, then convert it to + // a tombstone in place, under the id key and the text keys. + for index in 0..64 { + let key = numbered_secret(index); + let sybil_card = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW); + db.store(vec![entry(&sybil_card)], NOW).await.unwrap(); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + sybil_card.podcast_id().clone(), + 2, + NOW, + sybil_card.dht_keys(), + ); + let tombstone = + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap(); + assert!(db.store(vec![entry(&tombstone)], NOW).await.unwrap() > 0); + } + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let tombstones = |key: &PodcastDhtKey| -> usize { + conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_keys WHERE dht_key = ?1 AND tombstone = 1", + [key.as_bytes().as_slice()], + |row| row.get::<_, i64>(0), + ) + .unwrap() as usize + }; + for key in [id_key, rust] { + assert_eq!( + tombstones(&key), + MAX_PODCAST_TOMBSTONES_PER_KEY, + "in-place conversions count against the tombstone budget" + ); + } + + // The honest publisher publishes later, then unpublishes. + assert!(db.store(vec![entry(&victim_card)], NOW + 1).await.unwrap() > 0); + let victim_tombstone = tombstone(&secret(1), 2, NOW + 2); + assert!( + db.store(vec![entry(&victim_tombstone)], NOW + 2) + .await + .unwrap() + > 0 + ); + for key in [id_key, rust] { + assert!(tombstones(&key) <= MAX_PODCAST_TOMBSTONES_PER_KEY); + let served = db + .records_by_key(key, vec![], NOW + 2, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!( + served.contains(&victim_tombstone), + "the revocation is served" + ); + // It overrules a stale replica still holding the card. + let mut all = served; + all.push(victim_card.clone()); + assert!(aggregate_hits(all).iter().all(|hit| { + hit.podcast_id != *victim_card.podcast_id() + || !hit.publishers.contains(&secret(1).public()) + })); + } } #[tokio::test] @@ -663,8 +854,8 @@ async fn sybil_tombstones_cannot_suppress_a_card() { written += db.store(chunk.to_vec(), NOW).await.unwrap(); } assert_eq!( - written, MAX_PODCAST_TOMBSTONES_PER_KEY, - "only the id key's tombstone budget is used" + written, 256, + "each sybil tombstone takes a young sybil tombstone's place under the id key" ); let by_id = db @@ -726,36 +917,67 @@ async fn cache_enforces_publisher_record_and_byte_quotas() { let temp = tempfile::tempdir().unwrap(); let db = open_db_with(&temp, limits).await; let show = |index: usize, key: &SecretKey| { - entry(&card_with( + card_with( spec(&format!("show-{index}.xml"), "Rust Radio"), key, 1, NOW, - )) + ) }; let heavy = secret(1); - let stored: Vec = { - let mut out = Vec::new(); - for index in 0..4 { - out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap()); - } - out - }; - assert!(stored[..3].iter().all(|written| *written > 0)); + for index in 0..3 { + let record = show(index, &heavy); + assert!( + db.store(vec![entry(&record)], NOW + index as u64) + .await + .unwrap() + > 0 + ); + } assert_eq!( - stored[3], 0, + db.store(vec![entry(&show(3, &heavy))], NOW + 3) + .await + .unwrap(), + 0, "the fourth podcast of one publisher is refused" ); // A refresh of an admitted record is always accepted. let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1); - assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0); - assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0); - assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0); - assert_eq!( - db.store(vec![show(12, &secret(4))], NOW).await.unwrap(), - 0, - "the cache holds at most five records" + assert!(db.store(vec![entry(&refresh)], NOW + 4).await.unwrap() > 0); + assert!( + db.store(vec![entry(&show(10, &secret(2)))], NOW + 10) + .await + .unwrap() + > 0 ); + assert!( + db.store(vec![entry(&show(11, &secret(3)))], NOW + 11) + .await + .unwrap() + > 0 + ); + assert_eq!(count_records(&temp).await, 5); + + // The cache is full: every newcomer still gets in by evicting a random + // record of the younger half; the two oldest records always stay. + let rust = PodcastDhtKey::token(&network(), "rust"); + for index in 0..6 { + let newcomer = show(20 + index, &secret(10 + index as u8)); + assert!( + db.store(vec![entry(&newcomer)], NOW + 20 + index as u64) + .await + .unwrap() + > 0 + ); + assert_eq!(count_records(&temp).await, 5); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 30, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(cached.contains(&refresh)); + assert!(cached.contains(&show(1, &heavy))); + assert!(cached.contains(&newcomer)); + } let one = card(&secret(1), 1); let limits = store::StoreLimits { @@ -764,25 +986,34 @@ async fn cache_enforces_publisher_record_and_byte_quotas() { }; let temp = tempfile::tempdir().unwrap(); let db = open_db_with(&temp, limits).await; - assert!( - db.store(vec![entry(&card(&secret(1), 1))], NOW) - .await - .unwrap() - > 0 - ); - assert!( - db.store(vec![entry(&card(&secret(2), 1))], NOW) - .await - .unwrap() - > 0 - ); + let id_key = PodcastDhtKey::podcast(&network(), one.podcast_id()); + for (index, now) in [(1, NOW), (2, NOW + 1), (3, NOW + 2)] { + assert!( + db.store(vec![entry(&card(&secret(index), 1))], now) + .await + .unwrap() + > 0 + ); + } + // The byte budget holds two cards: the younger half of the two others + // (the second card) made room for the third. assert_eq!( - db.store(vec![entry(&card(&secret(3), 1))], NOW) + db.records_by_key(id_key, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS) .await - .unwrap(), - 0, - "the byte budget is full" + .unwrap() + .into_iter() + .map(|record| record.publisher()) + .collect::>(), + HashSet::from([secret(1).public(), secret(3).public()]) ); + // A record larger than the whole budget evicts nothing. + let limits = store::StoreLimits { + bytes: one.encoded_len() - 1, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + assert_eq!(db.store(vec![entry(&one)], NOW).await.unwrap(), 0); } #[tokio::test] @@ -931,6 +1162,71 @@ fn replicas_only_accept_keys_they_are_close_to() { assert!(is_responsible(&own, &near[1..], &key)); } +fn contact_of(key: &SecretKey) -> NodeContact { + let peer = key.public(); + NodeContact { + node_id: NodeId::from_endpoint(&peer), + peer_id: peer, + ticket: String::new(), + last_seen_ms: NOW, + } +} + +#[test] +fn refused_placements_fall_back_to_the_next_closest_nodes() { + let key = PodcastDhtKey::token(&network(), "rust"); + let contacts: Vec = (0..3 * K) + .map(|index| contact_of(&numbered_secret(index))) + .collect(); + let mut by_distance = contacts.clone(); + by_distance.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + // This node is the farthest one: it is no replica and no fallback. + let own = by_distance.pop().unwrap(); + let others: Vec = by_distance.clone(); + let targets = replica_targets(own.clone(), others.clone(), &key); + assert!(!targets.local); + let ids = |contacts: &[NodeContact]| -> Vec { + contacts.iter().map(|contact| contact.peer_id).collect() + }; + assert_eq!(ids(&targets.primary), ids(&by_distance[..K])); + let mut fallback = targets.fallback.clone(); + fallback.reverse(); + assert_eq!(ids(&fallback), ids(&by_distance[K..2 * K])); + + // Two refusals of one placement go to the two next closest nodes; + // another record's refusal has its own fallbacks. + let mut fallbacks = HashMap::from([ + ((0, key), targets.fallback.clone()), + ((1, key), targets.fallback), + ]); + let retry = retry_plan(vec![(0, key), (0, key), (1, key)], &mut fallbacks); + let first = by_distance[K].peer_id; + let second = by_distance[K + 1].peer_id; + assert_eq!(retry.targets.len(), 2); + assert_eq!( + retry.targets[&first].1.keys().copied().collect::>(), + vec![0, 1] + ); + assert_eq!( + retry.targets[&second].1.keys().copied().collect::>(), + vec![0] + ); + assert_eq!(fallbacks[&(0, key)].len(), K - 2); + + // With this node among the closest it keeps a local replica and is never + // its own fallback. + let targets = replica_targets(by_distance[0].clone(), others, &key); + assert!(targets.local); + assert_eq!(targets.primary.len(), K - 1); + assert!( + targets + .fallback + .iter() + .chain(&targets.primary) + .all(|contact| contact.peer_id != by_distance[0].peer_id) + ); +} + #[test] fn inbound_requests_are_limited_per_peer() { let inflight = Arc::new(Mutex::new(HashMap::new())); @@ -946,21 +1242,50 @@ fn inbound_requests_are_limited_per_peer() { } #[test] -fn lookups_keep_a_bounded_number_of_owners() { +fn lookups_give_each_response_a_bounded_share_of_owners() { let mut winners = HashMap::new(); let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id()); - let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS) - .map(|index| { - card_with( - spec("show.xml", "Rust Radio"), - &numbered_secret(index), - 1, - now_ms(), - ) - }) - .collect(); - let first = records[0].clone(); - merge_records(&mut winners, records, &network(), &id_key, &[]); + let response = |first: usize| -> Vec { + (first..first + MAX_PODCAST_DHT_RECORDS) + .map(|index| { + card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + now_ms(), + ) + }) + .collect() + }; + // Four attacker responders answer first, each with 128 sybil owners. + for responder in 0..4 { + merge_records( + &mut winners, + response(10_000 + responder * MAX_PODCAST_DHT_RECORDS), + &network(), + &id_key, + &[], + ); + } + assert_eq!(winners.len(), 4 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE); + // An honest replica answering later still contributes its owners. + let honest = response(0); + let victim = honest[0].clone(); + merge_records(&mut winners, honest, &network(), &id_key, &[]); + let owner = (victim.podcast_id().clone(), victim.publisher()); + assert_eq!(winners[&owner], victim); + assert_eq!(winners.len(), 5 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE); + + // The lookup as a whole stays bounded. + for responder in 0..8 { + merge_records( + &mut winners, + response(20_000 + responder * MAX_PODCAST_DHT_RECORDS), + &network(), + &id_key, + &[], + ); + } assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS); // A newer record of a known owner still replaces its winner. let newer = card_with( @@ -970,7 +1295,6 @@ fn lookups_keep_a_bounded_number_of_owners() { now_ms(), ); merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]); - let owner = (first.podcast_id().clone(), first.publisher()); assert_eq!(winners[&owner], newer); } diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs index b3ba983..395f3d5 100644 --- a/crates/music-dht/tests/podcast_dht.rs +++ b/crates/music-dht/tests/podcast_dht.rs @@ -18,6 +18,8 @@ //! and is never asked for records or to store them. //! * `sync_shares` stays fail-safe for revocation and holds back URLs that //! look private (single node). +//! * Contacts a peer returns are never trusted: fabricated, unreachable +//! contacts do not enter the routing table (two nodes). use std::path::Path; use std::sync::Arc; @@ -566,3 +568,91 @@ async fn single_node_scenario() { a.stop().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fabricated_contacts_never_join_the_overlay() { + tokio::time::timeout(TEST_TIMEOUT, fabricated_contacts_scenario()) + .await + .unwrap(); +} + +/// K contacts whose node ids are genuine hashes of their endpoint ids but +/// whose tickets carry no address, as a malicious peer could return them. +fn fabricated_contacts(network: NetworkId) -> Vec { + (0..music_dht::K) + .map(|index| { + let peer = federation_net::SecretKey::from_bytes(&[index as u8 + 101; 32]).public(); + let ticket = federation_net::PeerTicket { + ticket_version: federation_net::TICKET_VERSION, + protocol_version: 1, + network_id: network, + schema_id: federation_net::SchemaId::from_bytes([0; 32]), + endpoint_addr: federation_net::EndpointAddr::new(peer), + }; + NodeContact { + node_id: NodeId::from_endpoint(&peer), + peer_id: peer, + ticket: ticket.to_string(), + last_seen_ms: u64::MAX, + } + }) + .collect() +} + +async fn fabricated_contacts_scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-fabricated-contacts"); + let a = Node::start(&temp.path().join("a"), network).await; + // M speaks the overlay ALPN but answers every request with fabricated + // contacts. + let (m, m_events) = start_service(&temp.path().join("m"), network, true).await; + let mut acceptor = m.stream_acceptor(PODCAST_DHT_ALPN).unwrap(); + let fabricated = fabricated_contacts(network); + let response = PodcastDhtResponse { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: true, + error: None, + nodes: fabricated.clone(), + records: Vec::new(), + stored: 0, + }; + let payload = postcard::to_stdvec(&response).unwrap(); + let server = tokio::spawn(async move { + while let Some(mut stream) = acceptor.accept().await { + let mut request = Vec::new(); + let _ = AsyncReadExt::read_to_end( + &mut (&mut stream.recv).take(MAX_PODCAST_DHT_REQUEST_BYTES as u64), + &mut request, + ) + .await; + let _ = stream.send.write_all(&payload).await; + let _ = stream.send.finish(); + let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await; + } + }); + m.connect(a.service.ticket().await.unwrap()).await.unwrap(); + wait_for("music DHT contact", async || { + (!a.service.known_peers().is_empty()).then_some(()) + }) + .await; + + // Bootstrap learns M, which answered, and none of the contacts it made + // up: they are tried once and never answer. + a.dht.bootstrap().await.unwrap(); + assert_eq!(a.dht.known_peers(), 1); + // A lookup treats them as candidates only. + let podcast_id = PodcastId::from_feed_url(&feed("nothing.xml")); + assert!(a.dht.lookup_podcast(&podcast_id).await.unwrap().is_none()); + assert_eq!(a.dht.known_peers(), 1); + a.dht.bootstrap().await.unwrap(); + assert_eq!( + a.dht.known_peers(), + 1, + "fabricated contacts never count for replica responsibility" + ); + + server.abort(); + a.stop().await; + m_events.abort(); + m.shutdown().await.unwrap(); +}