diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs index f71554f..ce8d5ba 100644 --- a/crates/music-dht/src/podcast_dht/mod.rs +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -124,6 +124,11 @@ pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8; /// Maximum distinct `(podcast_id, publisher)` records one lookup keeps; /// records of further owners are ignored. pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512; +/// Most new `(podcast_id, publisher)` owners one response (or the local +/// cache) may add to a lookup: a `1 / K` share of +/// [`MAX_PODCAST_LOOKUP_RECORDS`], so the first responders cannot fill the +/// lookup and crowd out later ones. +pub const MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE: usize = MAX_PODCAST_LOOKUP_RECORDS / K; /// A node stores a placement only when fewer than this many of its known /// overlay contacts are closer to the key than itself (`2 * K`: the `K` /// closest with slack for differing routing views). @@ -141,6 +146,9 @@ const SERVE_TIMEOUT: Duration = Duration::from_secs(10); /// A peer that failed to answer (network error or timeout) is skipped by /// lookups, publication and bootstrap for this long. const FAILED_PEER_BACKOFF: Duration = Duration::from_secs(30); +/// A contact that failed this many times in a row (each after its backoff) +/// is removed from the routing table and the persisted peers. +const MAX_CONSECUTIVE_FAILURES: u32 = 3; const MAX_TICKET_BYTES: usize = 16 * 1024; const MAX_ERROR_BYTES: usize = 1024; @@ -354,6 +362,7 @@ pub struct PodcastDht { routing: Mutex, unsupported: Mutex>, failed: Mutex>, + failures: Mutex>, publish_gate: Semaphore, serve_limit: Arc, inflight: Arc>>, @@ -401,6 +410,7 @@ impl PodcastDht { routing: Mutex::new(routing), unsupported: Mutex::new(HashMap::new()), failed: Mutex::new(HashMap::new()), + failures: Mutex::new(HashMap::new()), publish_gate: Semaphore::new(1), serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), inflight: Arc::new(Mutex::new(HashMap::new())), @@ -481,9 +491,12 @@ impl PodcastDht { /// A peer whose manifest lacks [`PODCAST_DHT_ID`] at /// [`PODCAST_DHT_PROTOCOL_VERSION`] is removed from the overlay routing /// table and not contacted for [`UNSUPPORTED_PEER_BACKOFF`]. Peers only - /// enter the routing table after answering on [`PODCAST_DHT_ALPN`] or - /// being returned by an overlay peer, so a peer without the overlay is - /// never queried for records or asked to store them. + /// enter the routing table after answering a request on + /// [`PODCAST_DHT_ALPN`] or sending one as an authenticated requester, so + /// a peer without the overlay is never asked to store records and never + /// counts for replica responsibility. A contact another peer returns is + /// only a lookup candidate until it answers itself; a candidate that does + /// not answer is backed off like any failed peer. pub async fn note_peer_capabilities( &self, peer: EndpointId, @@ -500,10 +513,11 @@ impl PodcastDht { } /// Discovers overlay peers among the routing table and the music DHT - /// contacts with one `FindNode` each. Music-DHT contacts whose capability - /// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are - /// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become - /// overlay contacts when they answer. + /// contacts with one `FindNode` each, then asks the not yet known contacts + /// they return once more. Music-DHT contacts whose capability manifest + /// lacks the overlay ([`Self::note_peer_capabilities`]) are skipped; + /// others are dialed on [`PODCAST_DHT_ALPN`]. A peer only becomes an + /// overlay contact when it answers. pub async fn bootstrap(&self) -> Result<()> { let known_feature: HashSet = lock(&self.routing) .contacts() @@ -517,25 +531,19 @@ impl PodcastDht { .into_iter() .filter(|contact| !known_feature.contains(&contact.peer_id)), ); - let mut seen = HashSet::new(); + let mut seen: HashSet = known_feature; + seen.insert(self.own_contact.peer_id); + let mut queried = HashSet::new(); candidates.retain(|contact| { contact.peer_id != self.own_contact.peer_id && !self.is_unsupported(&contact.peer_id) && !self.is_backed_off(&contact.peer_id) - && seen.insert(contact.peer_id) + && queried.insert(contact.peer_id) }); + seen.extend(queried); candidates.truncate(MAX_BOOTSTRAP_PEERS); - let target = *self.own_contact.node_id.as_bytes(); - let responses = stream::iter(candidates.into_iter().map(|contact| async move { - let response = self - .exchange(&contact, PodcastDhtOperation::FindNode { target }) - .await; - (contact, response) - })) - .buffer_unordered(BOOTSTRAP_CONCURRENCY) - .collect::>() - .await; - for (contact, response) in responses { + let mut discovered = Vec::new(); + for (contact, response) in self.find_own_node(candidates).await { let response = match response { Ok(response) => response, Err(error) => { @@ -544,11 +552,46 @@ impl PodcastDht { } }; self.learn_contact(contact).await?; - self.learn_response_nodes(response.nodes).await; + for node in response.nodes { + if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)).is_ok() + && !self.is_unsupported(&node.peer_id) + && !self.is_backed_off(&node.peer_id) + && seen.insert(node.peer_id) + { + discovered.push(node); + } + } + } + // Returned contacts are unverified: they join only by answering. + discovered.truncate(MAX_BOOTSTRAP_PEERS); + for (contact, response) in self.find_own_node(discovered).await { + match response { + Ok(_) => self.learn_contact(contact).await?, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT contact did not answer"); + } + } } Ok(()) } + /// One `FindNode` for this node's id to each contact. + async fn find_own_node( + &self, + contacts: Vec, + ) -> Vec<(NodeContact, Result)> { + let target = *self.own_contact.node_id.as_bytes(); + stream::iter(contacts.into_iter().map(|contact| async move { + let response = self + .exchange(&contact, PodcastDhtOperation::FindNode { target }) + .await; + (contact, response) + })) + .buffer_unordered(BOOTSTRAP_CONCURRENCY) + .collect::>() + .await + } + /// Publishes (or refreshes) this device's card of a podcast. /// /// Refused with [`MusicDhtError::PodcastPublication`] unless @@ -892,7 +935,10 @@ impl PodcastDht { } /// Stores records on the `K` closest overlay nodes of each of their keys - /// (including this node when it is among them). + /// (including this node when it is among them). When a remote node + /// refuses a batch (`stored = 0`, for example because it does not count + /// itself responsible for the keys) or fails, its records are offered + /// once to the next closest nodes of the same keys, one per refusal. async fn push(&self, records: Vec) -> Result { let mut stats = PodcastDhtPublishStats { records: records.len(), @@ -902,69 +948,105 @@ impl PodcastDht { return Ok(stats); } let mut local = Vec::new(); - let mut remote: HashMap)> = HashMap::new(); - for record in records { + let mut plan = StorePlan::new(); + let mut fallbacks: HashMap<(usize, PodcastDhtKey), Vec> = HashMap::new(); + for (index, record) in records.iter().enumerate() { let mut local_keys = Vec::new(); - let mut remote_keys: HashMap)> = - HashMap::new(); for key in record.dht_keys() { stats.keys += 1; - let (store_local, contacts) = self.replica_targets(&key); - if store_local { + let targets = self.replica_targets(&key); + if targets.local { local_keys.push(key); } - for contact in contacts { - remote_keys - .entry(contact.peer_id) - .or_insert_with(|| (contact, Vec::new())) - .1 - .push(key); + for contact in targets.primary { + plan.add(contact, index, key); } + fallbacks.insert((index, key), targets.fallback); } if !local_keys.is_empty() { local.push((local_keys, record.clone())); } - for (peer, (contact, keys)) in remote_keys { - remote - .entry(peer) - .or_insert_with(|| (contact, Vec::new())) - .1 - .push(PodcastStoreEntry { - keys, - record: record.clone(), - }); - } } if !local.is_empty() { self.database.store(local, now_ms()).await?; stats.local_replica = true; } - let sends = stream::iter(remote.into_values().map(|(contact, entries)| async move { - let mut accepted = false; - for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { - let response = self - .exchange( - &contact, - PodcastDhtOperation::StoreBatch { - records: batch.to_vec(), + let (mut accepted, refused) = self.send_store_plan(&records, plan).await; + let retry = retry_plan(refused, &mut fallbacks); + if !retry.is_empty() { + accepted.extend(self.send_store_plan(&records, retry).await.0); + } + stats.remote_nodes = accepted.len(); + Ok(stats) + } + + /// Sends a store plan. Returns the nodes that stored at least one + /// placement and the `(record index, key)` placements offered in batches + /// that were refused or failed. + async fn send_store_plan( + &self, + records: &[SignedPodcastRecord], + plan: StorePlan, + ) -> (HashSet, Vec<(usize, PodcastDhtKey)>) { + let sends = stream::iter(plan.targets.into_values().map(|(contact, entries)| async move { + let entries: Vec<(usize, PodcastStoreEntry)> = entries + .into_iter() + .map(|(index, keys)| { + ( + index, + PodcastStoreEntry { + keys, + record: records[index].clone(), }, ) - .await; - match response { - Ok(response) => accepted |= response.stored > 0, - Err(error) => { - debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); - break; + }) + .collect(); + let mut accepted = false; + let mut refused = Vec::new(); + let mut failed = false; + for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + let stored = if failed { + false + } else { + let response = self + .exchange( + &contact, + PodcastDhtOperation::StoreBatch { + records: batch.iter().map(|(_, entry)| entry.clone()).collect(), + }, + ) + .await; + match response { + Ok(response) => response.stored > 0, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); + failed = true; + false + } + } + }; + if stored { + accepted = true; + } else { + for (index, entry) in batch { + refused.extend(entry.keys.iter().map(|key| (*index, *key))); } } } - accepted + (contact.peer_id, accepted, refused) })) .buffer_unordered(PUBLISH_CONCURRENCY) .collect::>() .await; - stats.remote_nodes = sends.into_iter().filter(|accepted| *accepted).count(); - Ok(stats) + let mut accepted = HashSet::new(); + let mut refused = Vec::new(); + for (peer, stored, keys) in sends { + if stored { + accepted.insert(peer); + } + refused.extend(keys); + } + (accepted, refused) } /// Iterative lookup of `key`: queries the `K` closest known overlay @@ -1013,16 +1095,23 @@ impl PodcastDht { filter_tokens: filter_tokens.clone(), }; let mut futures: FuturesUnordered<_> = batch - .iter() + .into_iter() .map(|contact| { let operation = operation.clone(); - async move { self.exchange(contact, operation).await } + async move { + let response = self.exchange(&contact, operation).await; + (contact, response) + } }) .collect(); - while let Ok(Some(response)) = tokio::time::timeout_at(deadline, futures.next()).await { + while let Ok(Some((contact, response))) = + tokio::time::timeout_at(deadline, futures.next()).await + { let Ok(response) = response else { continue; }; + // It answered on the overlay: now it is a verified contact. + self.learn_contact(contact).await?; merge_records( &mut winners, response.records, @@ -1031,12 +1120,12 @@ impl PodcastDht { &filter_tokens, ); for node in response.nodes { + // Returned contacts are only candidates until they answer. if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)) .is_ok() && !self.is_unsupported(&node.peer_id) && known.insert(node.peer_id) { - self.learn_contact(node.clone()).await?; candidates.push(node); } } @@ -1098,9 +1187,11 @@ impl PodcastDht { PodcastDhtOperation::StoreBatch { records } => { let entries = validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?; + // Only verified contacts that are not backed off count. let nodes: Vec = lock(&self.routing) .contacts() .into_iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) .map(|contact| contact.node_id) .collect(); let entries: Vec<_> = entries @@ -1137,10 +1228,21 @@ impl PodcastDht { match &result { Ok(_) => { lock(&self.failed).remove(&contact.peer_id); + lock(&self.failures).remove(&contact.peer_id); } Err(MusicDhtError::Network(_) | MusicDhtError::Timeout) => { let until = now_ms().saturating_add(FAILED_PEER_BACKOFF.as_millis() as u64); lock(&self.failed).insert(contact.peer_id, until); + let failures = { + let mut failures = lock(&self.failures); + let count = failures.entry(contact.peer_id).or_default(); + *count += 1; + *count + }; + if failures >= MAX_CONSECUTIVE_FAILURES { + lock(&self.failures).remove(&contact.peer_id); + self.forget_contact(contact.peer_id).await; + } } Err(_) => {} } @@ -1180,6 +1282,16 @@ impl PodcastDht { Ok(response) } + /// Removes a contact that keeps failing from routing and the cache. + async fn forget_contact(&self, peer: EndpointId) { + if lock(&self.routing).remove(&peer) { + debug!(%peer, "podcast DHT contact dropped after repeated failures"); + } + if let Err(error) = self.database.remove_peer(peer).await { + debug!(%peer, %error, "podcast DHT contact removal failed"); + } + } + fn is_unsupported(&self, peer: &EndpointId) -> bool { still_listed(&self.unsupported, peer) } @@ -1188,21 +1300,15 @@ impl PodcastDht { still_listed(&self.failed, peer) } - fn replica_targets(&self, key: &PodcastDhtKey) -> (bool, Vec) { - let mut contacts: Vec<_> = lock(&self.routing) + /// Replica targets of `key` among the contacts that are not backed off + /// ([`replica_targets`]). + fn replica_targets(&self, key: &PodcastDhtKey) -> ReplicaTargets { + let contacts: Vec<_> = lock(&self.routing) .contacts() .into_iter() .filter(|contact| !self.is_backed_off(&contact.peer_id)) .collect(); - contacts.push(self.fresh_own_contact()); - contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); - contacts.dedup_by_key(|contact| contact.peer_id); - contacts.truncate(K); - let local = contacts - .iter() - .any(|contact| contact.peer_id == self.own_contact.peer_id); - contacts.retain(|contact| contact.peer_id != self.own_contact.peer_id); - (local, contacts) + replica_targets(self.fresh_own_contact(), contacts, key) } fn closest_contacts(&self, target: &[u8; 32], count: usize) -> Vec { @@ -1220,17 +1326,6 @@ impl PodcastDht { contact } - async fn learn_response_nodes(&self, nodes: Vec) { - for contact in nodes { - if self.is_unsupported(&contact.peer_id) { - continue; - } - if let Err(error) = self.learn_contact(contact).await { - debug!(%error, "invalid podcast DHT contact ignored"); - } - } - } - async fn learn_contact(&self, mut contact: NodeContact) -> Result<()> { validate_contact(&contact, &self.network_id, Some(self.own_contact.peer_id))?; contact.node_id = NodeId::from_endpoint(&contact.peer_id); @@ -1309,8 +1404,12 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> { Ok(()) } -/// Adds verified records found under `key` to the per-owner winners, keeping -/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners. +/// Adds verified records of one response (or the local cache) found under +/// `key` to the per-owner winners. A newer record of a known owner always +/// replaces its winner; the response may add at most +/// [`MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE`] new owners, and the lookup +/// keeps at most [`MAX_PODCAST_LOOKUP_RECORDS`]. So the order in which +/// responders answer does not let the first ones take every slot. fn merge_records( winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, records: Vec, @@ -1319,6 +1418,7 @@ fn merge_records( filter_tokens: &[String], ) { let now = now_ms(); + let mut added = 0usize; for record in records { if record.verify_at(network_id, now).is_err() || !record.belongs_under(key) { continue; @@ -1332,14 +1432,99 @@ fn merge_records( let owner = (record.podcast_id().clone(), record.publisher()); match winners.get(&owner) { Some(current) if compare_records(current, &record).is_ge() => {} - None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {} - _ => { + Some(_) => { winners.insert(owner, record); } + None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS + || added >= MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE => {} + None => { + winners.insert(owner, record); + added += 1; + } } } } +/// Replica nodes of one key, from [`replica_targets`]. +#[derive(Debug)] +struct ReplicaTargets { + /// Whether this node is among the `K` closest. + local: bool, + /// The other nodes among the `K` closest. + primary: Vec, + /// The next up to `K` nodes, farthest first (popped closest first). + fallback: Vec, +} + +/// The `K` closest nodes of `key` among `contacts` and `own` (this node), and +/// the next `K` as fallbacks. +fn replica_targets( + own: NodeContact, + mut contacts: Vec, + key: &PodcastDhtKey, +) -> ReplicaTargets { + let own_peer = own.peer_id; + contacts.push(own); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + contacts.dedup_by_key(|contact| contact.peer_id); + let mut fallback = contacts.split_off(K.min(contacts.len())); + fallback.retain(|contact| contact.peer_id != own_peer); + fallback.truncate(K); + fallback.reverse(); + let local = contacts.iter().any(|contact| contact.peer_id == own_peer); + contacts.retain(|contact| contact.peer_id != own_peer); + ReplicaTargets { + local, + primary: contacts, + fallback, + } +} + +/// One fallback node per refused `(record index, key)` placement: the +/// closest one not yet offered that placement. +fn retry_plan( + refused: Vec<(usize, PodcastDhtKey)>, + fallbacks: &mut HashMap<(usize, PodcastDhtKey), Vec>, +) -> StorePlan { + let mut retry = StorePlan::new(); + for (index, key) in refused { + if let Some(contact) = fallbacks + .get_mut(&(index, key)) + .and_then(|fallback| fallback.pop()) + { + retry.add(contact, index, key); + } + } + retry +} + +/// Placements to offer per remote node: record index and keys. +struct StorePlan { + targets: HashMap>)>, +} + +impl StorePlan { + fn new() -> Self { + Self { + targets: HashMap::new(), + } + } + + fn add(&mut self, contact: NodeContact, index: usize, key: PodcastDhtKey) { + self.targets + .entry(contact.peer_id) + .or_insert_with(|| (contact, BTreeMap::new())) + .1 + .entry(index) + .or_default() + .push(key); + } + + fn is_empty(&self) -> bool { + self.targets.is_empty() + } +} + /// One in-flight inbound request of a peer; released on drop. struct PeerSlot { inflight: Arc>>, diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs index acaedbc..1a86e82 100644 --- a/crates/music-dht/src/podcast_dht/store.rs +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -17,33 +17,53 @@ use crate::error::{MusicDhtError, Result}; use crate::podcast::PodcastId; use crate::routing::{NodeContact, NodeId}; +use super::MAX_PODCAST_DHT_RECORDS; use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records}; /// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1 /// (pre-release) stored one payload per key; its replica table is dropped on /// open because the cache is replaceable. Own publications are kept. pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2; -/// Maximum records (cards and tombstones) indexed under one key. A full key -/// admits no new `(podcast_id, publisher)` until a record expires: existing -/// records are never evicted for newcomers, and their owners keep refreshing -/// them in place. -pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; -/// Maximum tombstones newly indexed under one key. A tombstone is admitted -/// under a text key only in place of a record of the same -/// `(podcast_id, publisher)` already indexed there, so this budget applies to -/// the podcast-id key, where tombstones of unknown cards land. -pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32; +/// Maximum records (cards and tombstones) indexed under one key. It equals +/// [`MAX_PODCAST_DHT_RECORDS`], so one `FindValue` response carries every +/// record of a key: nothing admitted is unreachable. +/// +/// A full key admits a newcomer by evicting a random placement from its +/// younger half (by first indexing time). The older half is never evicted +/// for newcomers, so cards that were there before a flood stay, and the +/// younger half stays contestable, so nobody who stored first holds the key +/// for good. +pub const MAX_PODCAST_RECORDS_PER_KEY: usize = MAX_PODCAST_DHT_RECORDS; +/// Maximum tombstones indexed under one key, including tombstones that +/// replaced their owner's card in place. Over budget, a random tombstone of +/// another owner from the younger half of the key's tombstones is evicted. +/// It equals [`MAX_PODCAST_RESPONSE_TOMBSTONES`], so every tombstone of a key +/// is served. +pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = MAX_PODCAST_RESPONSE_TOMBSTONES; /// Maximum tombstones in one `FindValue` response; cards fill the rest. pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32; /// Maximum `(podcast_id, publisher)` records one publisher device keeps in a -/// replica. +/// replica. A publisher over it is refused (nothing is evicted for it). pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256; -/// Maximum `(podcast_id, publisher)` records in a replica cache. +/// Maximum `(podcast_id, publisher)` records in a replica cache. A full cache +/// admits a new record by evicting random records from the younger half of +/// the cache (by first indexing time), like a full key. pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096; -/// Maximum total encoded size of the records in a replica cache. +/// Maximum total encoded size of the records in a replica cache, enforced +/// like [`MAX_PODCAST_CACHE_RECORDS`]. pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024; /// Maximum persisted overlay contacts; the least recently seen are dropped. pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024; +/// Maximum records evicted from the cache to admit one record; when that is +/// not enough room, the record is refused and nothing is evicted. +const MAX_EVICTIONS_PER_RECORD: usize = 16; + +const _: () = assert!( + MAX_PODCAST_RECORDS_PER_KEY <= MAX_PODCAST_DHT_RECORDS + && MAX_PODCAST_TOMBSTONES_PER_KEY <= MAX_PODCAST_RESPONSE_TOMBSTONES + && MAX_PODCAST_RESPONSE_TOMBSTONES <= MAX_PODCAST_DHT_RECORDS, + "every record admitted under a key must fit one FindValue response" +); const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS podcast_dht_records ( @@ -227,7 +247,8 @@ impl PodcastDhtDatabase { /// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale /// replicas elsewhere can be overruled) and cards containing every token /// in `filter_tokens` up to `limit` records in all. Records indexed longest - /// come first. Cards precede tombstones. + /// come first. Cards precede tombstones. With the default limits and + /// `limit` = [`MAX_PODCAST_DHT_RECORDS`], every record of the key fits. pub async fn records_by_key( &self, key: PodcastDhtKey, @@ -475,10 +496,10 @@ fn store_one( }; let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; if replace - && !admits_record( + && !make_room_for_record( conn, limits, - &publisher, + (&podcast_id, &publisher), existing.is_none(), existing_bytes, payload.len(), @@ -536,6 +557,14 @@ fn store_one( clamp_i64(record.expires_at_ms()), ], )?; + // A card converted to a tombstone in place counts against the + // tombstone budget of each key it stays under, like a new one. + if record.is_tombstone() { + for key in &indexed { + let key = PodcastDhtKey::from_bytes(*key); + make_room_under_key(conn, limits, &key, true, (&podcast_id, &publisher), now, 0)?; + } + } } // New placements. A tombstone only takes a new place under the podcast-id @@ -549,7 +578,15 @@ fn store_one( if record.is_tombstone() && *key != id_key { continue; } - if !admits_placement(conn, limits, key, record.is_tombstone(), now)? { + if !make_room_under_key( + conn, + limits, + key, + record.is_tombstone(), + (&podcast_id, &publisher), + now, + 1, + )? { continue; } conn.execute( @@ -596,58 +633,197 @@ fn store_one( Ok(written) } -/// Whether the cache may hold the new version of a record: per-publisher and -/// global record counts for a new owner, and the global byte budget. -fn admits_record( +/// Makes room in the cache for the new version of a record of `owner`. +/// +/// * A publisher at [`StoreLimits::per_publisher`] gets no new record. +/// * When the record count (for a new owner) or the byte budget would be +/// exceeded, random records of other owners from the younger half of the +/// cache (by first indexing time) are evicted, at most +/// [`MAX_EVICTIONS_PER_RECORD`]. If that is not enough, nothing is evicted +/// and the record is refused. +fn make_room_for_record( conn: &Connection, limits: &StoreLimits, - publisher: &str, + owner: (&str, &str), new_owner: bool, previous_bytes: usize, new_bytes: usize, now: i64, ) -> rusqlite::Result { + if new_bytes > limits.bytes { + return Ok(false); + } + if new_owner { + let own: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", + params![owner.1, now], + |row| row.get(0), + )?; + if own.max(0) as usize >= limits.per_publisher { + return Ok(false); + } + } let (records, bytes): (i64, i64) = conn.query_row( "SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records WHERE expires_at_ms > ?1", [now], |row| Ok((row.get(0)?, row.get(1)?)), )?; - let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); - if bytes.saturating_add(new_bytes) > limits.bytes { - return Ok(false); - } - if !new_owner { + let mut records = records.max(0) as usize; + let mut bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); + let fits = |records: usize, bytes: usize| { + bytes.saturating_add(new_bytes) <= limits.bytes && (!new_owner || records < limits.records) + }; + if fits(records, bytes) { return Ok(true); } - if records.max(0) as usize >= limits.records { + + // Candidates: the younger half of the other owners' records, shuffled. + let others = records.saturating_sub(usize::from(!new_owner)); + let mut statement = conn.prepare( + "SELECT podcast_id, publisher, payload_bytes FROM ( + SELECT r.podcast_id, r.publisher, r.payload_bytes, + (SELECT MIN(k.first_stored_at_ms) FROM podcast_dht_keys k + WHERE k.podcast_id = r.podcast_id AND k.publisher = r.publisher) AS since + FROM podcast_dht_records r + WHERE r.expires_at_ms > ?1 AND NOT (r.podcast_id = ?2 AND r.publisher = ?3) + ORDER BY since DESC, random() + LIMIT ?4) + ORDER BY random() LIMIT ?5", + )?; + let candidates = statement + .query_map( + params![ + now, + owner.0, + owner.1, + others.div_ceil(2) as i64, + MAX_EVICTIONS_PER_RECORD as i64 + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + )? + .collect::>>()?; + let mut evict = Vec::new(); + for (podcast_id, publisher, payload_bytes) in candidates { + if fits(records, bytes) { + break; + } + records = records.saturating_sub(1); + bytes = bytes.saturating_sub(payload_bytes.max(0) as usize); + evict.push((podcast_id, publisher)); + } + if !fits(records, bytes) { return Ok(false); } - let own: i64 = conn.query_row( - "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", - params![publisher, now], - |row| row.get(0), - )?; - Ok((own.max(0) as usize) < limits.per_publisher) + for (podcast_id, publisher) in evict { + delete_owner(conn, &podcast_id, &publisher)?; + } + Ok(true) } -/// Whether `key` has room for one more placement. Existing placements are -/// never evicted. -fn admits_placement( +/// Makes room under `key` for `incoming` more placements (0 when an existing +/// placement of `owner` was converted to a tombstone in place) and returns +/// whether the key has room. +/// +/// Over [`StoreLimits::per_key`], or over [`StoreLimits::tombstones_per_key`] +/// for a tombstone, a random placement of another owner from the younger +/// half of the key (by first indexing time; only tombstones for the +/// tombstone budget) is evicted. The older half is never evicted for a +/// newcomer. +fn make_room_under_key( conn: &Connection, limits: &StoreLimits, key: &PodcastDhtKey, tombstone: bool, + owner: (&str, &str), now: i64, + incoming: usize, ) -> rusqlite::Result { - let (total, tombstones): (i64, i64) = conn.query_row( - "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys - WHERE dht_key = ?1 AND expires_at_ms > ?2", - params![key.as_bytes().as_slice(), now], - |row| Ok((row.get(0)?, row.get(1)?)), + for _ in 0..=MAX_EVICTIONS_PER_RECORD { + let (total, tombstones): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys + WHERE dht_key = ?1 AND expires_at_ms > ?2", + params![key.as_bytes().as_slice(), now], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let over_tombstones = + tombstone && tombstones.max(0) as usize + incoming > limits.tombstones_per_key; + let over_total = total.max(0) as usize + incoming > limits.per_key; + if !over_tombstones && !over_total { + return Ok(true); + } + let Some((podcast_id, publisher)) = + young_placement(conn, key, over_tombstones, owner, now)? + else { + return Ok(false); + }; + conn.execute( + "DELETE FROM podcast_dht_keys + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_bytes().as_slice(), podcast_id, publisher], + )?; + let placed: bool = conn.query_row( + "SELECT EXISTS (SELECT 1 FROM podcast_dht_keys + WHERE podcast_id = ?1 AND publisher = ?2)", + params![podcast_id, publisher], + |row| row.get(0), + )?; + if !placed { + delete_owner(conn, &podcast_id, &publisher)?; + } + } + Ok(false) +} + +/// A random unexpired placement of another owner than `owner` from the +/// younger half of `key` (only tombstones with `tombstones_only`). +fn young_placement( + conn: &Connection, + key: &PodcastDhtKey, + tombstones_only: bool, + owner: (&str, &str), + now: i64, +) -> rusqlite::Result> { + let filter = "dht_key = ?1 AND expires_at_ms > ?2 AND (?3 = 0 OR tombstone = 1) + AND NOT (podcast_id = ?4 AND publisher = ?5)"; + let count: i64 = conn.query_row( + &format!("SELECT COUNT(*) FROM podcast_dht_keys WHERE {filter}"), + params![ + key.as_bytes().as_slice(), + now, + tombstones_only, + owner.0, + owner.1 + ], + |row| row.get(0), )?; - Ok((total.max(0) as usize) < limits.per_key - && (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key)) + if count <= 0 { + return Ok(None); + } + conn.query_row( + &format!( + "SELECT podcast_id, publisher FROM ( + SELECT podcast_id, publisher FROM podcast_dht_keys WHERE {filter} + ORDER BY first_stored_at_ms DESC, random() LIMIT ?6) + ORDER BY random() LIMIT 1" + ), + params![ + key.as_bytes().as_slice(), + now, + tombstones_only, + owner.0, + owner.1, + (count as usize).div_ceil(2) as i64 + ], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() } fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> { diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs index 034f6c3..316bbb5 100644 --- a/crates/music-dht/src/podcast_dht/tests.rs +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -1,6 +1,8 @@ //! Unit tests of the podcast-discovery overlay: records, ordering, cache and //! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`. +use std::collections::HashSet; + use federation_net::{NetworkId, SecretKey}; use crate::routing::NodeId; @@ -584,50 +586,239 @@ async fn open_db_with( .unwrap() } +fn rust_card(index: usize, issued: u64) -> SignedPodcastRecord { + card_with( + spec(&format!("show-{index}.xml"), "Rust Radio"), + &numbered_secret(index), + 1, + issued, + ) +} + +async fn store_all( + db: &store::PodcastDhtDatabase, + entries: Vec<(Vec, SignedPodcastRecord)>, + now: u64, +) -> usize { + let mut written = 0; + for chunk in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + written += db.store(chunk.to_vec(), now).await.unwrap(); + } + written +} + +async fn count_records(temp: &tempfile::TempDir) -> i64 { + rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")) + .unwrap() + .query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| { + row.get(0) + }) + .unwrap() +} + #[tokio::test] -async fn cache_full_key_keeps_its_records_and_refuses_newcomers() { +async fn cache_full_key_admits_newcomers_from_its_younger_half() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; - let token_key = PodcastDhtKey::token(&network(), "rust"); - let mut entries = Vec::new(); - for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { - // Later publishers expire later; that buys them nothing. - let record = card_with( - spec("show.xml", "Rust Radio"), - &numbered_secret(index), + let rust = PodcastDhtKey::token(&network(), "rust"); + let half = MAX_PODCAST_RECORDS_PER_KEY / 2; + let older: Vec<_> = (0..half).map(|index| rust_card(index, NOW)).collect(); + let younger: Vec<_> = (half..MAX_PODCAST_RECORDS_PER_KEY) + .map(|index| rust_card(index, NOW)) + .collect(); + let keyed = |records: &[SignedPodcastRecord]| -> Vec<_> { + records + .iter() + .map(|record| (vec![rust], record.clone())) + .collect() + }; + assert_eq!(store_all(&db, keyed(&older), NOW).await, half); + assert_eq!(store_all(&db, keyed(&younger), NOW + 1).await, half); + + // Newcomers keep getting in; each takes a random place of the younger + // half, the older half stays. + let mut newcomers = Vec::new(); + for index in 0..half { + let newcomer = rust_card(1_000 + index, NOW); + assert_eq!( + db.store(vec![(vec![rust], newcomer.clone())], NOW + 2 + index as u64) + .await + .unwrap(), 1, - NOW + index as u64, + "a full key admits a newcomer" ); - entries.push((vec![token_key], record)); + newcomers.push(newcomer); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(older.iter().all(|record| cached.contains(record))); + assert!(cached.contains(newcomers.last().unwrap())); } - let first = entries[0].1.clone(); - let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone(); - // The first publisher arrived first; the rest a moment later. - db.store(entries[..1].to_vec(), NOW).await.unwrap(); - for chunk in entries[1..].chunks(64) { - db.store(chunk.to_vec(), NOW + 1).await.unwrap(); - } - let cached = db - .records_by_key(token_key, Vec::new(), NOW, usize::MAX) - .await - .unwrap(); - assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); - assert!(cached.contains(&first)); - assert!(!cached.contains(&last)); - assert_eq!(cached[0], first, "records indexed longest come first"); - // The first publisher keeps refreshing its place. + // Owners refresh in place without evicting anybody. let refresh = card_with( - spec("show.xml", "Rust Radio"), + spec("show-0.xml", "Rust Radio"), &numbered_secret(0), 1, NOW + 1_000, ); assert_eq!( - db.store(vec![(vec![token_key], refresh)], NOW) + db.store(vec![(vec![rust], refresh.clone())], NOW + 1_000) .await .unwrap(), 1 ); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 1_000, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(cached.contains(&refresh)); +} + +#[tokio::test] +async fn sybils_planted_first_cannot_lock_a_late_card_out_of_a_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let rust = PodcastDhtKey::token(&network(), "rust"); + // 128 throwaway identities, each with its own podcast titled like the + // victim's, fill the key first. + let sybils: Vec<_> = (0..MAX_PODCAST_DHT_RECORDS) + .map(|index| (vec![rust], rust_card(index, NOW))) + .collect(); + store_all(&db, sybils, NOW).await; + let victim = card(&secret(1), 1); + assert_eq!( + db.store(vec![entry(&victim)], NOW + 1).await.unwrap(), + victim.dht_keys().len() + ); + let tokens = vec!["rust".to_string(), "radio".to_string()]; + let served = db + .records_by_key(rust, tokens.clone(), NOW + 1, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(served.len() <= MAX_PODCAST_DHT_RECORDS); + assert!(served.contains(&victim), "everything admitted is served"); + let hits = rank_hits(aggregate_hits(served), "Rust Radio", &tokens); + assert!( + hits.iter() + .any(|hit| hit.podcast_id == *victim.podcast_id()) + ); +} + +#[tokio::test] +async fn a_card_back_after_expiring_gets_into_a_cache_full_of_sybils() { + let limits = store::StoreLimits { + records: 16, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + let victim = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id()); + db.store(vec![entry(&victim)], NOW).await.unwrap(); + let sybils = |issued: u64| -> Vec<_> { + (0..16) + .map(|index| { + let record = rust_card(index, issued); + (record.dht_keys(), record) + }) + .collect() + }; + store_all(&db, sybils(NOW), NOW + 1).await; + assert_eq!(count_records(&temp).await, 16); + + // The victim's device is offline longer than a card lives; the sybils + // keep refreshing theirs in place. + let later = NOW + PODCAST_CARD_TTL.as_millis() as u64 + 3_600_000; + store_all(&db, sybils(later), later).await; + assert!( + db.records_by_key(id_key, Vec::new(), later, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap() + .is_empty() + ); + let back = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later + 1); + assert!(db.store(vec![entry(&back)], later + 1).await.unwrap() > 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), later + 1, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(), + vec![back] + ); + assert_eq!(count_records(&temp).await, 16); +} + +#[tokio::test] +async fn sybil_tombstones_planted_first_cannot_hide_a_revocation() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let victim_card = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim_card.podcast_id()); + let rust = PodcastDhtKey::token(&network(), "rust"); + // 64 sybils publish a card of the victim's podcast, then convert it to + // a tombstone in place, under the id key and the text keys. + for index in 0..64 { + let key = numbered_secret(index); + let sybil_card = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW); + db.store(vec![entry(&sybil_card)], NOW).await.unwrap(); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + sybil_card.podcast_id().clone(), + 2, + NOW, + sybil_card.dht_keys(), + ); + let tombstone = + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap(); + assert!(db.store(vec![entry(&tombstone)], NOW).await.unwrap() > 0); + } + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let tombstones = |key: &PodcastDhtKey| -> usize { + conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_keys WHERE dht_key = ?1 AND tombstone = 1", + [key.as_bytes().as_slice()], + |row| row.get::<_, i64>(0), + ) + .unwrap() as usize + }; + for key in [id_key, rust] { + assert_eq!( + tombstones(&key), + MAX_PODCAST_TOMBSTONES_PER_KEY, + "in-place conversions count against the tombstone budget" + ); + } + + // The honest publisher publishes later, then unpublishes. + assert!(db.store(vec![entry(&victim_card)], NOW + 1).await.unwrap() > 0); + let victim_tombstone = tombstone(&secret(1), 2, NOW + 2); + assert!( + db.store(vec![entry(&victim_tombstone)], NOW + 2) + .await + .unwrap() + > 0 + ); + for key in [id_key, rust] { + assert!(tombstones(&key) <= MAX_PODCAST_TOMBSTONES_PER_KEY); + let served = db + .records_by_key(key, vec![], NOW + 2, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!( + served.contains(&victim_tombstone), + "the revocation is served" + ); + // It overrules a stale replica still holding the card. + let mut all = served; + all.push(victim_card.clone()); + assert!(aggregate_hits(all).iter().all(|hit| { + hit.podcast_id != *victim_card.podcast_id() + || !hit.publishers.contains(&secret(1).public()) + })); + } } #[tokio::test] @@ -663,8 +854,8 @@ async fn sybil_tombstones_cannot_suppress_a_card() { written += db.store(chunk.to_vec(), NOW).await.unwrap(); } assert_eq!( - written, MAX_PODCAST_TOMBSTONES_PER_KEY, - "only the id key's tombstone budget is used" + written, 256, + "each sybil tombstone takes a young sybil tombstone's place under the id key" ); let by_id = db @@ -726,36 +917,67 @@ async fn cache_enforces_publisher_record_and_byte_quotas() { let temp = tempfile::tempdir().unwrap(); let db = open_db_with(&temp, limits).await; let show = |index: usize, key: &SecretKey| { - entry(&card_with( + card_with( spec(&format!("show-{index}.xml"), "Rust Radio"), key, 1, NOW, - )) + ) }; let heavy = secret(1); - let stored: Vec = { - let mut out = Vec::new(); - for index in 0..4 { - out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap()); - } - out - }; - assert!(stored[..3].iter().all(|written| *written > 0)); + for index in 0..3 { + let record = show(index, &heavy); + assert!( + db.store(vec![entry(&record)], NOW + index as u64) + .await + .unwrap() + > 0 + ); + } assert_eq!( - stored[3], 0, + db.store(vec![entry(&show(3, &heavy))], NOW + 3) + .await + .unwrap(), + 0, "the fourth podcast of one publisher is refused" ); // A refresh of an admitted record is always accepted. let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1); - assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0); - assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0); - assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0); - assert_eq!( - db.store(vec![show(12, &secret(4))], NOW).await.unwrap(), - 0, - "the cache holds at most five records" + assert!(db.store(vec![entry(&refresh)], NOW + 4).await.unwrap() > 0); + assert!( + db.store(vec![entry(&show(10, &secret(2)))], NOW + 10) + .await + .unwrap() + > 0 ); + assert!( + db.store(vec![entry(&show(11, &secret(3)))], NOW + 11) + .await + .unwrap() + > 0 + ); + assert_eq!(count_records(&temp).await, 5); + + // The cache is full: every newcomer still gets in by evicting a random + // record of the younger half; the two oldest records always stay. + let rust = PodcastDhtKey::token(&network(), "rust"); + for index in 0..6 { + let newcomer = show(20 + index, &secret(10 + index as u8)); + assert!( + db.store(vec![entry(&newcomer)], NOW + 20 + index as u64) + .await + .unwrap() + > 0 + ); + assert_eq!(count_records(&temp).await, 5); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 30, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(cached.contains(&refresh)); + assert!(cached.contains(&show(1, &heavy))); + assert!(cached.contains(&newcomer)); + } let one = card(&secret(1), 1); let limits = store::StoreLimits { @@ -764,25 +986,34 @@ async fn cache_enforces_publisher_record_and_byte_quotas() { }; let temp = tempfile::tempdir().unwrap(); let db = open_db_with(&temp, limits).await; - assert!( - db.store(vec![entry(&card(&secret(1), 1))], NOW) - .await - .unwrap() - > 0 - ); - assert!( - db.store(vec![entry(&card(&secret(2), 1))], NOW) - .await - .unwrap() - > 0 - ); + let id_key = PodcastDhtKey::podcast(&network(), one.podcast_id()); + for (index, now) in [(1, NOW), (2, NOW + 1), (3, NOW + 2)] { + assert!( + db.store(vec![entry(&card(&secret(index), 1))], now) + .await + .unwrap() + > 0 + ); + } + // The byte budget holds two cards: the younger half of the two others + // (the second card) made room for the third. assert_eq!( - db.store(vec![entry(&card(&secret(3), 1))], NOW) + db.records_by_key(id_key, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS) .await - .unwrap(), - 0, - "the byte budget is full" + .unwrap() + .into_iter() + .map(|record| record.publisher()) + .collect::>(), + HashSet::from([secret(1).public(), secret(3).public()]) ); + // A record larger than the whole budget evicts nothing. + let limits = store::StoreLimits { + bytes: one.encoded_len() - 1, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + assert_eq!(db.store(vec![entry(&one)], NOW).await.unwrap(), 0); } #[tokio::test] @@ -931,6 +1162,71 @@ fn replicas_only_accept_keys_they_are_close_to() { assert!(is_responsible(&own, &near[1..], &key)); } +fn contact_of(key: &SecretKey) -> NodeContact { + let peer = key.public(); + NodeContact { + node_id: NodeId::from_endpoint(&peer), + peer_id: peer, + ticket: String::new(), + last_seen_ms: NOW, + } +} + +#[test] +fn refused_placements_fall_back_to_the_next_closest_nodes() { + let key = PodcastDhtKey::token(&network(), "rust"); + let contacts: Vec = (0..3 * K) + .map(|index| contact_of(&numbered_secret(index))) + .collect(); + let mut by_distance = contacts.clone(); + by_distance.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + // This node is the farthest one: it is no replica and no fallback. + let own = by_distance.pop().unwrap(); + let others: Vec = by_distance.clone(); + let targets = replica_targets(own.clone(), others.clone(), &key); + assert!(!targets.local); + let ids = |contacts: &[NodeContact]| -> Vec { + contacts.iter().map(|contact| contact.peer_id).collect() + }; + assert_eq!(ids(&targets.primary), ids(&by_distance[..K])); + let mut fallback = targets.fallback.clone(); + fallback.reverse(); + assert_eq!(ids(&fallback), ids(&by_distance[K..2 * K])); + + // Two refusals of one placement go to the two next closest nodes; + // another record's refusal has its own fallbacks. + let mut fallbacks = HashMap::from([ + ((0, key), targets.fallback.clone()), + ((1, key), targets.fallback), + ]); + let retry = retry_plan(vec![(0, key), (0, key), (1, key)], &mut fallbacks); + let first = by_distance[K].peer_id; + let second = by_distance[K + 1].peer_id; + assert_eq!(retry.targets.len(), 2); + assert_eq!( + retry.targets[&first].1.keys().copied().collect::>(), + vec![0, 1] + ); + assert_eq!( + retry.targets[&second].1.keys().copied().collect::>(), + vec![0] + ); + assert_eq!(fallbacks[&(0, key)].len(), K - 2); + + // With this node among the closest it keeps a local replica and is never + // its own fallback. + let targets = replica_targets(by_distance[0].clone(), others, &key); + assert!(targets.local); + assert_eq!(targets.primary.len(), K - 1); + assert!( + targets + .fallback + .iter() + .chain(&targets.primary) + .all(|contact| contact.peer_id != by_distance[0].peer_id) + ); +} + #[test] fn inbound_requests_are_limited_per_peer() { let inflight = Arc::new(Mutex::new(HashMap::new())); @@ -946,21 +1242,50 @@ fn inbound_requests_are_limited_per_peer() { } #[test] -fn lookups_keep_a_bounded_number_of_owners() { +fn lookups_give_each_response_a_bounded_share_of_owners() { let mut winners = HashMap::new(); let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id()); - let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS) - .map(|index| { - card_with( - spec("show.xml", "Rust Radio"), - &numbered_secret(index), - 1, - now_ms(), - ) - }) - .collect(); - let first = records[0].clone(); - merge_records(&mut winners, records, &network(), &id_key, &[]); + let response = |first: usize| -> Vec { + (first..first + MAX_PODCAST_DHT_RECORDS) + .map(|index| { + card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + now_ms(), + ) + }) + .collect() + }; + // Four attacker responders answer first, each with 128 sybil owners. + for responder in 0..4 { + merge_records( + &mut winners, + response(10_000 + responder * MAX_PODCAST_DHT_RECORDS), + &network(), + &id_key, + &[], + ); + } + assert_eq!(winners.len(), 4 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE); + // An honest replica answering later still contributes its owners. + let honest = response(0); + let victim = honest[0].clone(); + merge_records(&mut winners, honest, &network(), &id_key, &[]); + let owner = (victim.podcast_id().clone(), victim.publisher()); + assert_eq!(winners[&owner], victim); + assert_eq!(winners.len(), 5 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE); + + // The lookup as a whole stays bounded. + for responder in 0..8 { + merge_records( + &mut winners, + response(20_000 + responder * MAX_PODCAST_DHT_RECORDS), + &network(), + &id_key, + &[], + ); + } assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS); // A newer record of a known owner still replaces its winner. let newer = card_with( @@ -970,7 +1295,6 @@ fn lookups_keep_a_bounded_number_of_owners() { now_ms(), ); merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]); - let owner = (first.podcast_id().clone(), first.publisher()); assert_eq!(winners[&owner], newer); } diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs index b3ba983..395f3d5 100644 --- a/crates/music-dht/tests/podcast_dht.rs +++ b/crates/music-dht/tests/podcast_dht.rs @@ -18,6 +18,8 @@ //! and is never asked for records or to store them. //! * `sync_shares` stays fail-safe for revocation and holds back URLs that //! look private (single node). +//! * Contacts a peer returns are never trusted: fabricated, unreachable +//! contacts do not enter the routing table (two nodes). use std::path::Path; use std::sync::Arc; @@ -566,3 +568,91 @@ async fn single_node_scenario() { a.stop().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fabricated_contacts_never_join_the_overlay() { + tokio::time::timeout(TEST_TIMEOUT, fabricated_contacts_scenario()) + .await + .unwrap(); +} + +/// K contacts whose node ids are genuine hashes of their endpoint ids but +/// whose tickets carry no address, as a malicious peer could return them. +fn fabricated_contacts(network: NetworkId) -> Vec { + (0..music_dht::K) + .map(|index| { + let peer = federation_net::SecretKey::from_bytes(&[index as u8 + 101; 32]).public(); + let ticket = federation_net::PeerTicket { + ticket_version: federation_net::TICKET_VERSION, + protocol_version: 1, + network_id: network, + schema_id: federation_net::SchemaId::from_bytes([0; 32]), + endpoint_addr: federation_net::EndpointAddr::new(peer), + }; + NodeContact { + node_id: NodeId::from_endpoint(&peer), + peer_id: peer, + ticket: ticket.to_string(), + last_seen_ms: u64::MAX, + } + }) + .collect() +} + +async fn fabricated_contacts_scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-fabricated-contacts"); + let a = Node::start(&temp.path().join("a"), network).await; + // M speaks the overlay ALPN but answers every request with fabricated + // contacts. + let (m, m_events) = start_service(&temp.path().join("m"), network, true).await; + let mut acceptor = m.stream_acceptor(PODCAST_DHT_ALPN).unwrap(); + let fabricated = fabricated_contacts(network); + let response = PodcastDhtResponse { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: true, + error: None, + nodes: fabricated.clone(), + records: Vec::new(), + stored: 0, + }; + let payload = postcard::to_stdvec(&response).unwrap(); + let server = tokio::spawn(async move { + while let Some(mut stream) = acceptor.accept().await { + let mut request = Vec::new(); + let _ = AsyncReadExt::read_to_end( + &mut (&mut stream.recv).take(MAX_PODCAST_DHT_REQUEST_BYTES as u64), + &mut request, + ) + .await; + let _ = stream.send.write_all(&payload).await; + let _ = stream.send.finish(); + let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await; + } + }); + m.connect(a.service.ticket().await.unwrap()).await.unwrap(); + wait_for("music DHT contact", async || { + (!a.service.known_peers().is_empty()).then_some(()) + }) + .await; + + // Bootstrap learns M, which answered, and none of the contacts it made + // up: they are tried once and never answer. + a.dht.bootstrap().await.unwrap(); + assert_eq!(a.dht.known_peers(), 1); + // A lookup treats them as candidates only. + let podcast_id = PodcastId::from_feed_url(&feed("nothing.xml")); + assert!(a.dht.lookup_podcast(&podcast_id).await.unwrap().is_none()); + assert_eq!(a.dht.known_peers(), 1); + a.dht.bootstrap().await.unwrap(); + assert_eq!( + a.dht.known_peers(), + 1, + "fabricated contacts never count for replica responsibility" + ); + + server.abort(); + a.stop().await; + m_events.abort(); + m.shutdown().await.unwrap(); +}