diff --git a/crates/music-dht/src/capabilities.rs b/crates/music-dht/src/capabilities.rs index f4525f8..06249d7 100644 --- a/crates/music-dht/src/capabilities.rs +++ b/crates/music-dht/src/capabilities.rs @@ -56,7 +56,13 @@ pub struct CapabilityManifest { } impl CapabilityManifest { - /// Creates a manifest containing every protocol owned by Frid. + /// Creates a manifest containing every protocol owned by Frid that a + /// music-DHT application serves. + /// + /// The public podcast-discovery overlay is opt-in: applications that run + /// [`crate::podcast_dht::PodcastDht`] add it with + /// [`Self::with_podcast_dht`], so an application that does not serve the + /// overlay never advertises it. pub fn frid(application: impl Into, application_version: impl Into) -> Self { let mut protocols = BTreeMap::new(); protocols.insert( @@ -81,10 +87,6 @@ impl CapabilityManifest { SIMILARITY_DHT_ID.to_string(), crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION, ); - protocols.insert( - PODCAST_DHT_ID.to_string(), - crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION, - ); protocols.insert( DEVICE_SYNC_ID.to_string(), crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION, @@ -111,6 +113,17 @@ impl CapabilityManifest { self } + /// Advertises the public podcast-discovery overlay + /// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]). + /// Call it only when the application serves + /// [`crate::podcast_dht::PODCAST_DHT_ALPN`]. + pub fn with_podcast_dht(self) -> Self { + self.with_protocol( + PODCAST_DHT_ID, + crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION, + ) + } + /// Version of protocol `id` advertised by the peer, if any. pub fn protocol_version(&self, id: &str) -> Option { self.protocols.get(id).copied() @@ -237,7 +250,6 @@ mod tests { CATALOG_ID, SIMILARITY_ID, SIMILARITY_DHT_ID, - PODCAST_DHT_ID, DEVICE_SYNC_ID, JAM_ID, ] { @@ -247,13 +259,17 @@ mod tests { } #[test] - fn podcast_dht_capability_is_advertised_and_removable() { - let manifest = CapabilityManifest::frid("furumi", "1.2.3"); + fn podcast_dht_capability_is_opt_in_and_removable() { + let plain = CapabilityManifest::frid("furumi", "1.2.3"); + assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None); + assert!(!plain.supports_podcast_dht()); + let manifest = plain.with_podcast_dht(); assert_eq!( manifest.protocol_version(PODCAST_DHT_ID), Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION) ); assert!(manifest.supports_podcast_dht()); + manifest.validate().unwrap(); let without = manifest.clone().without_protocol(PODCAST_DHT_ID); assert!(!without.supports_podcast_dht()); let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2); diff --git a/crates/music-dht/src/podcast_dht/card.rs b/crates/music-dht/src/podcast_dht/card.rs index 2bcfd75..0b549da 100644 --- a/crates/music-dht/src/podcast_dht/card.rs +++ b/crates/music-dht/src/podcast_dht/card.rs @@ -396,6 +396,19 @@ impl PodcastCard { card_index_tokens(&self.title, self.author.as_deref()) } + /// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url` + /// when the feed has not moved). A valid card's id hashes from it, so it + /// is the only URL of the card bound to the podcast. + pub fn origin_feed_url(&self) -> &FeedUrl { + self.id_feed_url.as_ref().unwrap_or(&self.feed_url) + } + + /// Whether `feed_url` is the URL the id is derived from. Otherwise + /// `feed_url` is an unverified claim of the publisher that the feed moved. + pub fn feed_url_is_origin(&self) -> bool { + self.id_feed_url.is_none() + } + /// Canonical signing input (see `PODCASTS.md`). pub fn signing_bytes(&self) -> Vec { let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN); @@ -735,18 +748,6 @@ pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) .then_with(|| left.signing_bytes().cmp(&right.signing_bytes())) } -/// Whether `winner` makes `other` obsolete everywhere, not just under one key: -/// `other` is a card of a lower revision, or `winner` is a tombstone of an -/// equal or higher revision. -pub(crate) fn supersedes_everywhere( - winner: &SignedPodcastRecord, - other: &SignedPodcastRecord, -) -> bool { - !other.is_tombstone() - && (other.revision() < winner.revision() - || (winner.is_tombstone() && other.revision() <= winner.revision())) -} - struct Canonical(Vec); impl Canonical { diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs index 276b897..f71554f 100644 --- a/crates/music-dht/src/podcast_dht/mod.rs +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -27,6 +27,13 @@ //! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with //! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take //! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`]. +//! Advertise the overlay with [`CapabilityManifest::with_podcast_dht`] (it is +//! opt-in) and pass peer manifests to [`PodcastDht::note_peer_capabilities`]. +//! +//! Only the URL a podcast id hashes from is bound to the podcast; search hits +//! offer that URL ([`PodcastSearchHit::feed_url`]) and report moved-feed URLs +//! as unverified claims. The publisher key is the device's transport identity +//! and therefore linkable to its network address and music-DHT records. mod card; mod privacy; @@ -36,7 +43,7 @@ mod store; #[cfg(test)] mod tests; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -69,10 +76,16 @@ pub use privacy::{ FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning, }; pub use search::{ - FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, - SearchHitKind, merge_search_results, podcast_query_tokens, + ClaimedFeedUrl, FederatedSearchHit, MAX_HIT_FEED_URL_CLAIMS, MAX_HIT_PUBLISHERS, + MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, SearchHitKind, merge_search_results, + podcast_query_tokens, +}; +pub use store::{ + MAX_PERSISTED_PODCAST_DHT_PEERS, MAX_PODCAST_CACHE_BYTES, MAX_PODCAST_CACHE_RECORDS, + MAX_PODCAST_RECORDS_PER_KEY, MAX_PODCAST_RECORDS_PER_PUBLISHER, + MAX_PODCAST_RESPONSE_TOMBSTONES, MAX_PODCAST_TOMBSTONES_PER_KEY, + PODCAST_DHT_CACHE_SCHEMA_VERSION, }; -pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION}; use card::publication_error; use search::{aggregate_hits, rank_hits}; @@ -105,6 +118,16 @@ pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60); /// Maximum inbound requests processed concurrently; further streams are /// dropped. pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64; +/// Maximum inbound requests of one peer processed concurrently; further +/// streams of that peer are dropped. +pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8; +/// Maximum distinct `(podcast_id, publisher)` records one lookup keeps; +/// records of further owners are ignored. +pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512; +/// A node stores a placement only when fewer than this many of its known +/// overlay contacts are closer to the key than itself (`2 * K`: the `K` +/// closest with slack for differing routing views). +pub const PODCAST_STORE_RESPONSIBILITY_WINDOW: usize = 2 * K; /// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is /// not contacted. pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60); @@ -228,7 +251,7 @@ pub struct PodcastDhtPublishStats { } /// Result of [`PodcastDht::sync_shares`]. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct PodcastDhtSyncStats { /// Cards published with a new revision, or refreshed because they used /// half their lifetime. @@ -237,10 +260,38 @@ pub struct PodcastDhtSyncStats { pub unchanged: usize, /// Podcasts unpublished with a tombstone. pub unpublished: usize, + /// Shared podcasts not published because their card has privacy + /// warnings the user has not confirmed. The UI asks, then calls + /// [`PodcastDht::publish`] or `sync_shares` with + /// [`PodcastShareState::privacy_confirmed`]. An earlier publication of + /// the podcast stays as it was. + pub held_back: Vec, + /// Podcasts whose card could not be built or signed. They are not + /// published, and an earlier publication of them is unpublished. + pub failed: Vec, /// Network statistics of the pushed records. pub push: PodcastDhtPublishStats, } +/// A shared podcast held back by [`PodcastDht::sync_shares`] until the user +/// confirms publishing URLs that look private. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastPrivacyHold { + /// Podcast. + pub podcast_id: PodcastId, + /// Warnings of [`PodcastCardSpec::privacy_warnings`]. + pub warnings: Vec, +} + +/// A podcast [`PodcastDht::sync_shares`] could not publish or unpublish. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastSyncFailure { + /// Podcast. + pub podcast_id: PodcastId, + /// Diagnostic. + pub error: String, +} + /// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PodcastShareState { @@ -251,6 +302,11 @@ pub struct PodcastShareState { /// Card to publish when shared; `None` derives it with /// [`PodcastCardSpec::from_subscription`]. pub card: Option, + /// Whether the user confirmed publishing this card's feed URLs despite + /// its [`PodcastCardSpec::privacy_warnings`]. Without it, a card with + /// warnings is only published when this device already publishes the + /// same feed URLs. + pub privacy_confirmed: bool, } impl PodcastShareState { @@ -258,6 +314,22 @@ impl PodcastShareState { pub fn is_shared(&self) -> bool { effective_share(Some(&self.subscription), self.share.as_ref()) } + + /// The bounded, validated card [`PodcastDht::sync_shares`] would + /// publish: [`Self::card`] or one derived from the subscription. + pub fn card_spec(&self) -> Result { + let card = match &self.card { + Some(card) => card.bounded(), + None => PodcastCardSpec::from_subscription(&self.subscription)?, + }; + if card.podcast_id != self.subscription.podcast_id { + return Err(publication_error( + "card and subscription describe different podcasts", + )); + } + card.validate()?; + Ok(card) + } } /// This device's publication state of one podcast. @@ -284,6 +356,7 @@ pub struct PodcastDht { failed: Mutex>, publish_gate: Semaphore, serve_limit: Arc, + inflight: Arc>>, } impl std::fmt::Debug for PodcastDht { @@ -330,6 +403,7 @@ impl PodcastDht { failed: Mutex::new(HashMap::new()), publish_gate: Semaphore::new(1), serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), + inflight: Arc::new(Mutex::new(HashMap::new())), })) } @@ -341,6 +415,10 @@ impl PodcastDht { /// Serves authenticated overlay requests until `acceptor` closes. pub async fn serve(self: Arc, mut acceptor: StreamAcceptor) { while let Some(stream) = acceptor.accept().await { + let Some(slot) = PeerSlot::acquire(&self.inflight, stream.peer_id) else { + debug!(peer = %stream.peer_id, "podcast DHT request dropped: peer limit"); + continue; + }; let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else { debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded"); continue; @@ -348,6 +426,7 @@ impl PodcastDht { let this = Arc::clone(&self); tokio::spawn(async move { let _permit = permit; + let _slot = slot; let peer = stream.peer_id; match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await { Ok(Ok(())) => {} @@ -421,7 +500,10 @@ impl PodcastDht { } /// Discovers overlay peers among the routing table and the music DHT - /// contacts. + /// contacts with one `FindNode` each. Music-DHT contacts whose capability + /// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are + /// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become + /// overlay contacts when they answer. pub async fn bootstrap(&self) -> Result<()> { let known_feature: HashSet = lock(&self.routing) .contacts() @@ -521,44 +603,92 @@ impl PodcastDht { /// [`Self::republish`]). Every other podcast this device published, /// including podcasts absent from `states`, is unpublished. The call is /// idempotent: repeating it publishes nothing new. + /// + /// Each podcast is handled on its own, so revocation is fail-safe: a + /// shared podcast whose card cannot be built is reported in + /// [`PodcastDhtSyncStats::failed`] and unpublished, and never prevents + /// other podcasts from being unpublished. A card with + /// [`PodcastCardSpec::privacy_warnings`] is held back + /// ([`PodcastDhtSyncStats::held_back`]) unless + /// [`PodcastShareState::privacy_confirmed`] is set or this device already + /// publishes the same feed URLs. pub async fn sync_shares(&self, states: Vec) -> Result { - let mut desired: HashMap = HashMap::new(); - for state in states { + let _permit = self.publish_permit().await?; + let local = self.database.local_publications().await?; + let active: HashMap<&PodcastId, &PodcastCardSpec> = local + .iter() + .filter_map(|publication| { + publication + .spec + .as_ref() + .map(|spec| (&publication.podcast_id, spec)) + }) + .collect(); + let mut stats = PodcastDhtSyncStats::default(); + let mut desired: BTreeMap = BTreeMap::new(); + let mut held: HashSet = HashSet::new(); + for state in &states { if !state.is_shared() { continue; } - let card = match state.card { - Some(card) => card.bounded(), - None => PodcastCardSpec::from_subscription(&state.subscription)?, + let podcast_id = state.subscription.podcast_id.clone(); + let card = match state.card_spec() { + Ok(card) => card, + Err(error) => { + debug!(%podcast_id, %error, "podcast card cannot be published"); + stats.failed.push(PodcastSyncFailure { + podcast_id, + error: error.to_string(), + }); + continue; + } }; - if card.podcast_id != state.subscription.podcast_id { - return Err(publication_error( - "card and subscription describe different podcasts", - )); + let warnings = card.privacy_warnings(); + let already_published = active.get(&podcast_id).is_some_and(|published| { + published.feed_url == card.feed_url && published.id_feed_url == card.id_feed_url + }); + if !warnings.is_empty() && !state.privacy_confirmed && !already_published { + held.insert(podcast_id.clone()); + stats.held_back.push(PodcastPrivacyHold { + podcast_id, + warnings, + }); + continue; } - card.validate()?; - desired.insert(card.podcast_id.clone(), card); + desired.insert(podcast_id, card); } - let _permit = self.publish_permit().await?; - let local = self.database.local_publications().await?; - let mut stats = PodcastDhtSyncStats::default(); let mut records = Vec::new(); for publication in &local { - if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) { - records.extend(self.unpublish_locked(&publication.podcast_id).await?); - stats.unpublished += 1; + let podcast_id = &publication.podcast_id; + if publication.spec.is_none() + || desired.contains_key(podcast_id) + || held.contains(podcast_id) + { + continue; + } + match self.unpublish_locked(podcast_id).await { + Ok(record) => { + records.extend(record); + stats.unpublished += 1; + } + Err(error) => stats.failed.push(PodcastSyncFailure { + podcast_id: podcast_id.clone(), + error: error.to_string(), + }), } } - let mut desired: Vec<_> = desired.into_values().collect(); - desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id)); - for card in desired { - match self.sign_card_locked(card, false).await? { - Some(record) => { + for (podcast_id, card) in desired { + match self.sign_card_locked(card, false).await { + Ok(Some(record)) => { records.push(record); stats.published += 1; } - None => stats.unchanged += 1, + Ok(None) => stats.unchanged += 1, + Err(error) => stats.failed.push(PodcastSyncFailure { + podcast_id, + error: error.to_string(), + }), } } stats.push = self.push(records).await?; @@ -613,8 +743,8 @@ impl PodcastDht { /// normalized token of `query` ([`podcast_query_tokens`]). /// /// Hits are merged by `podcast_id` over publishers and ranked: exact - /// normalized title first, then more publishers. Partial results are - /// returned when peers fail. + /// normalized title first, then hits with a card under the origin feed + /// URL. Partial results are returned when peers fail. pub async fn search(&self, query: &str, limit: usize) -> Result> { let tokens = podcast_query_tokens(query); if tokens.is_empty() || limit == 0 { @@ -966,8 +1096,25 @@ impl PodcastDht { )) } PodcastDhtOperation::StoreBatch { records } => { - let entries = validate_store_batch(records, &self.network_id, now_ms())?; - let stored = self.database.store(entries, now_ms()).await?; + let entries = + validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?; + let nodes: Vec = lock(&self.routing) + .contacts() + .into_iter() + .map(|contact| contact.node_id) + .collect(); + let entries: Vec<_> = entries + .into_iter() + .filter_map(|(mut keys, record)| { + keys.retain(|key| is_responsible(&self.own_contact.node_id, &nodes, key)); + (!keys.is_empty()).then_some((keys, record)) + }) + .collect(); + let stored = if entries.is_empty() { + 0 + } else { + self.database.store(entries, now_ms()).await? + }; Ok(PodcastDhtResponse::success( Vec::new(), Vec::new(), @@ -1099,11 +1246,17 @@ impl PodcastDht { } } -/// Validates a store batch received from a peer: bounds, every record's -/// signature and lifetime, and that every key belongs to its record. +/// Validates a store batch received from `requester`: bounds, that every +/// record is published by the requester itself, every record's signature and +/// lifetime, and that every key belongs to its record. +/// +/// Replicas never forward records, so only a record's publisher stores it. +/// Nobody can therefore replay or inject another device's records, and +/// per-publisher quotas apply to the storing peer. fn validate_store_batch( entries: Vec, network_id: &NetworkId, + requester: EndpointId, now_ms: u64, ) -> Result, SignedPodcastRecord)>> { if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS { @@ -1117,6 +1270,11 @@ fn validate_store_batch( if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() { return Err(protocol_error("invalid podcast DHT store keys")); } + if entry.record.publisher() != requester { + return Err(protocol_error( + "podcast records must be stored by their publisher", + )); + } entry.record.verify_at(network_id, now_ms)?; let valid_keys = entry.record.dht_keys(); if keys.iter().any(|key| !valid_keys.contains(key)) { @@ -1128,6 +1286,18 @@ fn validate_store_batch( .collect() } +/// Whether a node is among the nodes that should store `key`: fewer than +/// [`PODCAST_STORE_RESPONSIBILITY_WINDOW`] known contacts are closer. +fn is_responsible(own: &NodeId, contacts: &[NodeId], key: &PodcastDhtKey) -> bool { + let own_distance = distance(own.as_bytes(), key.as_bytes()); + contacts + .iter() + .filter(|node| distance(node.as_bytes(), key.as_bytes()) < own_distance) + .take(PODCAST_STORE_RESPONSIBILITY_WINDOW) + .count() + < PODCAST_STORE_RESPONSIBILITY_WINDOW +} + fn validate_filter_tokens(tokens: &[String]) -> Result<()> { if tokens.len() > MAX_PODCAST_QUERY_TOKENS || tokens @@ -1139,7 +1309,8 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> { Ok(()) } -/// Adds verified records found under `key` to the per-owner winners. +/// Adds verified records found under `key` to the per-owner winners, keeping +/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners. fn merge_records( winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, records: Vec, @@ -1161,6 +1332,7 @@ fn merge_records( let owner = (record.podcast_id().clone(), record.publisher()); match winners.get(&owner) { Some(current) if compare_records(current, &record).is_ge() => {} + None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {} _ => { winners.insert(owner, record); } @@ -1168,6 +1340,42 @@ fn merge_records( } } +/// One in-flight inbound request of a peer; released on drop. +struct PeerSlot { + inflight: Arc>>, + peer: EndpointId, +} + +impl PeerSlot { + fn acquire( + inflight: &Arc>>, + peer: EndpointId, + ) -> Option { + let mut map = lock(inflight); + let count = map.entry(peer).or_default(); + if *count >= MAX_PODCAST_DHT_REQUESTS_PER_PEER { + return None; + } + *count += 1; + Some(Self { + inflight: Arc::clone(inflight), + peer, + }) + } +} + +impl Drop for PeerSlot { + fn drop(&mut self) { + let mut map = lock(&self.inflight); + if let Some(count) = map.get_mut(&self.peer) { + *count = count.saturating_sub(1); + if *count == 0 { + map.remove(&self.peer); + } + } + } +} + fn validate_contact( contact: &NodeContact, network_id: &NetworkId, diff --git a/crates/music-dht/src/podcast_dht/search.rs b/crates/music-dht/src/podcast_dht/search.rs index 8b9e3b9..2ab8781 100644 --- a/crates/music-dht/src/podcast_dht/search.rs +++ b/crates/music-dht/src/podcast_dht/search.rs @@ -8,15 +8,17 @@ use federation_net::EndpointId; use serde::{Deserialize, Serialize}; use crate::normalization::{normalize_name, tokenize}; -use crate::podcast::PodcastId; +use crate::podcast::{FeedUrl, PodcastId}; use crate::record::{ItemKind, LibraryItem}; use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records}; /// Maximum distinct query tokens used by a podcast text search. pub const MAX_PODCAST_QUERY_TOKENS: usize = 8; -/// Maximum publishers reported per hit. +/// Maximum publishers reported per hit and per claimed feed URL. pub const MAX_HIT_PUBLISHERS: usize = 64; +/// Maximum claimed moved feed URLs reported per hit. +pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8; /// Kind of a federated search hit. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -64,18 +66,35 @@ impl From for SearchHitKind { /// /// Cards of the same podcast published by several devices are merged by /// `podcast_id`. The hit carries no personal state of any publisher. +/// +/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every +/// other field is a claim of some publisher device, and device identities are +/// free, so neither the metadata nor the number of publishers is a trust +/// signal. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PodcastSearchHit { /// Podcast. pub podcast_id: PodcastId, - /// Representative card: the most common `feed_url` among the - /// publishers' current cards, then the most recently issued card, then - /// the greatest publisher key. + /// The URL `podcast_id` is derived from + /// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with + /// this URL; follow the feed's own HTTP redirects and + /// `itunes:new-feed-url` from there. + pub feed_url: FeedUrl, + /// Representative card for display: a card whose `feed_url` is the + /// origin URL when one exists, then the title most publishers agree on, + /// then the smallest publisher key. Its metadata is a publisher's claim. pub card: PodcastCard, /// Sorted distinct publisher devices with a current valid card, at most /// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers: - /// one user may publish from several devices. + /// one user may publish from several devices, and anyone can mint + /// devices. pub publishers: Vec, + /// Feed URLs other than [`Self::feed_url`] that publishers claim the + /// podcast moved to, most claimed first, at most + /// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this + /// podcast or record it as an alias unless the feed at + /// [`Self::feed_url`] itself redirects there. + pub claimed_feed_urls: Vec, } impl PodcastSearchHit { @@ -85,6 +104,16 @@ impl PodcastSearchHit { } } +/// An unverified moved-feed URL claimed by some publishers of a podcast. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClaimedFeedUrl { + /// Claimed current feed URL. + pub feed_url: FeedUrl, + /// Sorted distinct publisher devices claiming it, at most + /// [`MAX_HIT_PUBLISHERS`]. + pub publishers: Vec, +} + /// A music or podcast result of a federated search. /// /// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use @@ -184,37 +213,66 @@ pub(crate) fn aggregate_hits( by_podcast .into_iter() .map(|(podcast_id, cards)| { - let mut feed_votes: HashMap<&str, usize> = HashMap::new(); + let mut title_votes: HashMap = HashMap::new(); for card in &cards { - *feed_votes.entry(card.feed_url.as_str()).or_default() += 1; + *title_votes.entry(normalize_name(&card.title)).or_default() += 1; } + // Publisher-controlled times and revisions never break ties. let card = cards .iter() .max_by_key(|card| { ( - feed_votes[card.feed_url.as_str()], - card.issued_at_ms, - *card.publisher.as_bytes(), + card.feed_url_is_origin(), + title_votes[&normalize_name(&card.title)], + Reverse(*card.publisher.as_bytes()), ) }) .cloned() .expect("a group has at least one card"); - let mut publishers: Vec = cards.iter().map(|card| card.publisher).collect(); - publishers.sort_by_key(|publisher| *publisher.as_bytes()); - publishers.dedup(); - publishers.truncate(MAX_HIT_PUBLISHERS); + let mut claims: BTreeMap<&FeedUrl, Vec> = BTreeMap::new(); + for card in cards.iter().filter(|card| !card.feed_url_is_origin()) { + claims + .entry(&card.feed_url) + .or_default() + .push(card.publisher); + } + let mut claimed_feed_urls: Vec = claims + .into_iter() + .map(|(feed_url, publishers)| ClaimedFeedUrl { + feed_url: feed_url.clone(), + publishers: sorted_publishers(publishers), + }) + .collect(); + claimed_feed_urls.sort_by(|left, right| { + right + .publishers + .len() + .cmp(&left.publishers.len()) + .then_with(|| left.feed_url.cmp(&right.feed_url)) + }); + claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS); PodcastSearchHit { podcast_id, + feed_url: card.origin_feed_url().clone(), + publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()), card, - publishers, + claimed_feed_urls, } }) .collect() } +fn sorted_publishers(mut publishers: Vec) -> Vec { + publishers.sort_by_key(|publisher| *publisher.as_bytes()); + publishers.dedup(); + publishers.truncate(MAX_HIT_PUBLISHERS); + publishers +} + /// Keeps hits whose card is indexed under every query token and ranks -/// them: exact normalized title first, then more publishers, then title and -/// id. +/// them: exact normalized title first, then hits with a card published under +/// the origin feed URL, then title and id. The publisher count is not used: +/// anyone can inflate it with new device identities. pub(crate) fn rank_hits( mut hits: Vec, query: &str, @@ -229,7 +287,7 @@ pub(crate) fn rank_hits( let title = normalize_name(&hit.card.title); ( title != normalized_query, - Reverse(hit.publishers.len()), + !hit.card.feed_url_is_origin(), title, hit.podcast_id.clone(), ) diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs index 9f27104..acaedbc 100644 --- a/crates/music-dht/src/podcast_dht/store.rs +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -5,7 +5,7 @@ //! (like the similarity-routing cache). Every record read from it is decoded //! and verified again before it is used or served. -use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -17,32 +17,66 @@ use crate::error::{MusicDhtError, Result}; use crate::podcast::PodcastId; use crate::routing::{NodeContact, NodeId}; -use super::card::{ - PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere, -}; +use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records}; -/// `PRAGMA user_version` of the podcast-discovery cache schema. -pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1; -/// Maximum distinct `(podcast_id, publisher)` records cached under one key. -/// When full, the record expiring first is evicted if the new one outlives it. +/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1 +/// (pre-release) stored one payload per key; its replica table is dropped on +/// open because the cache is replaceable. Own publications are kept. +pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2; +/// Maximum records (cards and tombstones) indexed under one key. A full key +/// admits no new `(podcast_id, publisher)` until a record expires: existing +/// records are never evicted for newcomers, and their owners keep refreshing +/// them in place. pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; +/// Maximum tombstones newly indexed under one key. A tombstone is admitted +/// under a text key only in place of a record of the same +/// `(podcast_id, publisher)` already indexed there, so this budget applies to +/// the podcast-id key, where tombstones of unknown cards land. +pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32; +/// Maximum tombstones in one `FindValue` response; cards fill the rest. +pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32; +/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a +/// replica. +pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256; +/// Maximum `(podcast_id, publisher)` records in a replica cache. +pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096; +/// Maximum total encoded size of the records in a replica cache. +pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024; +/// Maximum persisted overlay contacts; the least recently seen are dropped. +pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024; const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS podcast_dht_records ( - dht_key BLOB NOT NULL, - podcast_id TEXT NOT NULL, - publisher TEXT NOT NULL, - tombstone INTEGER NOT NULL, - expires_at_ms INTEGER NOT NULL, - payload BLOB NOT NULL, - PRIMARY KEY (dht_key, podcast_id, publisher) + podcast_id TEXT NOT NULL, + publisher TEXT NOT NULL, + tombstone INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + payload_bytes INTEGER NOT NULL, + payload BLOB NOT NULL, + PRIMARY KEY (podcast_id, publisher) ); CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry ON podcast_dht_records(expires_at_ms); -CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner - ON podcast_dht_records(podcast_id, publisher); +CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher + ON podcast_dht_records(publisher); + +CREATE TABLE IF NOT EXISTS podcast_dht_keys ( + dht_key BLOB NOT NULL, + podcast_id TEXT NOT NULL, + publisher TEXT NOT NULL, + tombstone INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + first_stored_at_ms INTEGER NOT NULL, + PRIMARY KEY (dht_key, podcast_id, publisher) +); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner + ON podcast_dht_keys(podcast_id, publisher); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry + ON podcast_dht_keys(expires_at_ms); CREATE TABLE IF NOT EXISTS podcast_dht_peers ( peer_id TEXT PRIMARY KEY, @@ -60,6 +94,28 @@ CREATE TABLE IF NOT EXISTS podcast_dht_local ( ); "; +/// Admission limits of a replica cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StoreLimits { + pub per_key: usize, + pub tombstones_per_key: usize, + pub per_publisher: usize, + pub records: usize, + pub bytes: usize, +} + +impl Default for StoreLimits { + fn default() -> Self { + Self { + per_key: MAX_PODCAST_RECORDS_PER_KEY, + tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY, + per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER, + records: MAX_PODCAST_CACHE_RECORDS, + bytes: MAX_PODCAST_CACHE_BYTES, + } + } +} + /// This device's latest publication of one podcast. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LocalPublication { @@ -76,6 +132,7 @@ pub(crate) struct LocalPublication { pub(crate) struct PodcastDhtDatabase { conn: Arc>, network_id: NetworkId, + limits: StoreLimits, } impl std::fmt::Debug for PodcastDhtDatabase { @@ -88,6 +145,14 @@ impl std::fmt::Debug for PodcastDhtDatabase { impl PodcastDhtDatabase { pub async fn open(path: &Path, network_id: NetworkId) -> Result { + Self::open_with_limits(path, network_id, StoreLimits::default()).await + } + + pub async fn open_with_limits( + path: &Path, + network_id: NetworkId, + limits: StoreLimits, + ) -> Result { let path = path.to_path_buf(); let conn = tokio::task::spawn_blocking(move || -> Result { let conn = Connection::open(&path).map_err(|error| { @@ -101,6 +166,11 @@ impl PodcastDhtDatabase { "podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}" ))); } + if version == 1 { + // Replicas are replaceable; the old layout is dropped. + conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;") + .map_err(database_error)?; + } conn.execute_batch(SCHEMA).map_err(|error| { MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}")) })?; @@ -113,37 +183,39 @@ impl PodcastDhtDatabase { Ok(Self { conn: Arc::new(Mutex::new(conn)), network_id, + limits, }) } async fn call(&self, operation: F) -> Result where - F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result + Send + 'static, + F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result + Send + 'static, T: Send + 'static, { let conn = Arc::clone(&self.conn); let network_id = self.network_id; + let limits = self.limits; tokio::task::spawn_blocking(move || { let guard = lock(&conn); - operation(&guard, &network_id).map_err(database_error) + operation(&guard, &network_id, &limits).map_err(database_error) }) .await .map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))? } /// Stores records under the given keys. The caller has verified every - /// record and checked that each key belongs to it. Returns the number of - /// `(key, record)` rows written. + /// record, checked that each key belongs to it and that the sender is its + /// publisher. Returns the number of `(key, record)` placements written. pub async fn store( &self, entries: Vec<(Vec, SignedPodcastRecord)>, now_ms: u64, ) -> Result { - self.call(move |conn, network_id| { + self.call(move |conn, network_id, limits| { let transaction = conn.unchecked_transaction()?; let mut written = 0; for (keys, record) in entries { - written += store_one(&transaction, network_id, &keys, &record, now_ms)?; + written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?; } transaction.commit()?; Ok(written) @@ -151,9 +223,11 @@ impl PodcastDhtDatabase { .await } - /// Valid, unexpired records under `key`, newest expiry first. Cards must - /// contain every token in `filter_tokens`; tombstones are always - /// returned so stale replicas elsewhere can be overruled. + /// Valid, unexpired records under `key`: at most + /// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale + /// replicas elsewhere can be overruled) and cards containing every token + /// in `filter_tokens` up to `limit` records in all. Records indexed longest + /// come first. Cards precede tombstones. pub async fn records_by_key( &self, key: PodcastDhtKey, @@ -161,52 +235,70 @@ impl PodcastDhtDatabase { now_ms: u64, limit: usize, ) -> Result> { - self.call(move |conn, network_id| { + self.call(move |conn, network_id, _| { let mut statement = conn.prepare( - "SELECT payload FROM podcast_dht_records - WHERE dht_key = ?1 AND expires_at_ms > ?2 - ORDER BY expires_at_ms DESC, podcast_id, publisher", + "SELECT r.payload FROM podcast_dht_keys k + JOIN podcast_dht_records r + ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher + WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3 + ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher", )?; - let rows = statement.query_map( - params![key.as_bytes().as_slice(), clamp_i64(now_ms)], - |row| row.get::<_, Vec>(0), - )?; - let mut records = Vec::new(); - for row in rows { - let Some(record) = decode_valid(&row?, network_id, now_ms) else { - continue; - }; - if !record.belongs_under(&key) { - continue; + let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result> { + let mut records = Vec::new(); + if limit == 0 { + return Ok(records); } - let matches = record.card().is_none_or(|card| { - let tokens = card.index_tokens(); - filter_tokens.iter().all(|token| tokens.contains(token)) - }); - if matches { - records.push(record); - if records.len() == limit { - break; + let rows = statement.query_map( + params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)], + |row| row.get::<_, Vec>(0), + )?; + for row in rows { + let Some(record) = decode_valid(&row?, network_id, now_ms) else { + continue; + }; + if record.is_tombstone() != tombstone || !record.belongs_under(&key) { + continue; + } + let matches = record.card().is_none_or(|card| { + let tokens = card.index_tokens(); + filter_tokens.iter().all(|token| tokens.contains(token)) + }); + if matches { + records.push(record); + if records.len() == limit { + break; + } } } - } + Ok(records) + }; + let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?; + let mut records = select(false, limit - tombstones.len())?; + records.extend(tombstones); Ok(records) }) .await } + /// Removes expired records and their key placements. Returns the number + /// of records removed. pub async fn delete_expired(&self, now_ms: u64) -> Result { - self.call(move |conn, _| { + self.call(move |conn, _, _| { + let now = clamp_i64(now_ms); + conn.execute( + "DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1", + [now], + )?; conn.execute( "DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1", - [clamp_i64(now_ms)], + [now], ) }) .await } pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { conn.execute( "INSERT OR REPLACE INTO podcast_dht_peers (peer_id, node_id, ticket, last_seen_ms) @@ -218,13 +310,20 @@ impl PodcastDhtDatabase { clamp_i64(contact.last_seen_ms), ], )?; + conn.execute( + "DELETE FROM podcast_dht_peers WHERE peer_id IN ( + SELECT peer_id FROM podcast_dht_peers + ORDER BY last_seen_ms DESC, peer_id + LIMIT -1 OFFSET ?1)", + [MAX_PERSISTED_PODCAST_DHT_PEERS as i64], + )?; Ok(()) }) .await } pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { conn.execute( "DELETE FROM podcast_dht_peers WHERE peer_id = ?1", [peer.to_string()], @@ -235,7 +334,7 @@ impl PodcastDhtDatabase { } pub async fn load_peers(&self) -> Result> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { let mut statement = conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?; let rows = statement.query_map([], |row| { @@ -267,7 +366,7 @@ impl PodcastDhtDatabase { &self, podcast_id: PodcastId, ) -> Result> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { conn.query_row( "SELECT podcast_id, revision, spec, record FROM podcast_dht_local WHERE podcast_id = ?1", @@ -281,7 +380,7 @@ impl PodcastDhtDatabase { } pub async fn local_publications(&self) -> Result> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { let mut statement = conn.prepare( "SELECT podcast_id, revision, spec, record FROM podcast_dht_local ORDER BY podcast_id", @@ -303,7 +402,7 @@ impl PodcastDhtDatabase { publication: LocalPublication, now_ms: u64, ) -> Result<()> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { let spec = publication .spec .as_ref() @@ -332,141 +431,235 @@ impl PodcastDhtDatabase { fn store_one( conn: &Connection, network_id: &NetworkId, + limits: &StoreLimits, keys: &[PodcastDhtKey], record: &SignedPodcastRecord, now_ms: u64, ) -> rusqlite::Result { let podcast_id = record.podcast_id().as_str().to_string(); let publisher = record.publisher().to_string(); + let now = clamp_i64(now_ms); - // Every cached version of this (podcast, publisher), re-verified. - let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new(); - let mut invalid_keys = Vec::new(); - { - let mut statement = conn.prepare( - "SELECT dht_key, payload FROM podcast_dht_records + // The cached record of this (podcast, publisher), re-verified. + let cached: Option<(Vec, i64)> = conn + .query_row( + "SELECT payload, payload_bytes FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2", - )?; - let rows = statement.query_map(params![podcast_id, publisher], |row| { - Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (key, payload) = row?; - let decoded = decode_valid(&payload, network_id, now_ms); - match (<[u8; 32]>::try_from(key.as_slice()), decoded) { - (Ok(key_bytes), Some(cached)) => { - existing.insert(key_bytes, cached); - } - _ => invalid_keys.push(key), + params![podcast_id, publisher], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + let (existing, existing_bytes) = match cached { + Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) { + Some(existing) => (Some(existing), bytes.max(0) as usize), + None => { + delete_owner(conn, &podcast_id, &publisher)?; + (None, 0) } + }, + None => { + // Stray placements without a record. + delete_owner(conn, &podcast_id, &publisher)?; + (None, 0) } - } - for key in invalid_keys { - conn.execute( - "DELETE FROM podcast_dht_records - WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", - params![key, podcast_id, publisher], - )?; - } + }; - // A tombstone or a newer revision cached under any key wins everywhere. - if existing - .values() - .any(|cached| supersedes_everywhere(cached, record)) + // Higher revision, tombstone at an equal revision, later republish... + let replace = match &existing { + Some(existing) => match compare_records(existing, record) { + std::cmp::Ordering::Greater => return Ok(0), + std::cmp::Ordering::Equal => false, + std::cmp::Ordering::Less => true, + }, + None => true, + }; + let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; + if replace + && !admits_record( + conn, + limits, + &publisher, + existing.is_none(), + existing_bytes, + payload.len(), + now, + )? { return Ok(0); } - let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; + let mut indexed: HashSet<[u8; 32]> = HashSet::new(); + { + let mut statement = conn.prepare( + "SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2", + )?; + let rows = statement.query_map(params![podcast_id, publisher], |row| { + row.get::<_, Vec>(0) + })?; + for row in rows { + if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) { + indexed.insert(key); + } + } + } + let mut written = 0; + if replace { + // Placements the new version no longer belongs under are dropped (a + // renamed card, or a tombstone that does not list the key); the others + // now serve the new version. + let belongs: HashSet<[u8; 32]> = record + .dht_keys() + .iter() + .map(|key| *key.as_bytes()) + .collect(); + let stale: Vec<[u8; 32]> = indexed + .iter() + .filter(|key| !belongs.contains(*key)) + .copied() + .collect(); + for key in stale { + conn.execute( + "DELETE FROM podcast_dht_keys + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_slice(), podcast_id, publisher], + )?; + indexed.remove(&key); + } + written += conn.execute( + "UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4 + WHERE podcast_id = ?1 AND publisher = ?2", + params![ + podcast_id, + publisher, + record.is_tombstone(), + clamp_i64(record.expires_at_ms()), + ], + )?; + } + + // New placements. A tombstone only takes a new place under the podcast-id + // key: under a text key it may only replace a record of its owner, so + // nobody can index tombstones under someone else's search terms. + let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id()); for key in keys { - if let Some(cached) = existing.get(key.as_bytes()) - && compare_records(cached, record).is_ge() - { + if indexed.contains(key.as_bytes()) { continue; } - if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? { + if record.is_tombstone() && *key != id_key { + continue; + } + if !admits_placement(conn, limits, key, record.is_tombstone(), now)? { continue; } conn.execute( - "INSERT INTO podcast_dht_records - (dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET - tombstone = excluded.tombstone, - expires_at_ms = excluded.expires_at_ms, - payload = excluded.payload", + "INSERT INTO podcast_dht_keys + (dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", params![ key.as_bytes().as_slice(), podcast_id, publisher, record.is_tombstone(), clamp_i64(record.expires_at_ms()), - payload, + now, ], )?; + indexed.insert(*key.as_bytes()); written += 1; } - // Drop versions the new record makes obsolete under other keys too. - if written > 0 { - for (key, cached) in &existing { - if !keys.iter().any(|stored| stored.as_bytes() == key) - && supersedes_everywhere(record, cached) - { - conn.execute( - "DELETE FROM podcast_dht_records - WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", - params![key.as_slice(), podcast_id, publisher], - )?; - } - } + if indexed.is_empty() { + delete_owner(conn, &podcast_id, &publisher)?; + return Ok(0); + } + if replace { + conn.execute( + "INSERT INTO podcast_dht_records + (podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(podcast_id, publisher) DO UPDATE SET + tombstone = excluded.tombstone, + expires_at_ms = excluded.expires_at_ms, + payload_bytes = excluded.payload_bytes, + payload = excluded.payload", + params![ + podcast_id, + publisher, + record.is_tombstone(), + clamp_i64(record.expires_at_ms()), + payload.len() as i64, + payload, + ], + )?; } Ok(written) } -/// Ensures there is room for one more record under `key`. -fn make_room( +/// Whether the cache may hold the new version of a record: per-publisher and +/// global record counts for a new owner, and the global byte budget. +fn admits_record( conn: &Connection, - key: &PodcastDhtKey, - record: &SignedPodcastRecord, - now_ms: u64, + limits: &StoreLimits, + publisher: &str, + new_owner: bool, + previous_bytes: usize, + new_bytes: usize, + now: i64, ) -> rusqlite::Result { - conn.execute( - "DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2", - params![key.as_bytes().as_slice(), clamp_i64(now_ms)], + let (records, bytes): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records + WHERE expires_at_ms > ?1", + [now], + |row| Ok((row.get(0)?, row.get(1)?)), )?; - let count: i64 = conn.query_row( - "SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1", - [key.as_bytes().as_slice()], - |row| row.get(0), - )?; - if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY { + let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); + if bytes.saturating_add(new_bytes) > limits.bytes { + return Ok(false); + } + if !new_owner { return Ok(true); } - let victim: Option<(String, String, i64)> = conn - .query_row( - "SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records - WHERE dht_key = ?1 - ORDER BY expires_at_ms ASC, podcast_id, publisher - LIMIT 1", - [key.as_bytes().as_slice()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .optional()?; - match victim { - Some((podcast_id, publisher, expires_at_ms)) - if expires_at_ms < clamp_i64(record.expires_at_ms()) => - { - conn.execute( - "DELETE FROM podcast_dht_records - WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", - params![key.as_bytes().as_slice(), podcast_id, publisher], - )?; - Ok(true) - } - _ => Ok(false), + if records.max(0) as usize >= limits.records { + return Ok(false); } + let own: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", + params![publisher, now], + |row| row.get(0), + )?; + Ok((own.max(0) as usize) < limits.per_publisher) +} + +/// Whether `key` has room for one more placement. Existing placements are +/// never evicted. +fn admits_placement( + conn: &Connection, + limits: &StoreLimits, + key: &PodcastDhtKey, + tombstone: bool, + now: i64, +) -> rusqlite::Result { + let (total, tombstones): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys + WHERE dht_key = ?1 AND expires_at_ms > ?2", + params![key.as_bytes().as_slice(), now], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + Ok((total.max(0) as usize) < limits.per_key + && (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key)) +} + +fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> { + conn.execute( + "DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2", + params![podcast_id, publisher], + )?; + conn.execute( + "DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2", + params![podcast_id, publisher], + )?; + Ok(()) } fn decode_valid( diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs index faf6159..034f6c3 100644 --- a/crates/music-dht/src/podcast_dht/tests.rs +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -3,7 +3,8 @@ use federation_net::{NetworkId, SecretKey}; -use super::card::supersedes_everywhere; +use crate::routing::NodeId; + use super::search::{aggregate_hits, rank_hits}; use super::*; use crate::podcast::{FeedUrl, PodcastMeta}; @@ -362,10 +363,6 @@ fn record_order_is_total_and_tombstones_win_ties() { compare_records(&edited, &older), compare_records(&older, &edited).reverse() ); - assert!(supersedes_everywhere(&grave, &older)); - assert!(supersedes_everywhere(&newer, &older)); - assert!(!supersedes_everywhere(&refresh, &older)); - assert!(!supersedes_everywhere(&older, &grave)); } // --------------------------------------------------------------------- cache @@ -437,7 +434,7 @@ async fn cache_converges_regardless_of_delivery_order() { } #[tokio::test] -async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() { +async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key_it_holds() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; let key = secret(1); @@ -451,7 +448,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() vec![active.clone()] ); - // A tombstone stored only under the id key still removes the card under + // A tombstone stored only under the id key still replaces the card under // the text keys of this node. let grave = tombstone(&key, 2, NOW); let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id()); @@ -462,7 +459,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() db.records_by_key(token_key, vec!["rust".into()], NOW, 10) .await .unwrap(), - Vec::::new() + vec![grave.clone()] ); // An equal-revision card arriving later loses, under any key. let same = card(&key, 2); @@ -564,43 +561,247 @@ async fn cache_read_path_filters_tokens_expiry_and_corruption() { .unwrap() .is_empty() ); - assert_eq!( - db.delete_expired(NOW + ttl).await.unwrap(), - MAX_RECORD_DHT_KEYS.min(record.dht_keys().len()) - ); + assert_eq!(db.delete_expired(NOW + ttl).await.unwrap(), 1); +} + +fn numbered_secret(index: usize) -> SecretKey { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes()); + bytes[31] = 0x5a; + SecretKey::from_bytes(&bytes) +} + +async fn open_db_with( + temp: &tempfile::TempDir, + limits: store::StoreLimits, +) -> store::PodcastDhtDatabase { + store::PodcastDhtDatabase::open_with_limits( + &temp.path().join("podcast-dht.sqlite3"), + network(), + limits, + ) + .await + .unwrap() } #[tokio::test] -async fn cache_bounds_records_per_key() { +async fn cache_full_key_keeps_its_records_and_refuses_newcomers() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; let token_key = PodcastDhtKey::token(&network(), "rust"); let mut entries = Vec::new(); for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { - let key = SecretKey::from_bytes(&{ - let mut bytes = [0u8; 32]; - bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes()); - bytes - }); - // Later publishers expire later, so the first one is evicted. - let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64); + // Later publishers expire later; that buys them nothing. + let record = card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + NOW + index as u64, + ); entries.push((vec![token_key], record)); } let first = entries[0].1.clone(); - for chunk in entries.chunks(64) { - db.store(chunk.to_vec(), NOW).await.unwrap(); + let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone(); + // The first publisher arrived first; the rest a moment later. + db.store(entries[..1].to_vec(), NOW).await.unwrap(); + for chunk in entries[1..].chunks(64) { + db.store(chunk.to_vec(), NOW + 1).await.unwrap(); } let cached = db .records_by_key(token_key, Vec::new(), NOW, usize::MAX) .await .unwrap(); assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); - assert!(!cached.contains(&first)); - // A record expiring before every cached one is not admitted. - assert_eq!( - db.store(vec![(vec![token_key], first)], NOW).await.unwrap(), - 0 + assert!(cached.contains(&first)); + assert!(!cached.contains(&last)); + assert_eq!(cached[0], first, "records indexed longest come first"); + // The first publisher keeps refreshing its place. + let refresh = card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(0), + 1, + NOW + 1_000, ); + assert_eq!( + db.store(vec![(vec![token_key], refresh)], NOW) + .await + .unwrap(), + 1 + ); +} + +#[tokio::test] +async fn sybil_tombstones_cannot_suppress_a_card() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let victim = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id()); + let rust = PodcastDhtKey::token(&network(), "rust"); + db.store(vec![entry(&victim)], NOW).await.unwrap(); + + // 256 throwaway identities sign tombstones of the victim's podcast that + // list the victim card's text keys, and issue them as late as allowed. + let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64; + let sybils: Vec<_> = (0..256) + .map(|index| { + let key = numbered_secret(index); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + victim.podcast_id().clone(), + u64::MAX, + NOW + skew, + victim.dht_keys(), + ); + let record = + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap(); + (record.dht_keys(), record) + }) + .collect(); + let mut written = 0; + for chunk in sybils.chunks(MAX_PODCAST_STORE_RECORDS) { + written += db.store(chunk.to_vec(), NOW).await.unwrap(); + } + assert_eq!( + written, MAX_PODCAST_TOMBSTONES_PER_KEY, + "only the id key's tombstone budget is used" + ); + + let by_id = db + .records_by_key(id_key, Vec::new(), NOW, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(by_id.contains(&victim)); + assert_eq!( + by_id.iter().filter(|record| record.is_tombstone()).count(), + MAX_PODCAST_RESPONSE_TOMBSTONES + ); + assert_eq!( + db.records_by_key(rust, vec!["rust".into()], NOW, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(), + vec![victim.clone()], + "no tombstone is indexed under a text key without its owner's card" + ); + // Eleven hours later the victim's republish still lands. + let later = NOW + 11 * 3_600_000; + let republished = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later); + assert_eq!( + db.store(vec![entry(&republished)], later).await.unwrap(), + republished.dht_keys().len() + ); + // Sybil cards issued in the future cannot displace it either. + let sybil_cards: Vec<_> = (300..600) + .map(|index| { + entry(&card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + u64::MAX, + later + skew, + )) + }) + .collect(); + for chunk in sybil_cards.chunks(MAX_PODCAST_STORE_RECORDS) { + db.store(chunk.to_vec(), later).await.unwrap(); + } + for key in [id_key, rust] { + let records = db + .records_by_key(key, vec![], later, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(records.contains(&republished)); + assert!(records.len() <= MAX_PODCAST_DHT_RECORDS); + } +} + +#[tokio::test] +async fn cache_enforces_publisher_record_and_byte_quotas() { + let limits = store::StoreLimits { + per_key: 100, + tombstones_per_key: 10, + per_publisher: 3, + records: 5, + bytes: usize::MAX, + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + let show = |index: usize, key: &SecretKey| { + entry(&card_with( + spec(&format!("show-{index}.xml"), "Rust Radio"), + key, + 1, + NOW, + )) + }; + let heavy = secret(1); + let stored: Vec = { + let mut out = Vec::new(); + for index in 0..4 { + out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap()); + } + out + }; + assert!(stored[..3].iter().all(|written| *written > 0)); + assert_eq!( + stored[3], 0, + "the fourth podcast of one publisher is refused" + ); + // A refresh of an admitted record is always accepted. + let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1); + assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0); + assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0); + assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0); + assert_eq!( + db.store(vec![show(12, &secret(4))], NOW).await.unwrap(), + 0, + "the cache holds at most five records" + ); + + let one = card(&secret(1), 1); + let limits = store::StoreLimits { + bytes: one.encoded_len() * 2, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + assert!( + db.store(vec![entry(&card(&secret(1), 1))], NOW) + .await + .unwrap() + > 0 + ); + assert!( + db.store(vec![entry(&card(&secret(2), 1))], NOW) + .await + .unwrap() + > 0 + ); + assert_eq!( + db.store(vec![entry(&card(&secret(3), 1))], NOW) + .await + .unwrap(), + 0, + "the byte budget is full" + ); +} + +#[tokio::test] +async fn cache_stores_each_payload_once() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let many: Vec = (0..50).map(|index| format!("w{index}")).collect(); + let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW); + assert_eq!( + db.store(vec![entry(&record)], NOW).await.unwrap(), + MAX_RECORD_DHT_KEYS + ); + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let payloads: i64 = conn + .query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(payloads, 1); } #[tokio::test] @@ -634,6 +835,25 @@ async fn cache_refuses_a_newer_schema_and_keeps_local_publications() { assert_eq!(local[0].record, record); drop(db); + // A pre-release version-1 replica table is dropped; own publications + // survive. + { + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute_batch( + "DROP TABLE podcast_dht_records; + CREATE TABLE podcast_dht_records (dht_key BLOB, payload BLOB); + INSERT INTO podcast_dht_records VALUES (x'00', x'00'); + PRAGMA user_version = 1;", + ) + .unwrap(); + } + let db = store::PodcastDhtDatabase::open(&path, network()) + .await + .unwrap(); + assert_eq!(db.local_publications().await.unwrap().len(), 1); + assert!(db.store(vec![entry(&record)], NOW).await.unwrap() > 0); + drop(db); + rusqlite::Connection::open(&path) .unwrap() .pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1) @@ -655,12 +875,14 @@ fn store_batches_are_bounded_and_keys_must_belong() { keys: record.dht_keys(), record: record.clone(), }; - validate_store_batch(vec![good.clone()], &network(), NOW).unwrap(); - assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err()); + let publisher = key.public(); + validate_store_batch(vec![good.clone()], &network(), publisher, NOW).unwrap(); + assert!(validate_store_batch(Vec::new(), &network(), publisher, NOW).is_err()); assert!( validate_store_batch( vec![good.clone(); MAX_PODCAST_STORE_RECORDS], &network(), + publisher, NOW ) .is_ok() @@ -669,19 +891,87 @@ fn store_batches_are_bounded_and_keys_must_belong() { validate_store_batch( vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1], &network(), + publisher, NOW ) .is_err() ); let mut foreign = good.clone(); foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")]; - assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err()); + assert!(validate_store_batch(vec![foreign], &network(), publisher, NOW).is_err()); let mut duplicated = good.clone(); duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]]; - assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err()); - let mut forged = good; + assert!(validate_store_batch(vec![duplicated], &network(), publisher, NOW).is_err()); + let mut forged = good.clone(); resign(&mut forged.record, &secret(2)); - assert!(validate_store_batch(vec![forged], &network(), NOW).is_err()); + assert!(validate_store_batch(vec![forged], &network(), publisher, NOW).is_err()); + // A valid record of another publisher may not be stored (or replayed) + // by anyone else. + let error = validate_store_batch(vec![good], &network(), secret(2).public(), NOW).unwrap_err(); + assert!(error.to_string().contains("publisher"), "{error}"); +} + +#[test] +fn replicas_only_accept_keys_they_are_close_to() { + let own = NodeId::from_bytes([0; 32]); + let key = PodcastDhtKey::from_bytes([0; 32]); + let far: Vec = (0..100) + .map(|index| NodeId::from_bytes([0x80 | index as u8; 32])) + .collect(); + assert!(is_responsible(&own, &far, &key)); + let near: Vec = (1..=PODCAST_STORE_RESPONSIBILITY_WINDOW) + .map(|index| { + let mut bytes = [0; 32]; + bytes[31] = index as u8; + NodeId::from_bytes(bytes) + }) + .collect(); + let own = NodeId::from_bytes([0xff; 32]); + assert!(!is_responsible(&own, &near, &key)); + assert!(is_responsible(&own, &near[1..], &key)); +} + +#[test] +fn inbound_requests_are_limited_per_peer() { + let inflight = Arc::new(Mutex::new(HashMap::new())); + let peer = secret(1).public(); + let slots: Vec<_> = (0..MAX_PODCAST_DHT_REQUESTS_PER_PEER) + .map(|_| PeerSlot::acquire(&inflight, peer).unwrap()) + .collect(); + assert!(PeerSlot::acquire(&inflight, peer).is_none()); + assert!(PeerSlot::acquire(&inflight, secret(2).public()).is_some()); + drop(slots); + assert!(PeerSlot::acquire(&inflight, peer).is_some()); + assert!(lock(&inflight).is_empty()); +} + +#[test] +fn lookups_keep_a_bounded_number_of_owners() { + let mut winners = HashMap::new(); + let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id()); + let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS) + .map(|index| { + card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + now_ms(), + ) + }) + .collect(); + let first = records[0].clone(); + merge_records(&mut winners, records, &network(), &id_key, &[]); + assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS); + // A newer record of a known owner still replaces its winner. + let newer = card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(0), + 2, + now_ms(), + ); + merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]); + let owner = (first.podcast_id().clone(), first.publisher()); + assert_eq!(winners[&owner], newer); } #[test] @@ -763,18 +1053,27 @@ fn music(name: &str, kind: ItemKind) -> LibraryItem { } } +fn moved_spec(path: &str, origin: &str, title: &str) -> PodcastCardSpec { + let mut moved = spec(path, title); + moved.id_feed_url = Some(feed(origin)); + moved.podcast_id = PodcastId::from_feed_url(&feed(origin)); + moved +} + #[test] fn hits_merge_publishers_and_drop_tombstoned_ones() { let a = secret(1); let b = secret(2); let c = secret(3); - let mut moved = spec("moved.xml", "Rust Radio"); - moved.id_feed_url = Some(feed("show.xml")); - moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml")); let records = vec![ card(&a, 1), card(&a, 2), - card_with(moved, &b, 1, NOW + 10), + card_with( + moved_spec("moved.xml", "show.xml", "Rust Radio"), + &b, + 1, + NOW + 10, + ), card(&c, 1), tombstone(&c, 2, NOW), card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW), @@ -790,8 +1089,81 @@ fn hits_merge_publishers_and_drop_tombstoned_ones() { let mut expected = vec![a.public(), b.public()]; expected.sort_by_key(|key| *key.as_bytes()); assert_eq!(hit.publishers, expected); - // Equal votes per feed URL: the most recently issued card represents. - assert_eq!(hit.card.publisher, b.public()); + // The card under the origin URL represents, whoever issued later. + assert_eq!(hit.card.publisher, a.public()); + assert_eq!(hit.feed_url, feed("show.xml")); + assert_eq!( + hit.claimed_feed_urls, + vec![ClaimedFeedUrl { + feed_url: feed("moved.xml"), + publishers: vec![b.public()], + }] + ); +} + +#[test] +fn sybil_cards_cannot_redirect_an_existing_podcast() { + let genuine = card(&secret(1), 1); + let id = genuine.podcast_id().clone(); + let evil_feed = FeedUrl::parse("https://evil.example/feed.xml").unwrap(); + let hijack = |seed: usize| { + let mut spec = spec("show.xml", "Rust Radio (official)"); + spec.id_feed_url = Some(spec.feed_url.clone()); + spec.feed_url = evil_feed.clone(); + spec.validate().unwrap(); + assert_eq!(spec.podcast_id, id, "a hijack card claims the genuine id"); + let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64; + card_with(spec, &numbered_secret(seed), u64::MAX, NOW + skew) + }; + let mut records = vec![genuine.clone()]; + records.extend((0..10).map(hijack)); + let hit = aggregate_hits(records).remove(0); + assert_eq!( + hit.feed_url, + feed("show.xml"), + "only the origin URL is offered" + ); + assert_eq!( + hit.card, + *genuine.card().unwrap(), + "the origin card represents" + ); + assert_eq!(hit.claimed_feed_urls.len(), 1); + assert_eq!(hit.claimed_feed_urls[0].feed_url, evil_feed); + assert_eq!(hit.claimed_feed_urls[0].publishers.len(), 10); + + // Without any origin card the hit still offers the origin URL, never the + // claimed one, and ranks below podcasts that have an origin card. + let only_claims = aggregate_hits((0..3).map(hijack)).remove(0); + assert_eq!(only_claims.feed_url, feed("show.xml")); + assert!(!only_claims.card.feed_url_is_origin()); + let honest = aggregate_hits(vec![card_with( + spec("b.xml", "Rust Radio Weekly"), + &secret(2), + 1, + NOW, + )]) + .remove(0); + let ranked = rank_hits( + vec![only_claims.clone(), honest.clone()], + "rust", + &podcast_query_tokens("rust"), + ); + assert_eq!(ranked[0].podcast_id, honest.podcast_id); + // Publisher counts do not rank. + let crowded = aggregate_hits( + (20..60).map(|seed| card_with(spec("c.xml", "Rust Zone"), &numbered_secret(seed), 1, NOW)), + ) + .remove(0); + let ranked = rank_hits( + vec![crowded, honest.clone()], + "rust", + &podcast_query_tokens("rust"), + ); + assert_eq!( + ranked[0].podcast_id, honest.podcast_id, + "ranked by title, not crowd" + ); } #[test] diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs index 548f869..b3ba983 100644 --- a/crates/music-dht/tests/podcast_dht.rs +++ b/crates/music-dht/tests/podcast_dht.rs @@ -1,15 +1,23 @@ //! Integration test: public podcast discovery on three real loopback iroh //! nodes. //! -//! * A publishes a podcast its user shared; B is one of A's trusted devices -//! (in this test merely another overlay node, then offline); C is an -//! unrelated peer outside A's trusted-device group, with no device-sync -//! pairing. -//! * C finds the card by text and by id; a podcast A never shared is never -//! discoverable; after A unpublishes, the tombstone hides the card from C. -//! * Forged, tampered, replayed, malformed, oversized and wrong-version -//! requests are rejected by a real peer. -//! * D, a music-DHT peer without the overlay, is never added to the overlay. +//! * A publishes a podcast its user shared. B is another device of the same +//! user: it applies the same merged registers (as device sync would deliver +//! them) and publishes the same podcast under its own key. C is an +//! unrelated peer outside the user's trusted-device group, with no +//! device-sync pairing. +//! * C finds the podcast by text and by id, merged over publishers {A, B}; +//! once B unshares and goes offline, C still finds A's card, so B is +//! irrelevant to discovery. A podcast the user never shared (held by both +//! A and B) is never discoverable; after A unpublishes, the tombstone hides +//! the card from C. +//! * Forged, foreign, tampered, replayed, malformed, oversized and +//! wrong-version requests are rejected by a real peer. +//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but +//! never answers on the overlay ALPN, so it never becomes an overlay contact +//! and is never asked for records or to store them. +//! * `sync_shares` stays fail-safe for revocation and holds back URLs that +//! look private (single node). use std::path::Path; use std::sync::Arc; @@ -214,14 +222,15 @@ async fn scenario() { (a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(()) }) .await; - // D never answers on the overlay ALPN, so it never becomes an overlay - // contact and is never queried or asked to store records. + // D is dialed by bootstrap but never answers on the overlay ALPN, so it + // never becomes an overlay contact: it is never asked for records or to + // store them. a.dht.bootstrap().await.unwrap(); assert_eq!(a.dht.known_peers(), 2); // A capability manifest without podcast_dht evicts a peer; one with it // lets the peer be learned again. - let manifest = CapabilityManifest::frid("furumi", "test"); + let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht(); a.dht .note_peer_capabilities( c.service.endpoint_id(), @@ -260,11 +269,13 @@ async fn scenario() { subscription: shared.clone(), share: Some(share(&shared, true, 2)), card: None, + privacy_confirmed: false, }, PodcastShareState { subscription: private.clone(), share: None, card: None, + privacy_confirmed: false, }, ]; let stats = a.dht.sync_shares(states.clone()).await.unwrap(); @@ -294,7 +305,54 @@ async fn scenario() { assert_eq!(published.len(), 1); let first_card = published[0].record.clone(); - // B (A's trusted device) is not needed for discovery. + // B, the same user's other device, holds the same merged registers and + // publishes the same podcast under its own key; the private podcast + // stays private there too. + let b_states = vec![ + PodcastShareState { + subscription: shared.clone(), + share: Some(share(&shared, true, 2)), + card: None, + privacy_confirmed: false, + }, + PodcastShareState { + subscription: private.clone(), + share: None, + card: None, + privacy_confirmed: false, + }, + ]; + let stats = b.dht.sync_shares(b_states).await.unwrap(); + assert_eq!((stats.published, stats.unpublished), (1, 0)); + let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()]; + both.sort_by_key(|key| *key.as_bytes()); + let hits = wait_for("C sees both publishers", async || { + let hits = search(&c, "rust radio").await; + (hits.len() == 1 && hits[0].publishers == both).then_some(hits) + }) + .await; + assert_eq!(hits[0].feed_url, shared.podcast.feed_url); + assert!(hits[0].claimed_feed_urls.is_empty()); + + // B unshares (for example a stale register on B) and goes offline: B is + // not needed for discovery, A's card is still found. + let stats = b + .dht + .sync_shares(vec![PodcastShareState { + subscription: shared.clone(), + share: Some(share(&shared, false, 3)), + card: None, + privacy_confirmed: false, + }]) + .await + .unwrap(); + assert_eq!(stats.unpublished, 1); + let a_only = vec![a.service.endpoint_id()]; + wait_for("B's tombstone reaches C's lookup", async || { + let hits = search(&c, "rust radio").await; + (hits.len() == 1 && hits[0].publishers == a_only).then_some(()) + }) + .await; b.stop().await; let hits = wait_for("C finds the card by text", async || { @@ -305,6 +363,7 @@ async fn scenario() { assert_eq!(hits.len(), 1); assert_eq!(hits[0].podcast_id, shared.podcast_id); assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]); + assert_eq!(hits[0].feed_url, shared.podcast.feed_url); assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url); assert_eq!(hits[0].card.title, "Rust Radio Weekly"); assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens"); @@ -350,7 +409,11 @@ async fn scenario() { let response = store_request(&c, &a, forged).await; assert!(!response.ok && response.stored == 0, "{response:?}"); - // Tampered: A's genuine card with a different feed URL. + // Foreign: A's genuine card stored by C. Only publishers store records. + let response = store_request(&c, &a, first_card.clone()).await; + assert!(!response.ok && response.stored == 0, "{response:?}"); + + // Tampered: A's genuine card with a different description. let mut tampered = first_card.clone(); if let PodcastRecordBody::Card(card) = &mut tampered.body { card.description = Some("Now with malware".into()); @@ -388,6 +451,7 @@ async fn scenario() { subscription: shared.clone(), share: Some(share(&shared, false, 3)), card: None, + privacy_confirmed: false, }]; let stats = a.dht.sync_shares(unshare).await.unwrap(); assert_eq!(stats.unpublished, 1); @@ -407,10 +471,12 @@ async fn scenario() { .is_none() ); - // Replaying the old card, even to the node that stores the tombstone, - // does not bring it back. - let response = store_request(&c, &a, first_card).await; + // A replaying its own old card to a replica that stores the tombstone + // does not bring it back; nobody else may replay it at all. + let response = store_request(&a, &c, first_card.clone()).await; assert!(response.ok && response.stored == 0, "{response:?}"); + let response = store_request(&c, &a, first_card).await; + assert!(!response.ok, "{response:?}"); assert!(search(&c, "rust radio").await.is_empty()); assert!(search(&a, "rust radio").await.is_empty()); @@ -419,3 +485,84 @@ async fn scenario() { d_events.abort(); d.shutdown().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sync_shares_is_fail_safe_and_holds_back_private_urls() { + tokio::time::timeout(TEST_TIMEOUT, single_node_scenario()) + .await + .unwrap(); +} + +async fn single_node_scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-sync-shares"); + let a = Node::start(&temp.path().join("a"), network).await; + let state = + |subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState { + subscription: subscription.clone(), + share: Some(share(subscription, shared, generation)), + card: None, + privacy_confirmed: false, + }; + let first = subscription("first.xml", "First Show"); + let second = subscription("second.xml", "Second Show"); + let stats = a + .dht + .sync_shares(vec![state(&first, true, 2), state(&second, true, 2)]) + .await + .unwrap(); + assert_eq!(stats.published, 2); + assert!(stats.failed.is_empty() && stats.held_back.is_empty()); + + // In one call the user unshares the first podcast while the second + // one's card cannot be built (the URL its id derives from is unknown). + let mut broken = second.clone(); + broken.podcast.feed_url = feed("elsewhere.xml"); + let stats = a + .dht + .sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)]) + .await + .unwrap(); + assert_eq!(stats.failed.len(), 1); + assert_eq!(stats.failed[0].podcast_id, second.podcast_id); + assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast"); + let published = a.dht.published().await.unwrap(); + assert!(published.iter().all(|publication| !publication.active)); + assert!( + published + .iter() + .all(|publication| publication.record.is_tombstone()) + ); + // Republishing re-sends tombstones only; nothing comes back. + a.dht.republish().await.unwrap(); + assert!(search(&a, "first show").await.is_empty()); + assert!(search(&a, "second show").await.is_empty()); + + // A feed URL that looks private is held back until the user confirms. + let mut member = subscription("member.xml?token=abc", "Member Show"); + member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url); + let stats = a + .dht + .sync_shares(vec![state(&member, true, 2)]) + .await + .unwrap(); + assert_eq!(stats.published, 0); + assert_eq!(stats.held_back.len(), 1); + assert_eq!(stats.held_back[0].podcast_id, member.podcast_id); + assert!(!stats.held_back[0].warnings.is_empty()); + assert!(search(&a, "member show").await.is_empty()); + let mut confirmed = state(&member, true, 2); + confirmed.privacy_confirmed = true; + let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap(); + assert_eq!(stats.published, 1); + assert_eq!(search(&a, "member show").await.len(), 1); + // Once published with the user's consent, the same URLs are kept. + let stats = a + .dht + .sync_shares(vec![state(&member, true, 2)]) + .await + .unwrap(); + assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0)); + + a.stop().await; +}