From d27155b25711d3984d34ecf2ee292b1120744567 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:45:52 +0200 Subject: [PATCH 1/9] Add signed public podcast discovery overlay New music_dht::podcast_dht: a schema-independent Kademlia overlay on furumi-fd/podcast-dht/1 (version 1) with its own SQLite cache. Devices publish signed public podcast cards only where effective_share holds, retract them with signed tombstones of the next revision, and search by normalized title/author tokens or podcast id. Records are verified on every receive and cache read. Adds typed federated search hits, the advisory feed_url_privacy_warning helper, the podcast_dht capability and MusicDhtError::PodcastPublication. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/capabilities.rs | 33 + crates/music-dht/src/error.rs | 5 + crates/music-dht/src/lib.rs | 4 + crates/music-dht/src/podcast_dht/card.rs | 898 +++++++++++++ crates/music-dht/src/podcast_dht/mod.rs | 1258 +++++++++++++++++++ crates/music-dht/src/podcast_dht/privacy.rs | 339 +++++ crates/music-dht/src/podcast_dht/search.rs | 238 ++++ crates/music-dht/src/podcast_dht/store.rs | 522 ++++++++ crates/music-dht/src/podcast_dht/tests.rs | 845 +++++++++++++ 9 files changed, 4142 insertions(+) create mode 100644 crates/music-dht/src/podcast_dht/card.rs create mode 100644 crates/music-dht/src/podcast_dht/mod.rs create mode 100644 crates/music-dht/src/podcast_dht/privacy.rs create mode 100644 crates/music-dht/src/podcast_dht/search.rs create mode 100644 crates/music-dht/src/podcast_dht/store.rs create mode 100644 crates/music-dht/src/podcast_dht/tests.rs diff --git a/crates/music-dht/src/capabilities.rs b/crates/music-dht/src/capabilities.rs index 94d27a6..f4525f8 100644 --- a/crates/music-dht/src/capabilities.rs +++ b/crates/music-dht/src/capabilities.rs @@ -35,6 +35,8 @@ pub const CATALOG_ID: &str = "catalog"; pub const SIMILARITY_ID: &str = "similarity"; /// Stable protocol identifier for DHT-routed similarity discovery. pub const SIMILARITY_DHT_ID: &str = "similarity_dht"; +/// Stable protocol identifier for the public podcast-discovery overlay. +pub const PODCAST_DHT_ID: &str = "podcast_dht"; /// Stable protocol identifier for personal-device synchronization. pub const DEVICE_SYNC_ID: &str = "device_sync"; /// Stable protocol identifier for Jam playback control. @@ -79,6 +81,10 @@ impl CapabilityManifest { SIMILARITY_DHT_ID.to_string(), crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION, ); + protocols.insert( + PODCAST_DHT_ID.to_string(), + crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION, + ); protocols.insert( DEVICE_SYNC_ID.to_string(), crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION, @@ -105,6 +111,18 @@ impl CapabilityManifest { self } + /// Version of protocol `id` advertised by the peer, if any. + pub fn protocol_version(&self, id: &str) -> Option { + self.protocols.get(id).copied() + } + + /// Whether the peer advertises the public podcast-discovery overlay at + /// the version this crate speaks. Peers without it are never queried. + pub fn supports_podcast_dht(&self) -> bool { + self.protocol_version(PODCAST_DHT_ID) + == Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION) + } + /// Validates bounds and identifiers received from a peer. pub fn validate(&self) -> Result<()> { if self.manifest_version != CAPABILITIES_PROTOCOL_VERSION { @@ -219,6 +237,7 @@ mod tests { CATALOG_ID, SIMILARITY_ID, SIMILARITY_DHT_ID, + PODCAST_DHT_ID, DEVICE_SYNC_ID, JAM_ID, ] { @@ -227,6 +246,20 @@ mod tests { manifest.validate().unwrap(); } + #[test] + fn podcast_dht_capability_is_advertised_and_removable() { + let manifest = CapabilityManifest::frid("furumi", "1.2.3"); + assert_eq!( + manifest.protocol_version(PODCAST_DHT_ID), + Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION) + ); + assert!(manifest.supports_podcast_dht()); + let without = manifest.clone().without_protocol(PODCAST_DHT_ID); + assert!(!without.supports_podcast_dht()); + let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2); + assert!(!other_version.supports_podcast_dht()); + } + #[test] fn application_protocol_is_additive() { let manifest = CapabilityManifest::frid("furumi", "1.2.3").with_protocol("audio", 1); diff --git a/crates/music-dht/src/error.rs b/crates/music-dht/src/error.rs index 8dee393..e1d80d6 100644 --- a/crates/music-dht/src/error.rs +++ b/crates/music-dht/src/error.rs @@ -50,6 +50,11 @@ pub enum MusicDhtError { /// The service is shutting down and no longer accepts operations. #[error("service is shutting down")] ShuttingDown, + + /// A podcast card cannot be published, for example because the user did + /// not share the podcast (see `music_dht::podcast::effective_share`). + #[error("podcast publication refused: {0}")] + PodcastPublication(String), } impl From for MusicDhtError { diff --git a/crates/music-dht/src/lib.rs b/crates/music-dht/src/lib.rs index 30447d8..7e18b1c 100644 --- a/crates/music-dht/src/lib.rs +++ b/crates/music-dht/src/lib.rs @@ -27,6 +27,9 @@ //! podcast/episode ids, metadata DTOs and convergent merge rules), with an //! optional RSS/Atom parser behind the `podcast-feed` feature; //! [`device_sync`] protocol v4 carries it between trusted devices. +//! * [`podcast_dht`] is a separate signed overlay for public podcast +//! discovery: cards a user explicitly shared, found by title/author tokens +//! or podcast id, retracted by signed tombstones. //! * Records are replicated to the `K` nodes whose ids are XOR-closest to //! each key. Publishers pick the targets from their routing table and send //! batched store requests (one pipeline per peer), so even a large library @@ -97,6 +100,7 @@ mod message; mod node; mod normalization; pub mod podcast; +pub mod podcast_dht; mod record; mod request; mod routing; diff --git a/crates/music-dht/src/podcast_dht/card.rs b/crates/music-dht/src/podcast_dht/card.rs new file mode 100644 index 0000000..2bcfd75 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/card.rs @@ -0,0 +1,898 @@ +//! Signed public podcast cards, tombstones, DHT keys and their canonical +//! signing encoding. +//! +//! Everything here is pure and deterministic: no clock, no I/O. Callers pass +//! the current time to the validation functions. + +use std::cmp::Ordering; +use std::collections::HashSet; +use std::time::Duration; + +use federation_net::{EndpointId, NetworkId, Signature}; +use serde::{Deserialize, Serialize}; + +use crate::error::{MusicDhtError, Result}; +use crate::normalization::{normalize_name, tokenize}; +use crate::podcast::{ + FeedUrl, MAX_URL_BYTES, PodcastId, SubscriptionRecord, bounded_text, is_http_url, +}; + +use super::privacy::{FeedUrlPrivacyWarning, feed_url_privacy_warning}; + +/// Current card, tombstone and overlay wire version. +pub const PODCAST_DHT_PROTOCOL_VERSION: u16 = 1; +/// Signing domain of a [`PodcastCard`]. Changing it invalidates every card. +pub const PODCAST_CARD_SIGNING_DOMAIN: &str = "furumi-podcast-card:v1"; +/// Signing domain of a [`PodcastTombstone`]. +pub const PODCAST_TOMBSTONE_SIGNING_DOMAIN: &str = "furumi-podcast-card-tombstone:v1"; +/// BLAKE3 derivation domain of [`PodcastDhtKey`]. +pub const PODCAST_DHT_KEY_DOMAIN: &str = "furumi-podcast-dht:key:v1"; + +/// Maximum lifetime of a card: the main DHT's active-record TTL. +pub const PODCAST_CARD_TTL: Duration = crate::record::ACTIVE_RECORD_TTL; +/// Maximum lifetime of a tombstone: the main DHT's tombstone TTL. It exceeds +/// [`PODCAST_CARD_TTL`], so a card issued before a tombstone always expires +/// before the tombstone does. +pub const PODCAST_TOMBSTONE_TTL: Duration = crate::record::TOMBSTONE_TTL; +/// Tolerated clock skew for `issued_at_ms` in the future. +pub const PODCAST_DHT_MAX_CLOCK_SKEW: Duration = Duration::from_secs(5 * 60); + +/// Maximum characters of a card title. +pub const MAX_CARD_TITLE_CHARS: usize = 300; +/// Maximum characters of a card author. +pub const MAX_CARD_AUTHOR_CHARS: usize = 300; +/// Maximum characters of a card description. +pub const MAX_CARD_DESCRIPTION_CHARS: usize = 1_000; +/// Maximum bytes of a card language tag (BCP 47 shape: ASCII letters, +/// digits and `-`). +pub const MAX_CARD_LANGUAGE_BYTES: usize = 35; +/// Maximum categories on a card. +pub const MAX_CARD_CATEGORIES: usize = 8; +/// Maximum characters of one category. +pub const MAX_CARD_CATEGORY_CHARS: usize = 64; +/// Maximum indexed search tokens of a card (title tokens, then author tokens). +pub const MAX_CARD_INDEX_TOKENS: usize = 32; +/// Maximum bytes of one indexed or query token. Longer tokens are not indexed. +pub const MAX_INDEX_TOKEN_BYTES: usize = 64; +/// Maximum DHT keys a record is stored under: the podcast-id key plus one +/// key per indexed token. +pub const MAX_RECORD_DHT_KEYS: usize = MAX_CARD_INDEX_TOKENS + 1; +/// Maximum postcard-encoded size of one signed record. +pub const MAX_PODCAST_RECORD_BYTES: usize = 24 * 1024; + +const KEY_KIND_PODCAST: u8 = 1; +const KEY_KIND_TOKEN: u8 = 2; + +/// A 256-bit point in the podcast-discovery key space. +/// +/// `BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind || material)` +/// with kind `0x01` and the podcast id string for the id key, or kind `0x02` +/// and a normalized token for a text key. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct PodcastDhtKey([u8; 32]); + +impl PodcastDhtKey { + /// Key under which every card and tombstone of `podcast_id` is stored. + pub fn podcast(network_id: &NetworkId, podcast_id: &PodcastId) -> Self { + Self::derive(network_id, KEY_KIND_PODCAST, podcast_id.as_str()) + } + + /// Key of one normalized search token (see [`card_index_tokens`]). + pub fn token(network_id: &NetworkId, token: &str) -> Self { + Self::derive(network_id, KEY_KIND_TOKEN, token) + } + + fn derive(network_id: &NetworkId, kind: u8, material: &str) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(PODCAST_DHT_KEY_DOMAIN.as_bytes()); + hasher.update(&[0]); + hasher.update(network_id.as_bytes()); + hasher.update(&[kind]); + hasher.update(material.as_bytes()); + Self(*hasher.finalize().as_bytes()) + } + + /// Creates a key from raw bytes received on the wire. + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Raw key bytes. + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +/// Indexed search tokens of a card: distinct [`normalize_name`] tokens of the +/// title, then of the author, in order, skipping tokens longer than +/// [`MAX_INDEX_TOKEN_BYTES`], at most [`MAX_CARD_INDEX_TOKENS`]. +pub fn card_index_tokens(title: &str, author: Option<&str>) -> Vec { + let mut seen = HashSet::new(); + let mut tokens = Vec::new(); + let sources = std::iter::once(title).chain(author); + for source in sources { + for token in tokenize(&normalize_name(source)) { + if tokens.len() == MAX_CARD_INDEX_TOKENS { + return tokens; + } + if token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()) { + tokens.push(token); + } + } + } + tokens +} + +/// Unsigned public projection of a podcast prepared by its publisher. +/// +/// It is built from the user's subscription ([`PodcastCardSpec::from_subscription`]) +/// and contains only publisher metadata: never progress, listening history, +/// subscription times, operation ids, device names or episodes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastCardSpec { + /// Podcast. + pub podcast_id: PodcastId, + /// Canonical URL the feed is currently fetched from. + pub feed_url: FeedUrl, + /// URL `podcast_id` was derived from, when it differs from `feed_url`. + pub id_feed_url: Option, + /// Podcast title. + pub title: String, + /// Author. + pub author: Option, + /// Plain-text description. + pub description: Option, + /// Artwork URL. + pub image_url: Option, + /// Publisher web page. + pub link: Option, + /// Feed language tag, for example `en` or `pt-BR`. + pub language: Option, + /// Feed categories. + pub categories: Vec, + /// Number of episodes in the feed, when known. + pub episode_count: Option, + /// Publication time of the newest episode in unix milliseconds. + pub latest_episode_at_ms: Option, +} + +impl PodcastCardSpec { + /// Builds the bounded public projection of a subscription. + /// + /// Only `podcast_id`, the feed URLs and the channel metadata are read. + /// `id_feed_url` is the alias that hashes to `podcast_id` + /// ([`crate::podcast::merge_feed_aliases`] always keeps it). Fails when no + /// such URL is known. Callers may then set `language`, `categories`, + /// `episode_count` and `latest_episode_at_ms` from their feed cache. + pub fn from_subscription(subscription: &SubscriptionRecord) -> Result { + let origin = subscription + .feed_urls() + .into_iter() + .find(|url| PodcastId::from_feed_url(url) == subscription.podcast_id) + .ok_or_else(|| { + publication_error("the feed URL the podcast id was derived from is unknown") + })?; + let feed_url = subscription.podcast.feed_url.clone(); + let meta = &subscription.podcast; + Ok(Self { + podcast_id: subscription.podcast_id.clone(), + id_feed_url: (origin != feed_url).then_some(origin), + feed_url, + title: meta.title.clone(), + author: meta.author.clone(), + description: meta.description.clone(), + image_url: meta.image_url.clone(), + link: meta.link.clone(), + language: None, + categories: Vec::new(), + episode_count: None, + latest_episode_at_ms: None, + } + .bounded()) + } + + /// Returns a copy that fits the card bounds: text is truncated with + /// [`bounded_text`], control characters are removed, invalid URLs and + /// language tags are dropped, and categories are trimmed, deduplicated and + /// bounded. An empty title becomes `"Untitled podcast"`. + pub fn bounded(&self) -> Self { + let title = clean_line(&self.title); + let mut seen = HashSet::new(); + let categories = self + .categories + .iter() + .map(|category| bounded_text(&clean_line(category), MAX_CARD_CATEGORY_CHARS)) + .filter(|category| !category.is_empty() && seen.insert(category.to_lowercase())) + .take(MAX_CARD_CATEGORIES) + .collect(); + Self { + podcast_id: self.podcast_id.clone(), + feed_url: self.feed_url.clone(), + id_feed_url: self + .id_feed_url + .clone() + .filter(|origin| *origin != self.feed_url), + title: if title.is_empty() { + "Untitled podcast".to_string() + } else { + bounded_text(&title, MAX_CARD_TITLE_CHARS) + }, + author: bounded_line(self.author.as_deref(), MAX_CARD_AUTHOR_CHARS), + description: self + .description + .as_deref() + .map(|text| bounded_text(clean_block(text).trim(), MAX_CARD_DESCRIPTION_CHARS)) + .filter(|text| !text.is_empty()), + image_url: self.image_url.clone().filter(|url| is_http_url(url)), + link: self.link.clone().filter(|url| is_http_url(url)), + language: self + .language + .as_deref() + .map(str::trim) + .filter(|language| valid_language(language)) + .map(str::to_string), + categories, + episode_count: self.episode_count, + latest_episode_at_ms: self.latest_episode_at_ms.filter(|at| *at >= 0), + } + } + + /// Validates the card bounds without bounding anything. + pub fn validate(&self) -> Result<()> { + let id_url = self.id_feed_url.as_ref().unwrap_or(&self.feed_url); + if PodcastId::from_feed_url(id_url) != self.podcast_id { + return Err(invalid("podcast id does not match its feed URL")); + } + if self.id_feed_url.as_ref() == Some(&self.feed_url) { + return Err(invalid("id_feed_url must differ from feed_url")); + } + check_line("title", &self.title, MAX_CARD_TITLE_CHARS, true)?; + if let Some(author) = &self.author { + check_line("author", author, MAX_CARD_AUTHOR_CHARS, false)?; + } + if let Some(description) = &self.description { + check_chars("description", description, MAX_CARD_DESCRIPTION_CHARS)?; + if description + .chars() + .any(|c| c.is_control() && c != '\n' && c != '\t') + { + return Err(invalid("description contains control characters")); + } + } + for url in [&self.image_url, &self.link].into_iter().flatten() { + if !is_http_url(url) { + return Err(invalid("card URLs must be absolute http(s) URLs")); + } + } + if self + .language + .as_deref() + .is_some_and(|language| !valid_language(language)) + { + return Err(invalid("invalid language tag")); + } + if self.categories.len() > MAX_CARD_CATEGORIES { + return Err(invalid("too many categories")); + } + for category in &self.categories { + check_line("category", category, MAX_CARD_CATEGORY_CHARS, true)?; + } + if self.latest_episode_at_ms.is_some_and(|at| at < 0) { + return Err(invalid("negative latest_episode_at_ms")); + } + Ok(()) + } + + /// Advisory privacy warnings for both feed URLs of the card (see + /// [`feed_url_privacy_warning`]). The UI must ask before publishing a + /// card with warnings. + pub fn privacy_warnings(&self) -> Vec { + std::iter::once(&self.feed_url) + .chain(self.id_feed_url.as_ref()) + .filter_map(feed_url_privacy_warning) + .collect() + } + + /// Indexed search tokens of this card. + pub fn index_tokens(&self) -> Vec { + card_index_tokens(&self.title, self.author.as_deref()) + } +} + +/// A signed public podcast card: one publisher device's projection of one +/// podcast. See `PODCASTS.md`, "Public discovery (podcast_dht)". +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastCard { + /// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]). + pub version: u16, + /// Network scope; prevents replay into another federation. + pub network_id: NetworkId, + /// Publisher device (its persistent transport identity); signs the card. + pub publisher: EndpointId, + /// Podcast. + pub podcast_id: PodcastId, + /// Canonical URL the feed is currently fetched from. + pub feed_url: FeedUrl, + /// URL `podcast_id` was derived from, when it differs from `feed_url`. + pub id_feed_url: Option, + /// Podcast title. + pub title: String, + /// Author. + pub author: Option, + /// Plain-text description. + pub description: Option, + /// Artwork URL. + pub image_url: Option, + /// Publisher web page. + pub link: Option, + /// Feed language tag. + pub language: Option, + /// Feed categories. + pub categories: Vec, + /// Number of episodes in the feed, when known. + pub episode_count: Option, + /// Publication time of the newest episode in unix milliseconds. + pub latest_episode_at_ms: Option, + /// Monotonic per `(podcast_id, publisher)` revision. + pub revision: u64, + /// Unix time at which this version was signed. + pub issued_at_ms: u64, + /// Unix time after which the card must be discarded. + pub expires_at_ms: u64, +} + +impl PodcastCard { + /// Builds a card from a spec. `expires_at_ms` is + /// `issued_at_ms + PODCAST_CARD_TTL`. + pub fn from_spec( + spec: PodcastCardSpec, + network_id: NetworkId, + publisher: EndpointId, + revision: u64, + issued_at_ms: u64, + ) -> Self { + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + network_id, + publisher, + podcast_id: spec.podcast_id, + feed_url: spec.feed_url, + id_feed_url: spec.id_feed_url, + title: spec.title, + author: spec.author, + description: spec.description, + image_url: spec.image_url, + link: spec.link, + language: spec.language, + categories: spec.categories, + episode_count: spec.episode_count, + latest_episode_at_ms: spec.latest_episode_at_ms, + revision, + issued_at_ms, + expires_at_ms: issued_at_ms.saturating_add(PODCAST_CARD_TTL.as_millis() as u64), + } + } + + /// The publisher-provided part of the card. + pub fn spec(&self) -> PodcastCardSpec { + PodcastCardSpec { + podcast_id: self.podcast_id.clone(), + feed_url: self.feed_url.clone(), + id_feed_url: self.id_feed_url.clone(), + title: self.title.clone(), + author: self.author.clone(), + description: self.description.clone(), + image_url: self.image_url.clone(), + link: self.link.clone(), + language: self.language.clone(), + categories: self.categories.clone(), + episode_count: self.episode_count, + latest_episode_at_ms: self.latest_episode_at_ms, + } + } + + /// Indexed search tokens. + pub fn index_tokens(&self) -> Vec { + card_index_tokens(&self.title, self.author.as_deref()) + } + + /// Canonical signing input (see `PODCASTS.md`). + pub fn signing_bytes(&self) -> Vec { + let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN); + out.u16(self.version); + out.raw(self.network_id.as_bytes()); + out.raw(self.publisher.as_bytes()); + out.str(self.podcast_id.as_str()); + out.str(self.feed_url.as_str()); + out.opt_str(self.id_feed_url.as_ref().map(FeedUrl::as_str)); + out.str(&self.title); + out.opt_str(self.author.as_deref()); + out.opt_str(self.description.as_deref()); + out.opt_str(self.image_url.as_deref()); + out.opt_str(self.link.as_deref()); + out.opt_str(self.language.as_deref()); + out.u32(self.categories.len() as u32); + for category in &self.categories { + out.str(category); + } + match self.episode_count { + Some(count) => { + out.u8(1); + out.u32(count); + } + None => out.u8(0), + } + match self.latest_episode_at_ms { + Some(at) => { + out.u8(1); + out.i64(at); + } + None => out.u8(0), + } + out.u64(self.revision); + out.u64(self.issued_at_ms); + out.u64(self.expires_at_ms); + out.finish() + } + + fn validate(&self, expected_network: &NetworkId) -> Result<()> { + check_header(self.version, &self.network_id, expected_network)?; + self.spec().validate()?; + check_lifetime( + self.revision, + self.issued_at_ms, + self.expires_at_ms, + PODCAST_CARD_TTL, + ) + } +} + +/// A signed retraction of one publisher's card of one podcast. +/// +/// It supersedes every card of the same `(podcast_id, publisher)` with a +/// revision lower than or equal to its own and is retained until it expires. +/// `index_keys` lists the text keys of the retracted cards so the tombstone +/// can be stored next to them; it carries hashes, not the tokens. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastTombstone { + /// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]). + pub version: u16, + /// Network scope. + pub network_id: NetworkId, + /// Publisher device; signs the tombstone. + pub publisher: EndpointId, + /// Podcast. + pub podcast_id: PodcastId, + /// Revision; greater than that of every retracted card. + pub revision: u64, + /// Unix time at which the tombstone was signed. + pub issued_at_ms: u64, + /// Unix time after which the tombstone may be discarded. + pub expires_at_ms: u64, + /// Sorted, distinct text keys besides the podcast-id key, at most + /// [`MAX_CARD_INDEX_TOKENS`]. + pub index_keys: Vec, +} + +impl PodcastTombstone { + /// Builds a tombstone. `expires_at_ms` is `issued_at_ms + PODCAST_TOMBSTONE_TTL`; + /// `index_keys` is sorted, deduplicated and stripped of the podcast-id key. + pub fn new( + network_id: NetworkId, + publisher: EndpointId, + podcast_id: PodcastId, + revision: u64, + issued_at_ms: u64, + index_keys: impl IntoIterator, + ) -> Self { + let id_key = PodcastDhtKey::podcast(&network_id, &podcast_id); + let mut index_keys: Vec<_> = index_keys + .into_iter() + .filter(|key| *key != id_key) + .collect(); + index_keys.sort(); + index_keys.dedup(); + index_keys.truncate(MAX_CARD_INDEX_TOKENS); + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + network_id, + publisher, + podcast_id, + revision, + issued_at_ms, + expires_at_ms: issued_at_ms.saturating_add(PODCAST_TOMBSTONE_TTL.as_millis() as u64), + index_keys, + } + } + + /// Canonical signing input (see `PODCASTS.md`). + pub fn signing_bytes(&self) -> Vec { + let mut out = Canonical::new(PODCAST_TOMBSTONE_SIGNING_DOMAIN); + out.u16(self.version); + out.raw(self.network_id.as_bytes()); + out.raw(self.publisher.as_bytes()); + out.str(self.podcast_id.as_str()); + out.u64(self.revision); + out.u64(self.issued_at_ms); + out.u64(self.expires_at_ms); + out.u32(self.index_keys.len() as u32); + for key in &self.index_keys { + out.raw(key.as_bytes()); + } + out.finish() + } + + fn validate(&self, expected_network: &NetworkId) -> Result<()> { + check_header(self.version, &self.network_id, expected_network)?; + if self.index_keys.len() > MAX_CARD_INDEX_TOKENS { + return Err(invalid("tombstone has too many keys")); + } + let id_key = PodcastDhtKey::podcast(&self.network_id, &self.podcast_id); + if self.index_keys.contains(&id_key) + || self.index_keys.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(invalid("tombstone keys are not canonical")); + } + check_lifetime( + self.revision, + self.issued_at_ms, + self.expires_at_ms, + PODCAST_TOMBSTONE_TTL, + ) + } +} + +/// Body of a signed podcast-discovery record. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PodcastRecordBody { + /// An active public card. + Card(Box), + /// A retraction. + Tombstone(PodcastTombstone), +} + +/// End-to-end signed podcast-discovery record. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SignedPodcastRecord { + /// Signed body. + pub body: PodcastRecordBody, + /// Ed25519 signature by `publisher` over [`Self::signing_bytes`]. + pub signature: Signature, +} + +impl SignedPodcastRecord { + /// Validates and signs a card. + pub fn sign_card(card: PodcastCard, signer: impl FnOnce(&[u8]) -> Signature) -> Result { + card.validate(&card.network_id)?; + let signature = signer(&card.signing_bytes()); + let record = Self { + body: PodcastRecordBody::Card(Box::new(card)), + signature, + }; + record.check_size()?; + Ok(record) + } + + /// Validates and signs a tombstone. + pub fn sign_tombstone( + tombstone: PodcastTombstone, + signer: impl FnOnce(&[u8]) -> Signature, + ) -> Result { + tombstone.validate(&tombstone.network_id)?; + let signature = signer(&tombstone.signing_bytes()); + let record = Self { + body: PodcastRecordBody::Tombstone(tombstone), + signature, + }; + record.check_size()?; + Ok(record) + } + + /// Canonical signing input of the body. + pub fn signing_bytes(&self) -> Vec { + match &self.body { + PodcastRecordBody::Card(card) => card.signing_bytes(), + PodcastRecordBody::Tombstone(tombstone) => tombstone.signing_bytes(), + } + } + + /// Validates bounds, network, lifetime shape, encoded size and the + /// publisher's signature. Does not look at the clock; see + /// [`Self::verify_at`]. + pub fn verify(&self, expected_network: &NetworkId) -> Result<()> { + match &self.body { + PodcastRecordBody::Card(card) => card.validate(expected_network)?, + PodcastRecordBody::Tombstone(tombstone) => tombstone.validate(expected_network)?, + } + self.check_size()?; + self.publisher() + .verify(&self.signing_bytes(), &self.signature) + .map_err(|_| invalid("invalid podcast record signature")) + } + + /// [`Self::verify`] plus time checks: not issued more than + /// [`PODCAST_DHT_MAX_CLOCK_SKEW`] in the future and not expired at `now_ms`. + pub fn verify_at(&self, expected_network: &NetworkId, now_ms: u64) -> Result<()> { + self.verify(expected_network)?; + let future_limit = now_ms.saturating_add(PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64); + if self.issued_at_ms() > future_limit || self.expires_at_ms() <= now_ms { + return Err(invalid("podcast record is expired or from the future")); + } + Ok(()) + } + + /// The card, when this is not a tombstone. + pub fn card(&self) -> Option<&PodcastCard> { + match &self.body { + PodcastRecordBody::Card(card) => Some(card.as_ref()), + PodcastRecordBody::Tombstone(_) => None, + } + } + + /// Whether this is a tombstone. + pub fn is_tombstone(&self) -> bool { + matches!(self.body, PodcastRecordBody::Tombstone(_)) + } + + /// Podcast. + pub fn podcast_id(&self) -> &PodcastId { + match &self.body { + PodcastRecordBody::Card(card) => &card.podcast_id, + PodcastRecordBody::Tombstone(tombstone) => &tombstone.podcast_id, + } + } + + /// Publisher device. + pub fn publisher(&self) -> EndpointId { + match &self.body { + PodcastRecordBody::Card(card) => card.publisher, + PodcastRecordBody::Tombstone(tombstone) => tombstone.publisher, + } + } + + /// Network scope. + pub fn network_id(&self) -> NetworkId { + match &self.body { + PodcastRecordBody::Card(card) => card.network_id, + PodcastRecordBody::Tombstone(tombstone) => tombstone.network_id, + } + } + + /// Revision. + pub fn revision(&self) -> u64 { + match &self.body { + PodcastRecordBody::Card(card) => card.revision, + PodcastRecordBody::Tombstone(tombstone) => tombstone.revision, + } + } + + /// Signing time. + pub fn issued_at_ms(&self) -> u64 { + match &self.body { + PodcastRecordBody::Card(card) => card.issued_at_ms, + PodcastRecordBody::Tombstone(tombstone) => tombstone.issued_at_ms, + } + } + + /// Expiry time. + pub fn expires_at_ms(&self) -> u64 { + match &self.body { + PodcastRecordBody::Card(card) => card.expires_at_ms, + PodcastRecordBody::Tombstone(tombstone) => tombstone.expires_at_ms, + } + } + + /// Every DHT key this record belongs under: the podcast-id key, then the + /// text keys (card tokens or the tombstone's `index_keys`). + pub fn dht_keys(&self) -> Vec { + let network_id = self.network_id(); + let mut keys = vec![PodcastDhtKey::podcast(&network_id, self.podcast_id())]; + match &self.body { + PodcastRecordBody::Card(card) => keys.extend( + card.index_tokens() + .iter() + .map(|token| PodcastDhtKey::token(&network_id, token)), + ), + PodcastRecordBody::Tombstone(tombstone) => { + keys.extend(tombstone.index_keys.iter().copied()); + } + } + keys + } + + /// Whether the record may be stored under `key`. + pub fn belongs_under(&self, key: &PodcastDhtKey) -> bool { + self.dht_keys().contains(key) + } + + /// postcard-encoded size. + pub fn encoded_len(&self) -> usize { + postcard::to_stdvec(self).map_or(usize::MAX, |bytes| bytes.len()) + } + + fn check_size(&self) -> Result<()> { + if self.encoded_len() > MAX_PODCAST_RECORD_BYTES { + Err(invalid("podcast record is too large")) + } else { + Ok(()) + } + } +} + +/// Total order of two records of the same `(podcast_id, publisher)`. +/// +/// Greater wins: higher `revision`; at an equal revision a tombstone beats a +/// card; then later `issued_at_ms` (a republish); then later +/// `expires_at_ms`; then the lexicographically greater canonical signing +/// input. Every node therefore keeps the same winner whatever the delivery +/// order. +pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) -> Ordering { + left.revision() + .cmp(&right.revision()) + .then_with(|| left.is_tombstone().cmp(&right.is_tombstone())) + .then_with(|| left.issued_at_ms().cmp(&right.issued_at_ms())) + .then_with(|| left.expires_at_ms().cmp(&right.expires_at_ms())) + .then_with(|| left.signing_bytes().cmp(&right.signing_bytes())) +} + +/// Whether `winner` makes `other` obsolete everywhere, not just under one key: +/// `other` is a card of a lower revision, or `winner` is a tombstone of an +/// equal or higher revision. +pub(crate) fn supersedes_everywhere( + winner: &SignedPodcastRecord, + other: &SignedPodcastRecord, +) -> bool { + !other.is_tombstone() + && (other.revision() < winner.revision() + || (winner.is_tombstone() && other.revision() <= winner.revision())) +} + +struct Canonical(Vec); + +impl Canonical { + fn new(domain: &str) -> Self { + let mut bytes = Vec::with_capacity(512); + bytes.extend_from_slice(domain.as_bytes()); + bytes.push(0); + Self(bytes) + } + + fn raw(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } + + fn u8(&mut self, value: u8) { + self.0.push(value); + } + + fn u16(&mut self, value: u16) { + self.raw(&value.to_le_bytes()); + } + + fn u32(&mut self, value: u32) { + self.raw(&value.to_le_bytes()); + } + + fn u64(&mut self, value: u64) { + self.raw(&value.to_le_bytes()); + } + + fn i64(&mut self, value: i64) { + self.raw(&value.to_le_bytes()); + } + + fn str(&mut self, value: &str) { + self.u32(value.len() as u32); + self.raw(value.as_bytes()); + } + + fn opt_str(&mut self, value: Option<&str>) { + match value { + Some(value) => { + self.u8(1); + self.str(value); + } + None => self.u8(0), + } + } + + fn finish(self) -> Vec { + self.0 + } +} + +fn check_header(version: u16, network_id: &NetworkId, expected: &NetworkId) -> Result<()> { + if version != PODCAST_DHT_PROTOCOL_VERSION { + return Err(invalid("unsupported podcast record version")); + } + if network_id != expected { + return Err(invalid("podcast record belongs to another network")); + } + Ok(()) +} + +fn check_lifetime( + revision: u64, + issued_at_ms: u64, + expires_at_ms: u64, + ttl: Duration, +) -> Result<()> { + if revision == 0 + || issued_at_ms == 0 + || expires_at_ms <= issued_at_ms + || expires_at_ms - issued_at_ms > ttl.as_millis() as u64 + { + return Err(invalid("invalid podcast record revision or lifetime")); + } + Ok(()) +} + +fn check_chars(field: &'static str, value: &str, max_chars: usize) -> Result<()> { + if value.len() > max_chars * 4 || value.chars().count() > max_chars { + return Err(MusicDhtError::Protocol(format!("card {field} is too long"))); + } + Ok(()) +} + +fn check_line(field: &'static str, value: &str, max_chars: usize, required: bool) -> Result<()> { + if required && value.trim().is_empty() { + return Err(MusicDhtError::Protocol(format!("card {field} is empty"))); + } + check_chars(field, value, max_chars)?; + if value.chars().any(char::is_control) { + return Err(MusicDhtError::Protocol(format!( + "card {field} contains control characters" + ))); + } + Ok(()) +} + +fn valid_language(language: &str) -> bool { + !language.is_empty() + && language.len() <= MAX_CARD_LANGUAGE_BYTES + && language + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + +fn clean_line(text: &str) -> String { + text.split(char::is_control) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" ") + .trim() + .to_string() +} + +fn clean_block(text: &str) -> String { + text.chars() + .filter(|c| !c.is_control() || *c == '\n' || *c == '\t') + .collect() +} + +fn bounded_line(text: Option<&str>, max_chars: usize) -> Option { + text.map(clean_line) + .filter(|text| !text.is_empty()) + .map(|text| bounded_text(&text, max_chars)) +} + +fn invalid(reason: &str) -> MusicDhtError { + MusicDhtError::Protocol(reason.to_string()) +} + +pub(crate) fn publication_error(reason: &str) -> MusicDhtError { + MusicDhtError::PodcastPublication(reason.to_string()) +} + +// Statically bound the worst-case card so `MAX_PODCAST_RECORD_BYTES` is a +// defensive limit, not the one users hit first. +const _: () = assert!( + MAX_CARD_TITLE_CHARS * 4 + + MAX_CARD_AUTHOR_CHARS * 4 + + MAX_CARD_DESCRIPTION_CHARS * 4 + + MAX_CARD_CATEGORIES * MAX_CARD_CATEGORY_CHARS * 4 + + MAX_CARD_LANGUAGE_BYTES + + 4 * MAX_URL_BYTES + + 1024 + <= MAX_PODCAST_RECORD_BYTES +); diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs new file mode 100644 index 0000000..276b897 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -0,0 +1,1258 @@ +//! Public podcast discovery: signed podcast cards in a separate DHT overlay. +//! +//! This overlay implements decision R7 of the podcast plan and the +//! "Public discovery (podcast_dht)" section of `PODCASTS.md`. It follows the +//! pattern of [`crate::similarity_dht`]: a schema-independent Kademlia +//! overlay on its own ALPN ([`PODCAST_DHT_ALPN`]) carried by `music-dht` byte +//! streams, with its own wire version, its own SQLite cache and its own +//! routing contacts. The music DHT schema and `federation-net` are unchanged. +//! +//! * A [`PodcastCard`] is the public projection of one podcast published by +//! one device: feed URL and channel metadata only, signed by the device's +//! persistent identity over a domain-separated canonical encoding. +//! Progress, history, subscription registers and device names never enter +//! a card. +//! * Cards are stored under the podcast-id key and one key per normalized +//! title/author token, so peers outside the publisher's trusted-device group +//! find them by text or by id. +//! * A device publishes only podcasts for which [`effective_share`] holds +//! ([`PodcastDht::publish`], [`PodcastDht::sync_shares`]). Unpublishing +//! writes a signed [`PodcastTombstone`] with the next revision. Nothing is +//! ever sent for a podcast this device never published. +//! * Every record is verified (bounds, network, key placement, lifetime and +//! signature) when it is received and again whenever it is read from the +//! cache. +//! +//! Applications own the serving and maintenance tasks, exactly as for +//! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with +//! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take +//! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`]. + +mod card; +mod privacy; +mod search; +mod store; + +#[cfg(test)] +mod tests; + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use federation_net::{ByteStream, EndpointId, NetworkId, PeerTicket, StreamAcceptor}; +use futures::stream::{self, FuturesUnordered, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Semaphore; +use tracing::{debug, info, warn}; + +use crate::MusicDhtService; +use crate::capabilities::CapabilityManifest; +use crate::error::{MusicDhtError, Result}; +use crate::podcast::{PodcastId, ShareRecord, SubscriptionRecord, effective_share}; +use crate::record::now_ms; +use crate::routing::{ALPHA, K, MAX_LOOKUP_REQUESTS, NodeContact, NodeId, RoutingTable, distance}; + +pub use card::{ + MAX_CARD_AUTHOR_CHARS, MAX_CARD_CATEGORIES, MAX_CARD_CATEGORY_CHARS, + MAX_CARD_DESCRIPTION_CHARS, MAX_CARD_INDEX_TOKENS, MAX_CARD_LANGUAGE_BYTES, + MAX_CARD_TITLE_CHARS, MAX_INDEX_TOKEN_BYTES, MAX_PODCAST_RECORD_BYTES, MAX_RECORD_DHT_KEYS, + PODCAST_CARD_SIGNING_DOMAIN, PODCAST_CARD_TTL, PODCAST_DHT_KEY_DOMAIN, + PODCAST_DHT_MAX_CLOCK_SKEW, PODCAST_DHT_PROTOCOL_VERSION, PODCAST_TOMBSTONE_SIGNING_DOMAIN, + PODCAST_TOMBSTONE_TTL, PodcastCard, PodcastCardSpec, PodcastDhtKey, PodcastRecordBody, + PodcastTombstone, SignedPodcastRecord, card_index_tokens, compare_records, +}; +pub use privacy::{ + FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning, +}; +pub use search::{ + FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, + SearchHitKind, merge_search_results, podcast_query_tokens, +}; +pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION}; + +use card::publication_error; +use search::{aggregate_hits, rank_hits}; +use store::{LocalPublication, PodcastDhtDatabase}; + +/// ALPN of the podcast-discovery overlay. +pub const PODCAST_DHT_ALPN: &[u8] = b"furumi-fd/podcast-dht/1"; +/// Stable identifier advertised in capability manifests. +pub const PODCAST_DHT_ID: &str = "podcast_dht"; +/// Maximum records returned for one key. +pub const MAX_PODCAST_DHT_RECORDS: usize = 128; +/// Maximum entries in one store request. +pub const MAX_PODCAST_STORE_RECORDS: usize = 16; +/// Maximum encoded request. A full store batch of worst-case records fits. +pub const MAX_PODCAST_DHT_REQUEST_BYTES: usize = 512 * 1024; +/// Maximum encoded response. A full key of worst-case records fits. +pub const MAX_PODCAST_DHT_RESPONSE_BYTES: usize = 4 * 1024 * 1024; +/// Maximum hits returned by one search. +pub const MAX_PODCAST_SEARCH_RESULTS: usize = 64; +/// Text keys looked up per search: the longest query tokens. Every replica +/// filters by all query tokens, so one key is enough; three add redundancy. +pub const MAX_PODCAST_QUERY_LOOKUPS: usize = 3; +/// Republish cadence of own cards and tombstones: the main DHT's +/// [`crate::DEFAULT_REPUBLISH_INTERVAL`]. +pub const PODCAST_REPUBLISH_INTERVAL: Duration = crate::config::DEFAULT_REPUBLISH_INTERVAL; +/// Replica expiry sweep: the main DHT's [`crate::DEFAULT_EXPIRE_INTERVAL`]. +pub const PODCAST_EXPIRE_INTERVAL: Duration = crate::config::DEFAULT_EXPIRE_INTERVAL; +/// Periodic bootstrap for discovering upgraded peers. +pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60); +/// Maximum inbound requests processed concurrently; further streams are +/// dropped. +pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64; +/// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is +/// not contacted. +pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60); + +const MAX_BOOTSTRAP_PEERS: usize = 64; +const BOOTSTRAP_CONCURRENCY: usize = 8; +const PUBLISH_CONCURRENCY: usize = 8; +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const SERVE_TIMEOUT: Duration = Duration::from_secs(10); +/// A peer that failed to answer (network error or timeout) is skipped by +/// lookups, publication and bootstrap for this long. +const FAILED_PEER_BACKOFF: Duration = Duration::from_secs(30); +const MAX_TICKET_BYTES: usize = 16 * 1024; +const MAX_ERROR_BYTES: usize = 1024; + +/// One request on [`PODCAST_DHT_ALPN`] (postcard, one per stream). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastDhtRequest { + /// [`PODCAST_DHT_PROTOCOL_VERSION`]. + pub version: u16, + /// Requesting overlay node; must match the authenticated peer. + pub requester: NodeContact, + /// Operation. + pub operation: PodcastDhtOperation, +} + +/// Overlay operations. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PodcastDhtOperation { + /// Returns the `K` closest overlay contacts to `target`. + FindNode { + /// Target id. + target: [u8; 32], + }, + /// Returns records stored under `key` and the closest contacts. + FindValue { + /// Key. + key: PodcastDhtKey, + /// Normalized tokens every returned card must be indexed under + /// (tombstones are always returned); at most + /// [`MAX_PODCAST_QUERY_TOKENS`]. + filter_tokens: Vec, + }, + /// Stores records; at most [`MAX_PODCAST_STORE_RECORDS`] entries. + StoreBatch { + /// Entries. + records: Vec, + }, +} + +/// A record and the keys it is to be stored under on the receiving node. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastStoreEntry { + /// Distinct keys the record belongs under, at most [`MAX_RECORD_DHT_KEYS`]. + pub keys: Vec, + /// Signed record. + pub record: SignedPodcastRecord, +} + +/// Response to a [`PodcastDhtRequest`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastDhtResponse { + /// [`PODCAST_DHT_PROTOCOL_VERSION`]. + pub version: u16, + /// Whether the request was accepted. + pub ok: bool, + /// Diagnostic when `ok` is false, at most 1 KiB. + pub error: Option, + /// Closest overlay contacts, at most `K`. + pub nodes: Vec, + /// Records, at most [`MAX_PODCAST_DHT_RECORDS`]. + pub records: Vec, + /// `(key, record)` rows stored by a `StoreBatch`. + pub stored: u32, +} + +impl PodcastDhtResponse { + fn success(nodes: Vec, records: Vec, stored: u32) -> Self { + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: true, + error: None, + nodes, + records, + stored, + } + } + + fn error(message: impl Into) -> Self { + let mut message = message.into(); + if message.len() > MAX_ERROR_BYTES { + let mut end = MAX_ERROR_BYTES; + while !message.is_char_boundary(end) { + end -= 1; + } + message.truncate(end); + } + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: false, + error: Some(message), + nodes: Vec::new(), + records: Vec::new(), + stored: 0, + } + } +} + +/// Statistics of one publication round. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PodcastDhtPublishStats { + /// Signed records pushed. + pub records: usize, + /// `(key, record)` placements pushed. + pub keys: usize, + /// Remote overlay nodes that stored at least one placement. + pub remote_nodes: usize, + /// Whether this node kept at least one replica itself. + pub local_replica: bool, +} + +/// Result of [`PodcastDht::sync_shares`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PodcastDhtSyncStats { + /// Cards published with a new revision, or refreshed because they used + /// half their lifetime. + pub published: usize, + /// Shared podcasts whose current card was already published. + pub unchanged: usize, + /// Podcasts unpublished with a tombstone. + pub unpublished: usize, + /// Network statistics of the pushed records. + pub push: PodcastDhtPublishStats, +} + +/// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastShareState { + /// Merged subscription register. + pub subscription: SubscriptionRecord, + /// Merged share register, if any. + pub share: Option, + /// Card to publish when shared; `None` derives it with + /// [`PodcastCardSpec::from_subscription`]. + pub card: Option, +} + +impl PodcastShareState { + /// Whether the podcast must be published ([`effective_share`]). + pub fn is_shared(&self) -> bool { + effective_share(Some(&self.subscription), self.share.as_ref()) + } +} + +/// This device's publication state of one podcast. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublishedPodcast { + /// Podcast. + pub podcast_id: PodcastId, + /// Latest revision used by this device. + pub revision: u64, + /// Whether a card is published (false after unpublishing). + pub active: bool, + /// Latest signed record (card or tombstone). + pub record: SignedPodcastRecord, +} + +/// Podcast-discovery overlay attached to a [`MusicDhtService`]. +pub struct PodcastDht { + service: Arc, + database: PodcastDhtDatabase, + network_id: NetworkId, + own_contact: NodeContact, + routing: Mutex, + unsupported: Mutex>, + failed: Mutex>, + publish_gate: Semaphore, + serve_limit: Arc, +} + +impl std::fmt::Debug for PodcastDht { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PodcastDht") + .field("endpoint_id", &self.own_contact.peer_id) + .field("known_peers", &lock(&self.routing).len()) + .finish_non_exhaustive() + } +} + +impl PodcastDht { + /// Opens the overlay cache at `database_path` and restores previously + /// learned overlay contacts. The caller owns the serving and maintenance + /// tasks so they share the application's shutdown lifecycle. + pub async fn open( + service: Arc, + database_path: impl AsRef, + ) -> Result> { + let network_id = service.network_id(); + let database = PodcastDhtDatabase::open(database_path.as_ref(), network_id).await?; + let ticket = service.ticket().await?.to_string(); + let owner = service.endpoint_id(); + let own_contact = NodeContact { + node_id: NodeId::from_endpoint(&owner), + peer_id: owner, + ticket, + last_seen_ms: now_ms(), + }; + let mut routing = RoutingTable::new(own_contact.node_id); + for contact in database.load_peers().await? { + if validate_contact(&contact, &network_id, Some(owner)).is_ok() { + routing.upsert(contact); + } + } + Ok(Arc::new(Self { + service, + database, + network_id, + own_contact, + routing: Mutex::new(routing), + unsupported: Mutex::new(HashMap::new()), + failed: Mutex::new(HashMap::new()), + publish_gate: Semaphore::new(1), + serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), + })) + } + + /// Number of known peers that implement the overlay. + pub fn known_peers(&self) -> usize { + lock(&self.routing).len() + } + + /// Serves authenticated overlay requests until `acceptor` closes. + pub async fn serve(self: Arc, mut acceptor: StreamAcceptor) { + while let Some(stream) = acceptor.accept().await { + let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else { + debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded"); + continue; + }; + let this = Arc::clone(&self); + tokio::spawn(async move { + let _permit = permit; + let peer = stream.peer_id; + match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await { + Ok(Ok(())) => {} + Ok(Err(error)) => debug!(%peer, %error, "podcast DHT request failed"), + Err(_) => debug!(%peer, "podcast DHT request timed out"), + } + }); + } + } + + /// Periodically discovers overlay peers, republishes own cards and + /// tombstones every [`PODCAST_REPUBLISH_INTERVAL`] and removes expired + /// replicas. The first bootstrap runs immediately. + pub async fn maintenance(self: Arc) { + let mut bootstrap = tokio::time::interval(PODCAST_BOOTSTRAP_INTERVAL); + let mut republish = tokio::time::interval(PODCAST_REPUBLISH_INTERVAL); + let mut expire = tokio::time::interval(PODCAST_EXPIRE_INTERVAL); + bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + republish.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + expire.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + republish.tick().await; + loop { + tokio::select! { + _ = bootstrap.tick() => { + let before = self.known_peers(); + if let Err(error) = self.bootstrap().await { + debug!(%error, "podcast DHT bootstrap failed"); + } else if self.known_peers() > before + && let Err(error) = self.republish().await + { + warn!(%error, "podcast DHT publication after bootstrap failed"); + } + } + _ = republish.tick() => { + if let Err(error) = self.republish().await { + warn!(%error, "podcast DHT republish failed"); + } + } + _ = expire.tick() => { + match self.database.delete_expired(now_ms()).await { + Ok(0) => {} + Ok(count) => debug!(count, "expired podcast DHT replicas removed"), + Err(error) => warn!(%error, "podcast DHT expiry sweep failed"), + } + } + } + } + } + + /// Records what a peer's capability manifest says about this overlay. + /// + /// A peer whose manifest lacks [`PODCAST_DHT_ID`] at + /// [`PODCAST_DHT_PROTOCOL_VERSION`] is removed from the overlay routing + /// table and not contacted for [`UNSUPPORTED_PEER_BACKOFF`]. Peers only + /// enter the routing table after answering on [`PODCAST_DHT_ALPN`] or + /// being returned by an overlay peer, so a peer without the overlay is + /// never queried for records or asked to store them. + pub async fn note_peer_capabilities( + &self, + peer: EndpointId, + manifest: &CapabilityManifest, + ) -> Result<()> { + if manifest.supports_podcast_dht() { + lock(&self.unsupported).remove(&peer); + return Ok(()); + } + let until = now_ms().saturating_add(UNSUPPORTED_PEER_BACKOFF.as_millis() as u64); + lock(&self.unsupported).insert(peer, until); + lock(&self.routing).remove(&peer); + self.database.remove_peer(peer).await + } + + /// Discovers overlay peers among the routing table and the music DHT + /// contacts. + pub async fn bootstrap(&self) -> Result<()> { + let known_feature: HashSet = lock(&self.routing) + .contacts() + .into_iter() + .map(|contact| contact.peer_id) + .collect(); + let mut candidates = lock(&self.routing).contacts(); + candidates.extend( + self.service + .known_peers() + .into_iter() + .filter(|contact| !known_feature.contains(&contact.peer_id)), + ); + let mut seen = HashSet::new(); + candidates.retain(|contact| { + contact.peer_id != self.own_contact.peer_id + && !self.is_unsupported(&contact.peer_id) + && !self.is_backed_off(&contact.peer_id) + && seen.insert(contact.peer_id) + }); + candidates.truncate(MAX_BOOTSTRAP_PEERS); + let target = *self.own_contact.node_id.as_bytes(); + let responses = stream::iter(candidates.into_iter().map(|contact| async move { + let response = self + .exchange(&contact, PodcastDhtOperation::FindNode { target }) + .await; + (contact, response) + })) + .buffer_unordered(BOOTSTRAP_CONCURRENCY) + .collect::>() + .await; + for (contact, response) in responses { + let response = match response { + Ok(response) => response, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT bootstrap peer unavailable"); + continue; + } + }; + self.learn_contact(contact).await?; + self.learn_response_nodes(response.nodes).await; + } + Ok(()) + } + + /// Publishes (or refreshes) this device's card of a podcast. + /// + /// Refused with [`MusicDhtError::PodcastPublication`] unless + /// [`effective_share`] holds for the given registers and they and `card` + /// describe the same podcast. `card` is bounded with + /// [`PodcastCardSpec::bounded`]. A changed card gets revision + /// `max(previous + 1, now_ms)`; an unchanged one keeps its revision and + /// is re-signed with a fresh lifetime. + /// + /// Call [`feed_url_privacy_warning`] (or + /// [`PodcastCardSpec::privacy_warnings`]) first and ask the user before + /// publishing a URL that looks private. + pub async fn publish( + &self, + subscription: &SubscriptionRecord, + share: &ShareRecord, + card: PodcastCardSpec, + ) -> Result { + if card.podcast_id != subscription.podcast_id { + return Err(publication_error( + "card and subscription describe different podcasts", + )); + } + if !effective_share(Some(subscription), Some(share)) { + return Err(publication_error("the podcast is not shared")); + } + let card = card.bounded(); + card.validate()?; + let _permit = self.publish_permit().await?; + let record = self.sign_card_locked(card, true).await?; + self.push(record.into_iter().collect()).await + } + + /// Unpublishes this device's card of `podcast_id` with a tombstone of + /// revision `previous + 1`, retained by replicas for + /// [`PODCAST_TOMBSTONE_TTL`]. + /// + /// Does nothing, and sends nothing, when this device never published the + /// podcast; re-sends the existing tombstone when already unpublished. + pub async fn unpublish(&self, podcast_id: &PodcastId) -> Result { + let _permit = self.publish_permit().await?; + let record = self.unpublish_locked(podcast_id).await?; + self.push(record.into_iter().collect()).await + } + + /// Declaratively reconciles this device's publications with the merged + /// personal registers. + /// + /// Podcasts for which [`effective_share`] holds are published when their + /// card changed, is not published yet or has used half its lifetime; + /// otherwise they count as unchanged (periodic refresh is + /// [`Self::republish`]). Every other podcast this device published, + /// including podcasts absent from `states`, is unpublished. The call is + /// idempotent: repeating it publishes nothing new. + pub async fn sync_shares(&self, states: Vec) -> Result { + let mut desired: HashMap = HashMap::new(); + for state in states { + if !state.is_shared() { + continue; + } + let card = match state.card { + Some(card) => card.bounded(), + None => PodcastCardSpec::from_subscription(&state.subscription)?, + }; + if card.podcast_id != state.subscription.podcast_id { + return Err(publication_error( + "card and subscription describe different podcasts", + )); + } + card.validate()?; + desired.insert(card.podcast_id.clone(), card); + } + + let _permit = self.publish_permit().await?; + let local = self.database.local_publications().await?; + let mut stats = PodcastDhtSyncStats::default(); + let mut records = Vec::new(); + for publication in &local { + if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) { + records.extend(self.unpublish_locked(&publication.podcast_id).await?); + stats.unpublished += 1; + } + } + let mut desired: Vec<_> = desired.into_values().collect(); + desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id)); + for card in desired { + match self.sign_card_locked(card, false).await? { + Some(record) => { + records.push(record); + stats.published += 1; + } + None => stats.unchanged += 1, + } + } + stats.push = self.push(records).await?; + Ok(stats) + } + + /// Re-signs every published card with a fresh lifetime (same revision) + /// and re-sends every unexpired tombstone. Called by + /// [`Self::maintenance`] every [`PODCAST_REPUBLISH_INTERVAL`]. + pub async fn republish(&self) -> Result { + let _permit = self.publish_permit().await?; + let now = now_ms(); + let mut records = Vec::new(); + for publication in self.database.local_publications().await? { + match publication.spec { + Some(spec) => records.extend(self.sign_card_locked(spec, true).await?), + None if publication.record.expires_at_ms() > now => { + records.push(publication.record); + } + None => {} + } + } + if records.is_empty() { + return Ok(PodcastDhtPublishStats::default()); + } + let stats = self.push(records).await?; + info!( + records = stats.records, + nodes = stats.remote_nodes, + "podcast DHT records republished" + ); + Ok(stats) + } + + /// This device's publication state, sorted by podcast id. + pub async fn published(&self) -> Result> { + Ok(self + .database + .local_publications() + .await? + .into_iter() + .map(|publication| PublishedPodcast { + podcast_id: publication.podcast_id, + revision: publication.revision, + active: publication.spec.is_some(), + record: publication.record, + }) + .collect()) + } + + /// Finds public podcasts whose title or author tokens contain every + /// normalized token of `query` ([`podcast_query_tokens`]). + /// + /// Hits are merged by `podcast_id` over publishers and ranked: exact + /// normalized title first, then more publishers. Partial results are + /// returned when peers fail. + pub async fn search(&self, query: &str, limit: usize) -> Result> { + let tokens = podcast_query_tokens(query); + if tokens.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + if self.known_peers() == 0 { + self.bootstrap().await?; + } + let mut lookup_tokens = tokens.clone(); + lookup_tokens.sort_by(|left, right| right.len().cmp(&left.len()).then(left.cmp(right))); + lookup_tokens.truncate(MAX_PODCAST_QUERY_LOOKUPS); + let outcomes = stream::iter(lookup_tokens.into_iter().map(|token| { + let filter = tokens.clone(); + async move { + self.lookup(PodcastDhtKey::token(&self.network_id, &token), filter) + .await + } + })) + .buffer_unordered(MAX_PODCAST_QUERY_LOOKUPS) + .collect::>() + .await; + let mut records = Vec::new(); + for outcome in outcomes { + match outcome { + Ok(found) => records.extend(found), + Err(error) => debug!(%error, "podcast DHT text lookup failed"), + } + } + let mut hits = rank_hits(aggregate_hits(records), query, &tokens); + hits.truncate(limit.min(MAX_PODCAST_SEARCH_RESULTS)); + Ok(hits) + } + + /// Finds the public cards of one podcast. + pub async fn lookup_podcast(&self, podcast_id: &PodcastId) -> Result> { + if self.known_peers() == 0 { + self.bootstrap().await?; + } + let records = self + .lookup( + PodcastDhtKey::podcast(&self.network_id, podcast_id), + Vec::new(), + ) + .await?; + Ok(aggregate_hits(records) + .into_iter() + .find(|hit| hit.podcast_id == *podcast_id)) + } + + async fn publish_permit(&self) -> Result> { + self.publish_gate + .acquire() + .await + .map_err(|_| publication_error("podcast DHT publisher closed")) + } + + /// Signs and records a card. With `force` false, an unchanged active card + /// that is valid for more than half its TTL is left alone and `None` is + /// returned. The publish gate must be held. + async fn sign_card_locked( + &self, + spec: PodcastCardSpec, + force: bool, + ) -> Result> { + let now = now_ms(); + let previous = self + .database + .local_publication(spec.podcast_id.clone()) + .await?; + let revision = match &previous { + Some(previous) if previous.spec.as_ref() == Some(&spec) => { + // Unchanged and still fresh: leave it to the republish loop. + let fresh_until = now.saturating_add(PODCAST_CARD_TTL.as_millis() as u64 / 2); + if !force && previous.record.expires_at_ms() > fresh_until { + return Ok(None); + } + previous.revision + } + Some(previous) => previous.revision.saturating_add(1).max(now), + None => now.max(1), + }; + let card = PodcastCard::from_spec( + spec.clone(), + self.network_id, + self.own_contact.peer_id, + revision, + now, + ); + let record = + SignedPodcastRecord::sign_card(card, |bytes| self.service.sign_identity(bytes))?; + self.database + .put_local_publication( + LocalPublication { + podcast_id: spec.podcast_id.clone(), + revision, + spec: Some(spec), + record: record.clone(), + }, + now, + ) + .await?; + Ok(Some(record)) + } + + /// Builds the tombstone of an own publication. The publish gate must be + /// held. + async fn unpublish_locked( + &self, + podcast_id: &PodcastId, + ) -> Result> { + let now = now_ms(); + let Some(previous) = self.database.local_publication(podcast_id.clone()).await? else { + return Ok(None); + }; + if previous.spec.is_none() { + return Ok( + (previous.record.is_tombstone() && previous.record.expires_at_ms() > now) + .then_some(previous.record), + ); + } + let revision = previous.revision.saturating_add(1); + let tombstone = PodcastTombstone::new( + self.network_id, + self.own_contact.peer_id, + podcast_id.clone(), + revision, + now, + previous.record.dht_keys(), + ); + let record = SignedPodcastRecord::sign_tombstone(tombstone, |bytes| { + self.service.sign_identity(bytes) + })?; + self.database + .put_local_publication( + LocalPublication { + podcast_id: podcast_id.clone(), + revision, + spec: None, + record: record.clone(), + }, + now, + ) + .await?; + Ok(Some(record)) + } + + /// Stores records on the `K` closest overlay nodes of each of their keys + /// (including this node when it is among them). + async fn push(&self, records: Vec) -> Result { + let mut stats = PodcastDhtPublishStats { + records: records.len(), + ..PodcastDhtPublishStats::default() + }; + if records.is_empty() { + return Ok(stats); + } + let mut local = Vec::new(); + let mut remote: HashMap)> = HashMap::new(); + for record in records { + let mut local_keys = Vec::new(); + let mut remote_keys: HashMap)> = + HashMap::new(); + for key in record.dht_keys() { + stats.keys += 1; + let (store_local, contacts) = self.replica_targets(&key); + if store_local { + local_keys.push(key); + } + for contact in contacts { + remote_keys + .entry(contact.peer_id) + .or_insert_with(|| (contact, Vec::new())) + .1 + .push(key); + } + } + if !local_keys.is_empty() { + local.push((local_keys, record.clone())); + } + for (peer, (contact, keys)) in remote_keys { + remote + .entry(peer) + .or_insert_with(|| (contact, Vec::new())) + .1 + .push(PodcastStoreEntry { + keys, + record: record.clone(), + }); + } + } + if !local.is_empty() { + self.database.store(local, now_ms()).await?; + stats.local_replica = true; + } + let sends = stream::iter(remote.into_values().map(|(contact, entries)| async move { + let mut accepted = false; + for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + let response = self + .exchange( + &contact, + PodcastDhtOperation::StoreBatch { + records: batch.to_vec(), + }, + ) + .await; + match response { + Ok(response) => accepted |= response.stored > 0, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); + break; + } + } + } + accepted + })) + .buffer_unordered(PUBLISH_CONCURRENCY) + .collect::>() + .await; + stats.remote_nodes = sends.into_iter().filter(|accepted| *accepted).count(); + Ok(stats) + } + + /// Iterative lookup of `key`: queries the `K` closest known overlay + /// nodes until all have answered, the request budget is spent or the + /// deadline passes. Returns the winning verified record per + /// `(podcast_id, publisher)`. + async fn lookup( + &self, + key: PodcastDhtKey, + filter_tokens: Vec, + ) -> Result> { + let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new(); + let local = self + .database + .records_by_key( + key, + filter_tokens.clone(), + now_ms(), + MAX_PODCAST_DHT_RECORDS, + ) + .await?; + merge_records(&mut winners, local, &self.network_id, &key, &filter_tokens); + + let mut candidates = lock(&self.routing).closest(key.as_bytes(), K); + let mut known: HashSet<_> = candidates.iter().map(|contact| contact.peer_id).collect(); + let mut queried = HashSet::new(); + let mut sent = 0usize; + let deadline = tokio::time::Instant::now() + LOOKUP_TIMEOUT; + while tokio::time::Instant::now() < deadline && sent < MAX_LOOKUP_REQUESTS { + candidates.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + let batch: Vec<_> = candidates + .iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) + .take(K) + .filter(|contact| !queried.contains(&contact.peer_id)) + .take(ALPHA.min(MAX_LOOKUP_REQUESTS - sent)) + .cloned() + .collect(); + if batch.is_empty() { + break; + } + sent += batch.len(); + queried.extend(batch.iter().map(|contact| contact.peer_id)); + let operation = PodcastDhtOperation::FindValue { + key, + filter_tokens: filter_tokens.clone(), + }; + let mut futures: FuturesUnordered<_> = batch + .iter() + .map(|contact| { + let operation = operation.clone(); + async move { self.exchange(contact, operation).await } + }) + .collect(); + while let Ok(Some(response)) = tokio::time::timeout_at(deadline, futures.next()).await { + let Ok(response) = response else { + continue; + }; + merge_records( + &mut winners, + response.records, + &self.network_id, + &key, + &filter_tokens, + ); + for node in response.nodes { + if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)) + .is_ok() + && !self.is_unsupported(&node.peer_id) + && known.insert(node.peer_id) + { + self.learn_contact(node.clone()).await?; + candidates.push(node); + } + } + } + } + Ok(winners.into_values().collect()) + } + + async fn serve_one(&self, mut stream: ByteStream) -> Result<()> { + let authenticated_peer = stream.peer_id; + let response = match read_postcard::<_, PodcastDhtRequest>( + &mut stream.recv, + MAX_PODCAST_DHT_REQUEST_BYTES, + ) + .await + { + Ok(request) => match self.handle_request(authenticated_peer, request).await { + Ok(response) => response, + Err(error) => PodcastDhtResponse::error(error.to_string()), + }, + Err(error) => PodcastDhtResponse::error(error.to_string()), + }; + write_postcard(&mut stream.send, &response, MAX_PODCAST_DHT_RESPONSE_BYTES).await?; + stream.send.finish().map_err(network_error)?; + let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await; + Ok(()) + } + + async fn handle_request( + &self, + authenticated_peer: EndpointId, + request: PodcastDhtRequest, + ) -> Result { + if request.version != PODCAST_DHT_PROTOCOL_VERSION { + return Err(protocol_error("unsupported podcast DHT request version")); + } + if request.requester.peer_id != authenticated_peer { + return Err(protocol_error("podcast DHT requester identity mismatch")); + } + self.learn_contact(request.requester).await?; + match request.operation { + PodcastDhtOperation::FindNode { target } => Ok(PodcastDhtResponse::success( + self.closest_contacts(&target, K), + Vec::new(), + 0, + )), + PodcastDhtOperation::FindValue { key, filter_tokens } => { + validate_filter_tokens(&filter_tokens)?; + let records = self + .database + .records_by_key(key, filter_tokens, now_ms(), MAX_PODCAST_DHT_RECORDS) + .await?; + Ok(PodcastDhtResponse::success( + self.closest_contacts(key.as_bytes(), K), + records, + 0, + )) + } + PodcastDhtOperation::StoreBatch { records } => { + let entries = validate_store_batch(records, &self.network_id, now_ms())?; + let stored = self.database.store(entries, now_ms()).await?; + Ok(PodcastDhtResponse::success( + Vec::new(), + Vec::new(), + u32::try_from(stored).unwrap_or(u32::MAX), + )) + } + } + } + + /// One request with [`REQUEST_TIMEOUT`]. Network failures and timeouts + /// put the peer in a short backoff; a success clears it. + async fn exchange( + &self, + contact: &NodeContact, + operation: PodcastDhtOperation, + ) -> Result { + let result = tokio::time::timeout(REQUEST_TIMEOUT, self.exchange_once(contact, operation)) + .await + .unwrap_or(Err(MusicDhtError::Timeout)); + match &result { + Ok(_) => { + lock(&self.failed).remove(&contact.peer_id); + } + Err(MusicDhtError::Network(_) | MusicDhtError::Timeout) => { + let until = now_ms().saturating_add(FAILED_PEER_BACKOFF.as_millis() as u64); + lock(&self.failed).insert(contact.peer_id, until); + } + Err(_) => {} + } + result + } + + async fn exchange_once( + &self, + contact: &NodeContact, + operation: PodcastDhtOperation, + ) -> Result { + validate_contact(contact, &self.network_id, Some(self.own_contact.peer_id))?; + let ticket = PeerTicket::from_str(&contact.ticket) + .map_err(|error| MusicDhtError::InvalidTicket(error.to_string()))?; + let mut stream = self + .service + .open_stream_to(&ticket, PODCAST_DHT_ALPN) + .await?; + let request = PodcastDhtRequest { + version: PODCAST_DHT_PROTOCOL_VERSION, + requester: self.fresh_own_contact(), + operation, + }; + write_postcard(&mut stream.send, &request, MAX_PODCAST_DHT_REQUEST_BYTES).await?; + stream.send.finish().map_err(network_error)?; + let response: PodcastDhtResponse = + read_postcard(&mut stream.recv, MAX_PODCAST_DHT_RESPONSE_BYTES).await?; + validate_response(&response)?; + if !response.ok { + return Err(protocol_error( + response + .error + .as_deref() + .unwrap_or("podcast DHT request refused"), + )); + } + Ok(response) + } + + fn is_unsupported(&self, peer: &EndpointId) -> bool { + still_listed(&self.unsupported, peer) + } + + fn is_backed_off(&self, peer: &EndpointId) -> bool { + still_listed(&self.failed, peer) + } + + fn replica_targets(&self, key: &PodcastDhtKey) -> (bool, Vec) { + let mut contacts: Vec<_> = lock(&self.routing) + .contacts() + .into_iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) + .collect(); + contacts.push(self.fresh_own_contact()); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + contacts.dedup_by_key(|contact| contact.peer_id); + contacts.truncate(K); + let local = contacts + .iter() + .any(|contact| contact.peer_id == self.own_contact.peer_id); + contacts.retain(|contact| contact.peer_id != self.own_contact.peer_id); + (local, contacts) + } + + fn closest_contacts(&self, target: &[u8; 32], count: usize) -> Vec { + let mut contacts = lock(&self.routing).contacts(); + contacts.push(self.fresh_own_contact()); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), target)); + contacts.dedup_by_key(|contact| contact.peer_id); + contacts.truncate(count); + contacts + } + + fn fresh_own_contact(&self) -> NodeContact { + let mut contact = self.own_contact.clone(); + contact.last_seen_ms = now_ms(); + contact + } + + async fn learn_response_nodes(&self, nodes: Vec) { + for contact in nodes { + if self.is_unsupported(&contact.peer_id) { + continue; + } + if let Err(error) = self.learn_contact(contact).await { + debug!(%error, "invalid podcast DHT contact ignored"); + } + } + } + + async fn learn_contact(&self, mut contact: NodeContact) -> Result<()> { + validate_contact(&contact, &self.network_id, Some(self.own_contact.peer_id))?; + contact.node_id = NodeId::from_endpoint(&contact.peer_id); + // A remote peer does not get to extend its own lifetime with a future + // timestamp; the authenticated contact is the local observation. + contact.last_seen_ms = now_ms(); + let is_new = lock(&self.routing).upsert(contact.clone()); + self.database.upsert_peer(contact.clone()).await?; + if is_new { + debug!(peer = %contact.peer_id, "learned podcast DHT peer"); + } + Ok(()) + } +} + +/// Validates a store batch received from a peer: bounds, every record's +/// signature and lifetime, and that every key belongs to its record. +fn validate_store_batch( + entries: Vec, + network_id: &NetworkId, + now_ms: u64, +) -> Result, SignedPodcastRecord)>> { + if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS { + return Err(protocol_error("invalid podcast DHT store batch")); + } + entries + .into_iter() + .map(|entry| { + let mut keys = entry.keys; + let unique: HashSet<_> = keys.iter().collect(); + if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() { + return Err(protocol_error("invalid podcast DHT store keys")); + } + entry.record.verify_at(network_id, now_ms)?; + let valid_keys = entry.record.dht_keys(); + if keys.iter().any(|key| !valid_keys.contains(key)) { + return Err(protocol_error("podcast record does not belong under key")); + } + keys.sort(); + Ok((keys, entry.record)) + }) + .collect() +} + +fn validate_filter_tokens(tokens: &[String]) -> Result<()> { + if tokens.len() > MAX_PODCAST_QUERY_TOKENS + || tokens + .iter() + .any(|token| token.is_empty() || token.len() > MAX_INDEX_TOKEN_BYTES) + { + return Err(protocol_error("invalid podcast DHT filter tokens")); + } + Ok(()) +} + +/// Adds verified records found under `key` to the per-owner winners. +fn merge_records( + winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, + records: Vec, + network_id: &NetworkId, + key: &PodcastDhtKey, + filter_tokens: &[String], +) { + let now = now_ms(); + for record in records { + if record.verify_at(network_id, now).is_err() || !record.belongs_under(key) { + continue; + } + if let Some(card) = record.card() { + let tokens = card.index_tokens(); + if !filter_tokens.iter().all(|token| tokens.contains(token)) { + continue; + } + } + let owner = (record.podcast_id().clone(), record.publisher()); + match winners.get(&owner) { + Some(current) if compare_records(current, &record).is_ge() => {} + _ => { + winners.insert(owner, record); + } + } + } +} + +fn validate_contact( + contact: &NodeContact, + network_id: &NetworkId, + excluded: Option, +) -> Result<()> { + if excluded == Some(contact.peer_id) + || contact.node_id != NodeId::from_endpoint(&contact.peer_id) + || contact.ticket.len() > MAX_TICKET_BYTES + { + return Err(protocol_error("invalid podcast DHT contact")); + } + let ticket = PeerTicket::from_str(&contact.ticket) + .map_err(|error| MusicDhtError::InvalidTicket(error.to_string()))?; + if ticket.endpoint_id() != contact.peer_id || &ticket.network_id != network_id { + return Err(protocol_error("podcast DHT contact ticket mismatch")); + } + Ok(()) +} + +fn validate_response(response: &PodcastDhtResponse) -> Result<()> { + if response.version != PODCAST_DHT_PROTOCOL_VERSION + || response.nodes.len() > K + || response.records.len() > MAX_PODCAST_DHT_RECORDS + || response + .error + .as_ref() + .is_some_and(|error| error.len() > MAX_ERROR_BYTES) + || (response.ok && response.error.is_some()) + || (!response.ok && response.error.is_none()) + { + return Err(protocol_error("invalid podcast DHT response")); + } + Ok(()) +} + +async fn write_postcard( + writer: &mut W, + value: &T, + maximum: usize, +) -> Result<()> { + let payload = postcard::to_stdvec(value).map_err(protocol_error)?; + if payload.len() > maximum { + return Err(protocol_error("podcast DHT message is too large")); + } + writer.write_all(&payload).await.map_err(network_error) +} + +async fn read_postcard Deserialize<'de>>( + reader: &mut R, + maximum: usize, +) -> Result { + let mut payload = Vec::new(); + reader + .take(maximum as u64 + 1) + .read_to_end(&mut payload) + .await + .map_err(network_error)?; + if payload.is_empty() || payload.len() > maximum { + return Err(protocol_error("invalid podcast DHT message size")); + } + postcard::from_bytes(&payload).map_err(protocol_error) +} + +/// Whether `peer` is in a timed list; expired entries are removed. +fn still_listed(list: &Mutex>, peer: &EndpointId) -> bool { + let now = now_ms(); + let mut list = lock(list); + match list.get(peer) { + Some(until) if *until > now => true, + Some(_) => { + list.remove(peer); + false + } + None => false, + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn protocol_error(error: impl std::fmt::Display) -> MusicDhtError { + MusicDhtError::Protocol(error.to_string()) +} + +fn network_error(error: impl std::fmt::Display) -> MusicDhtError { + MusicDhtError::Network(error.to_string()) +} diff --git a/crates/music-dht/src/podcast_dht/privacy.rs b/crates/music-dht/src/podcast_dht/privacy.rs new file mode 100644 index 0000000..c298682 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/privacy.rs @@ -0,0 +1,339 @@ +//! Advisory detection of feed URLs that look private. +//! +//! Premium and member feeds usually carry a personal secret in the URL. A +//! card publishes its feed URL verbatim, so the UI must ask the user before +//! publishing a URL for which [`feed_url_privacy_warning`] returns a warning. +//! The heuristics are deliberately broad and can produce false positives +//! (and miss secrets); they never block publication by themselves. + +use std::collections::HashMap; +use std::fmt; + +use serde::{Deserialize, Serialize}; + +use crate::podcast::FeedUrl; + +/// Query-parameter name words that usually carry a credential. A name is +/// split into lowercase words at non-alphanumeric characters and camelCase +/// boundaries; any listed word triggers a warning. +pub const SECRET_QUERY_WORDS: &[&str] = &[ + "token", + "key", + "auth", + "secret", + "sig", + "signature", + "uid", + "user", + "pass", + "password", + "passwd", + "pwd", + "session", + "sess", + "sid", + "access", + "private", + "credential", + "credentials", + "apikey", + "jwt", + "hmac", +]; + +/// Substrings that trigger a warning anywhere in a query-parameter name, +/// for names such as `apitoken` or `privatefeed` that do not split. +const SECRET_QUERY_SUBSTRINGS: &[&str] = &[ + "token", + "secret", + "signature", + "password", + "passwd", + "session", + "private", + "credential", + "apikey", +]; + +/// Hosts (and their subdomains) known to serve personal member feeds. +pub const PRIVATE_FEED_HOSTS: &[&str] = &[ + "patreon.com", + "supercast.com", + "supportingcast.fm", + "glow.fm", +]; + +/// Minimum length of a query value considered a possible secret. +const MIN_SECRET_VALUE_CHARS: usize = 16; +/// Minimum length of a path segment considered a possible secret. +const MIN_SECRET_SEGMENT_CHARS: usize = 32; +/// Minimum Shannon entropy, in bits per character, of a possible secret. +const MIN_SECRET_ENTROPY_BITS: f64 = 3.0; + +/// Why a feed URL may contain a personal secret. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "reason", rename_all = "snake_case")] +#[non_exhaustive] +pub enum FeedUrlPrivacyWarning { + /// A query parameter whose name suggests a credential. + SecretQueryParameter { + /// Parameter name as it appears in the URL. + name: String, + }, + /// The host is known to serve personal member feeds. + PrivateFeedHost { + /// Lowercase host. + host: String, + }, + /// A long, random-looking query value. + HighEntropyQueryValue { + /// Name of the parameter carrying the value. + name: String, + }, + /// A long, random-looking path segment. + HighEntropyPathSegment { + /// Zero-based index of the segment after the host. + index: usize, + }, +} + +impl fmt::Display for FeedUrlPrivacyWarning { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SecretQueryParameter { name } => write!( + f, + "the feed URL has a query parameter \"{name}\" that looks like a credential" + ), + Self::PrivateFeedHost { host } => { + write!(f, "{host} usually serves personal member feeds") + } + Self::HighEntropyQueryValue { name } => write!( + f, + "the value of the query parameter \"{name}\" looks like a secret token" + ), + Self::HighEntropyPathSegment { index } => write!( + f, + "path segment {} of the feed URL looks like a secret token", + index + 1 + ), + } + } +} + +/// Returns why `url` may contain a personal secret, or `None`. +/// +/// Advisory only (see the module documentation). Checks, in order: query +/// parameter names ([`SECRET_QUERY_WORDS`]), known member-feed hosts +/// ([`PRIVATE_FEED_HOSTS`]), long high-entropy query values, and long +/// high-entropy path segments. +pub fn feed_url_privacy_warning(url: &FeedUrl) -> Option { + let value = url.as_str(); + let rest = value.split_once("://").map_or(value, |(_, rest)| rest); + let (authority, path_and_query) = match rest.find('/') { + Some(index) => rest.split_at(index), + None => (rest, "/"), + }; + let host = if authority.starts_with('[') { + authority + .find(']') + .map_or(authority, |end| &authority[..=end]) + } else { + authority.split(':').next().unwrap_or(authority) + }; + let (path, query) = path_and_query + .split_once('?') + .unwrap_or((path_and_query, "")); + let parameters: Vec<(&str, &str)> = query + .split('&') + .filter(|pair| !pair.is_empty()) + .map(|pair| pair.split_once('=').unwrap_or((pair, ""))) + .collect(); + + if let Some((name, _)) = parameters + .iter() + .find(|(name, _)| is_secret_parameter_name(name)) + { + return Some(FeedUrlPrivacyWarning::SecretQueryParameter { + name: (*name).to_string(), + }); + } + if PRIVATE_FEED_HOSTS + .iter() + .any(|known| host == *known || host.ends_with(&format!(".{known}"))) + { + return Some(FeedUrlPrivacyWarning::PrivateFeedHost { + host: host.to_string(), + }); + } + if let Some((name, _)) = parameters + .iter() + .find(|(_, value)| looks_random(value, MIN_SECRET_VALUE_CHARS)) + { + return Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { + name: (*name).to_string(), + }); + } + path.split('/') + .filter(|segment| !segment.is_empty()) + .position(|segment| { + let stem = segment.rsplit_once('.').map_or(segment, |(stem, _)| stem); + looks_random(stem, MIN_SECRET_SEGMENT_CHARS) + }) + .map(|index| FeedUrlPrivacyWarning::HighEntropyPathSegment { index }) +} + +fn is_secret_parameter_name(name: &str) -> bool { + let words = split_words(name); + if words + .iter() + .any(|word| SECRET_QUERY_WORDS.contains(&word.as_str())) + { + return true; + } + let compact: String = name + .chars() + .filter(char::is_ascii_alphanumeric) + .collect::() + .to_ascii_lowercase(); + SECRET_QUERY_SUBSTRINGS + .iter() + .any(|needle| compact.contains(needle)) +} + +/// Splits at non-alphanumeric characters and lower-to-upper case changes. +fn split_words(name: &str) -> Vec { + let mut words = Vec::new(); + let mut current = String::new(); + let mut previous_lower = false; + for c in name.chars() { + if !c.is_ascii_alphanumeric() { + if !current.is_empty() { + words.push(std::mem::take(&mut current)); + } + previous_lower = false; + continue; + } + if c.is_ascii_uppercase() && previous_lower && !current.is_empty() { + words.push(std::mem::take(&mut current)); + } + previous_lower = c.is_ascii_lowercase() || c.is_ascii_digit(); + current.push(c.to_ascii_lowercase()); + } + if !current.is_empty() { + words.push(current); + } + words +} + +/// A long token-alphabet string mixing letters and digits (or both letter +/// cases) with high per-character entropy. +fn looks_random(value: &str, min_chars: usize) -> bool { + if value.chars().count() < min_chars + || !value + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.~%=+".contains(c)) + { + return false; + } + let has_digit = value.chars().any(|c| c.is_ascii_digit()); + let has_lower = value.chars().any(|c| c.is_ascii_lowercase()); + let has_upper = value.chars().any(|c| c.is_ascii_uppercase()); + let classes = [has_digit, has_lower, has_upper] + .into_iter() + .filter(|present| *present) + .count(); + classes >= 2 && shannon_entropy(value) >= MIN_SECRET_ENTROPY_BITS +} + +fn shannon_entropy(value: &str) -> f64 { + let mut counts: HashMap = HashMap::new(); + for c in value.chars() { + *counts.entry(c).or_default() += 1; + } + let total = value.chars().count() as f64; + counts + .values() + .map(|count| { + let p = *count as f64 / total; + -p * p.log2() + }) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn warn(url: &str) -> Option { + feed_url_privacy_warning(&FeedUrl::parse(url).unwrap()) + } + + #[test] + fn public_feeds_have_no_warning() { + for url in [ + "https://feeds.example.com/podcast.xml", + "https://example.com/feed?format=rss&page=2", + "https://anchor.example/s/abc123de/podcast/rss", + "https://feeds.example.com/12345/rss", + "https://example.com/feed?author=jane", + "https://example.com:8443/shows/my-great-show/feed.xml", + ] { + assert_eq!(warn(url), None, "{url}"); + } + } + + #[test] + fn secret_query_names_warn() { + for (url, name) in [ + ("https://example.com/feed?token=abc", "token"), + ("https://example.com/feed?auth_token=abc", "auth_token"), + ( + "https://example.com/feed?X-Amz-Signature=abc", + "X-Amz-Signature", + ), + ("https://example.com/feed?accessKey=abc", "accessKey"), + ("https://example.com/feed?a=1&uid=42", "uid"), + ("https://example.com/feed?apitoken=x", "apitoken"), + ("https://example.com/feed?PRIVATE=1", "PRIVATE"), + ("https://example.com/feed?session_id=1", "session_id"), + ] { + assert_eq!( + warn(url), + Some(FeedUrlPrivacyWarning::SecretQueryParameter { name: name.into() }), + "{url}" + ); + } + } + + #[test] + fn known_member_hosts_warn() { + assert_eq!( + warn("https://www.patreon.com/rss/show"), + Some(FeedUrlPrivacyWarning::PrivateFeedHost { + host: "www.patreon.com".into() + }) + ); + assert_eq!(warn("https://notpatreon.com/rss"), None); + } + + #[test] + fn high_entropy_values_and_segments_warn() { + assert_eq!( + warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb"), + Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { name: "c".into() }) + ); + assert_eq!(warn("https://example.com/feed?c=aaaaaaaaaaaaaaaa1"), None); + assert_eq!( + warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB4sYc1HdWe9GuJa3F.rss"), + Some(FeedUrlPrivacyWarning::HighEntropyPathSegment { index: 1 }) + ); + // Below the segment length: an ordinary short id. + assert_eq!(warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB.rss"), None); + } + + #[test] + fn warnings_render_without_the_secret() { + let warning = warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb").unwrap(); + assert!(!warning.to_string().contains("Zx81")); + } +} diff --git a/crates/music-dht/src/podcast_dht/search.rs b/crates/music-dht/src/podcast_dht/search.rs new file mode 100644 index 0000000..8b9e3b9 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/search.rs @@ -0,0 +1,238 @@ +//! Podcast search hits and their merge with music search results. + +use std::cmp::Reverse; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; + +use federation_net::EndpointId; +use serde::{Deserialize, Serialize}; + +use crate::normalization::{normalize_name, tokenize}; +use crate::podcast::PodcastId; +use crate::record::{ItemKind, LibraryItem}; + +use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records}; + +/// Maximum distinct query tokens used by a podcast text search. +pub const MAX_PODCAST_QUERY_TOKENS: usize = 8; +/// Maximum publishers reported per hit. +pub const MAX_HIT_PUBLISHERS: usize = 64; + +/// Kind of a federated search hit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SearchHitKind { + /// A musical artist. + Artist, + /// A release. + Release, + /// A track. + Track, + /// A public podcast card. + Podcast, +} + +impl SearchHitKind { + /// Stable lowercase name. + pub fn as_str(&self) -> &'static str { + match self { + Self::Artist => "artist", + Self::Release => "release", + Self::Track => "track", + Self::Podcast => "podcast", + } + } +} + +impl fmt::Display for SearchHitKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From for SearchHitKind { + fn from(kind: ItemKind) -> Self { + match kind { + ItemKind::Artist => Self::Artist, + ItemKind::Release => Self::Release, + ItemKind::Track => Self::Track, + } + } +} + +/// One public podcast found in the discovery overlay. +/// +/// Cards of the same podcast published by several devices are merged by +/// `podcast_id`. The hit carries no personal state of any publisher. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastSearchHit { + /// Podcast. + pub podcast_id: PodcastId, + /// Representative card: the most common `feed_url` among the + /// publishers' current cards, then the most recently issued card, then + /// the greatest publisher key. + pub card: PodcastCard, + /// Sorted distinct publisher devices with a current valid card, at most + /// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers: + /// one user may publish from several devices. + pub publishers: Vec, +} + +impl PodcastSearchHit { + /// Always [`SearchHitKind::Podcast`]. + pub fn kind(&self) -> SearchHitKind { + SearchHitKind::Podcast + } +} + +/// A music or podcast result of a federated search. +/// +/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use +/// [`FederatedSearchHit::kind`] for the fine-grained kind. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", content = "hit", rename_all = "snake_case")] +pub enum FederatedSearchHit { + /// A music-DHT record (artist, release or track). + Music(LibraryItem), + /// A public podcast. + Podcast(PodcastSearchHit), +} + +impl FederatedSearchHit { + /// Kind of the hit: `artist`, `release`, `track` or `podcast`. + pub fn kind(&self) -> SearchHitKind { + match self { + Self::Music(item) => item.kind.into(), + Self::Podcast(_) => SearchHitKind::Podcast, + } + } + + /// Display name: the item name or the podcast title. + pub fn title(&self) -> &str { + match self { + Self::Music(item) => &item.name, + Self::Podcast(hit) => &hit.card.title, + } + } +} + +/// Merges music results (as returned by `MusicDhtService::search_network`) +/// with podcast hits for the same query. +/// +/// Results whose normalized name equals the normalized query come first; +/// within each tier music keeps its order and precedes podcasts, which keep +/// theirs. Existing music consumers that only read `SearchOutcome` are +/// unaffected. +pub fn merge_search_results( + query: &str, + music: Vec, + podcasts: Vec, +) -> Vec { + let normalized = normalize_name(query); + let mut exact = Vec::new(); + let mut rest = Vec::new(); + let hits = music + .into_iter() + .map(FederatedSearchHit::Music) + .chain(podcasts.into_iter().map(FederatedSearchHit::Podcast)); + for hit in hits { + let is_exact = !normalized.is_empty() && normalize_name(hit.title()) == normalized; + if is_exact { + exact.push(hit); + } else { + rest.push(hit); + } + } + exact.extend(rest); + exact +} + +/// Distinct normalized query tokens, at most [`MAX_PODCAST_QUERY_TOKENS`]. +pub fn podcast_query_tokens(query: &str) -> Vec { + let mut seen = HashSet::new(); + tokenize(&normalize_name(query)) + .into_iter() + .filter(|token| token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone())) + .take(MAX_PODCAST_QUERY_TOKENS) + .collect() +} + +/// Keeps the winning record per `(podcast_id, publisher)` and groups the +/// cards that are still active by podcast. Records must be verified. +pub(crate) fn aggregate_hits( + records: impl IntoIterator, +) -> Vec { + let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new(); + for record in records { + let owner = (record.podcast_id().clone(), record.publisher()); + match winners.get(&owner) { + Some(current) if compare_records(current, &record).is_ge() => {} + _ => { + winners.insert(owner, record); + } + } + } + let mut by_podcast: BTreeMap> = BTreeMap::new(); + for record in winners.into_values() { + if let Some(card) = record.card() { + by_podcast + .entry(card.podcast_id.clone()) + .or_default() + .push(card.clone()); + } + } + by_podcast + .into_iter() + .map(|(podcast_id, cards)| { + let mut feed_votes: HashMap<&str, usize> = HashMap::new(); + for card in &cards { + *feed_votes.entry(card.feed_url.as_str()).or_default() += 1; + } + let card = cards + .iter() + .max_by_key(|card| { + ( + feed_votes[card.feed_url.as_str()], + card.issued_at_ms, + *card.publisher.as_bytes(), + ) + }) + .cloned() + .expect("a group has at least one card"); + let mut publishers: Vec = cards.iter().map(|card| card.publisher).collect(); + publishers.sort_by_key(|publisher| *publisher.as_bytes()); + publishers.dedup(); + publishers.truncate(MAX_HIT_PUBLISHERS); + PodcastSearchHit { + podcast_id, + card, + publishers, + } + }) + .collect() +} + +/// Keeps hits whose card is indexed under every query token and ranks +/// them: exact normalized title first, then more publishers, then title and +/// id. +pub(crate) fn rank_hits( + mut hits: Vec, + query: &str, + tokens: &[String], +) -> Vec { + let normalized_query = normalize_name(query); + hits.retain(|hit| { + let indexed = hit.card.index_tokens(); + tokens.iter().all(|token| indexed.contains(token)) + }); + hits.sort_by_cached_key(|hit| { + let title = normalize_name(&hit.card.title); + ( + title != normalized_query, + Reverse(hit.publishers.len()), + title, + hit.podcast_id.clone(), + ) + }); + hits +} diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs new file mode 100644 index 0000000..9f27104 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -0,0 +1,522 @@ +//! SQLite cache of the podcast-discovery overlay: replicas, routing contacts +//! and this device's own publications. +//! +//! The cache is replaceable federation state, kept in its own database file +//! (like the similarity-routing cache). Every record read from it is decoded +//! and verified again before it is used or served. + +use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use federation_net::{EndpointId, NetworkId}; +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::error::{MusicDhtError, Result}; +use crate::podcast::PodcastId; +use crate::routing::{NodeContact, NodeId}; + +use super::card::{ + PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere, +}; + +/// `PRAGMA user_version` of the podcast-discovery cache schema. +pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1; +/// Maximum distinct `(podcast_id, publisher)` records cached under one key. +/// When full, the record expiring first is evicted if the new one outlives it. +pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; + +const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS podcast_dht_records ( + dht_key BLOB NOT NULL, + podcast_id TEXT NOT NULL, + publisher TEXT NOT NULL, + tombstone INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + payload BLOB NOT NULL, + PRIMARY KEY (dht_key, podcast_id, publisher) +); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry + ON podcast_dht_records(expires_at_ms); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner + ON podcast_dht_records(podcast_id, publisher); + +CREATE TABLE IF NOT EXISTS podcast_dht_peers ( + peer_id TEXT PRIMARY KEY, + node_id BLOB NOT NULL, + ticket TEXT NOT NULL, + last_seen_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS podcast_dht_local ( + podcast_id TEXT PRIMARY KEY, + revision INTEGER NOT NULL, + spec BLOB, + record BLOB NOT NULL, + updated_at_ms INTEGER NOT NULL +); +"; + +/// This device's latest publication of one podcast. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LocalPublication { + pub podcast_id: PodcastId, + /// Greatest revision ever used for this podcast by this device. + pub revision: u64, + /// Published card content; `None` once unpublished. + pub spec: Option, + /// Latest signed record (card or tombstone). + pub record: SignedPodcastRecord, +} + +#[derive(Clone)] +pub(crate) struct PodcastDhtDatabase { + conn: Arc>, + network_id: NetworkId, +} + +impl std::fmt::Debug for PodcastDhtDatabase { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PodcastDhtDatabase") + .finish_non_exhaustive() + } +} + +impl PodcastDhtDatabase { + pub async fn open(path: &Path, network_id: NetworkId) -> Result { + let path = path.to_path_buf(); + let conn = tokio::task::spawn_blocking(move || -> Result { + let conn = Connection::open(&path).map_err(|error| { + MusicDhtError::Database(format!("failed to open {}: {error}", path.display())) + })?; + let version: i32 = conn + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(database_error)?; + if version > PODCAST_DHT_CACHE_SCHEMA_VERSION { + return Err(MusicDhtError::Database(format!( + "podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}" + ))); + } + conn.execute_batch(SCHEMA).map_err(|error| { + MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}")) + })?; + conn.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION) + .map_err(database_error)?; + Ok(conn) + }) + .await + .map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))??; + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + network_id, + }) + } + + async fn call(&self, operation: F) -> Result + where + F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result + Send + 'static, + T: Send + 'static, + { + let conn = Arc::clone(&self.conn); + let network_id = self.network_id; + tokio::task::spawn_blocking(move || { + let guard = lock(&conn); + operation(&guard, &network_id).map_err(database_error) + }) + .await + .map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))? + } + + /// Stores records under the given keys. The caller has verified every + /// record and checked that each key belongs to it. Returns the number of + /// `(key, record)` rows written. + pub async fn store( + &self, + entries: Vec<(Vec, SignedPodcastRecord)>, + now_ms: u64, + ) -> Result { + self.call(move |conn, network_id| { + let transaction = conn.unchecked_transaction()?; + let mut written = 0; + for (keys, record) in entries { + written += store_one(&transaction, network_id, &keys, &record, now_ms)?; + } + transaction.commit()?; + Ok(written) + }) + .await + } + + /// Valid, unexpired records under `key`, newest expiry first. Cards must + /// contain every token in `filter_tokens`; tombstones are always + /// returned so stale replicas elsewhere can be overruled. + pub async fn records_by_key( + &self, + key: PodcastDhtKey, + filter_tokens: Vec, + now_ms: u64, + limit: usize, + ) -> Result> { + self.call(move |conn, network_id| { + let mut statement = conn.prepare( + "SELECT payload FROM podcast_dht_records + WHERE dht_key = ?1 AND expires_at_ms > ?2 + ORDER BY expires_at_ms DESC, podcast_id, publisher", + )?; + let rows = statement.query_map( + params![key.as_bytes().as_slice(), clamp_i64(now_ms)], + |row| row.get::<_, Vec>(0), + )?; + let mut records = Vec::new(); + for row in rows { + let Some(record) = decode_valid(&row?, network_id, now_ms) else { + continue; + }; + if !record.belongs_under(&key) { + continue; + } + let matches = record.card().is_none_or(|card| { + let tokens = card.index_tokens(); + filter_tokens.iter().all(|token| tokens.contains(token)) + }); + if matches { + records.push(record); + if records.len() == limit { + break; + } + } + } + Ok(records) + }) + .await + } + + pub async fn delete_expired(&self, now_ms: u64) -> Result { + self.call(move |conn, _| { + conn.execute( + "DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1", + [clamp_i64(now_ms)], + ) + }) + .await + } + + pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> { + self.call(move |conn, _| { + conn.execute( + "INSERT OR REPLACE INTO podcast_dht_peers + (peer_id, node_id, ticket, last_seen_ms) + VALUES (?1, ?2, ?3, ?4)", + params![ + contact.peer_id.to_string(), + contact.node_id.as_bytes().as_slice(), + contact.ticket, + clamp_i64(contact.last_seen_ms), + ], + )?; + Ok(()) + }) + .await + } + + pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> { + self.call(move |conn, _| { + conn.execute( + "DELETE FROM podcast_dht_peers WHERE peer_id = ?1", + [peer.to_string()], + )?; + Ok(()) + }) + .await + } + + pub async fn load_peers(&self) -> Result> { + self.call(move |conn, _| { + let mut statement = + conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + let mut peers = Vec::new(); + for row in rows { + let (peer, ticket, last_seen_ms) = row?; + let Ok(peer_id) = EndpointId::from_str(&peer) else { + continue; + }; + peers.push(NodeContact { + node_id: NodeId::from_endpoint(&peer_id), + peer_id, + ticket, + last_seen_ms: last_seen_ms.max(0) as u64, + }); + } + Ok(peers) + }) + .await + } + + pub async fn local_publication( + &self, + podcast_id: PodcastId, + ) -> Result> { + self.call(move |conn, _| { + conn.query_row( + "SELECT podcast_id, revision, spec, record FROM podcast_dht_local + WHERE podcast_id = ?1", + [podcast_id.as_str()], + local_from_row, + ) + .optional() + .map(Option::flatten) + }) + .await + } + + pub async fn local_publications(&self) -> Result> { + self.call(move |conn, _| { + let mut statement = conn.prepare( + "SELECT podcast_id, revision, spec, record FROM podcast_dht_local + ORDER BY podcast_id", + )?; + let rows = statement.query_map([], local_from_row)?; + let mut out = Vec::new(); + for row in rows { + if let Some(publication) = row? { + out.push(publication); + } + } + Ok(out) + }) + .await + } + + pub async fn put_local_publication( + &self, + publication: LocalPublication, + now_ms: u64, + ) -> Result<()> { + self.call(move |conn, _| { + let spec = publication + .spec + .as_ref() + .map(postcard::to_stdvec) + .transpose() + .map_err(to_sql_error)?; + let record = postcard::to_stdvec(&publication.record).map_err(to_sql_error)?; + conn.execute( + "INSERT OR REPLACE INTO podcast_dht_local + (podcast_id, revision, spec, record, updated_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + publication.podcast_id.as_str(), + publication.revision as i64, + spec, + record, + clamp_i64(now_ms), + ], + )?; + Ok(()) + }) + .await + } +} + +fn store_one( + conn: &Connection, + network_id: &NetworkId, + keys: &[PodcastDhtKey], + record: &SignedPodcastRecord, + now_ms: u64, +) -> rusqlite::Result { + let podcast_id = record.podcast_id().as_str().to_string(); + let publisher = record.publisher().to_string(); + + // Every cached version of this (podcast, publisher), re-verified. + let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new(); + let mut invalid_keys = Vec::new(); + { + let mut statement = conn.prepare( + "SELECT dht_key, payload FROM podcast_dht_records + WHERE podcast_id = ?1 AND publisher = ?2", + )?; + let rows = statement.query_map(params![podcast_id, publisher], |row| { + Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)) + })?; + for row in rows { + let (key, payload) = row?; + let decoded = decode_valid(&payload, network_id, now_ms); + match (<[u8; 32]>::try_from(key.as_slice()), decoded) { + (Ok(key_bytes), Some(cached)) => { + existing.insert(key_bytes, cached); + } + _ => invalid_keys.push(key), + } + } + } + for key in invalid_keys { + conn.execute( + "DELETE FROM podcast_dht_records + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key, podcast_id, publisher], + )?; + } + + // A tombstone or a newer revision cached under any key wins everywhere. + if existing + .values() + .any(|cached| supersedes_everywhere(cached, record)) + { + return Ok(0); + } + + let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; + let mut written = 0; + for key in keys { + if let Some(cached) = existing.get(key.as_bytes()) + && compare_records(cached, record).is_ge() + { + continue; + } + if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? { + continue; + } + conn.execute( + "INSERT INTO podcast_dht_records + (dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET + tombstone = excluded.tombstone, + expires_at_ms = excluded.expires_at_ms, + payload = excluded.payload", + params![ + key.as_bytes().as_slice(), + podcast_id, + publisher, + record.is_tombstone(), + clamp_i64(record.expires_at_ms()), + payload, + ], + )?; + written += 1; + } + + // Drop versions the new record makes obsolete under other keys too. + if written > 0 { + for (key, cached) in &existing { + if !keys.iter().any(|stored| stored.as_bytes() == key) + && supersedes_everywhere(record, cached) + { + conn.execute( + "DELETE FROM podcast_dht_records + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_slice(), podcast_id, publisher], + )?; + } + } + } + Ok(written) +} + +/// Ensures there is room for one more record under `key`. +fn make_room( + conn: &Connection, + key: &PodcastDhtKey, + record: &SignedPodcastRecord, + now_ms: u64, +) -> rusqlite::Result { + conn.execute( + "DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2", + params![key.as_bytes().as_slice(), clamp_i64(now_ms)], + )?; + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1", + [key.as_bytes().as_slice()], + |row| row.get(0), + )?; + if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY { + return Ok(true); + } + let victim: Option<(String, String, i64)> = conn + .query_row( + "SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records + WHERE dht_key = ?1 + ORDER BY expires_at_ms ASC, podcast_id, publisher + LIMIT 1", + [key.as_bytes().as_slice()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + match victim { + Some((podcast_id, publisher, expires_at_ms)) + if expires_at_ms < clamp_i64(record.expires_at_ms()) => + { + conn.execute( + "DELETE FROM podcast_dht_records + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_bytes().as_slice(), podcast_id, publisher], + )?; + Ok(true) + } + _ => Ok(false), + } +} + +fn decode_valid( + payload: &[u8], + network_id: &NetworkId, + now_ms: u64, +) -> Option { + let record: SignedPodcastRecord = postcard::from_bytes(payload).ok()?; + record.verify_at(network_id, now_ms).ok()?; + Some(record) +} + +fn local_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result> { + let podcast_id: String = row.get(0)?; + let revision: i64 = row.get(1)?; + let spec: Option> = row.get(2)?; + let record: Vec = row.get(3)?; + let Ok(podcast_id) = PodcastId::parse(&podcast_id) else { + return Ok(None); + }; + let spec = match spec { + Some(bytes) => match postcard::from_bytes::(&bytes) { + Ok(spec) => Some(spec), + Err(_) => return Ok(None), + }, + None => None, + }; + let Ok(record) = postcard::from_bytes::(&record) else { + return Ok(None); + }; + Ok(Some(LocalPublication { + podcast_id, + revision: revision as u64, + spec, + record, + })) +} + +fn clamp_i64(value: u64) -> i64 { + value.min(i64::MAX as u64) as i64 +} + +fn to_sql_error(error: impl std::error::Error + Send + Sync + 'static) -> rusqlite::Error { + rusqlite::Error::ToSqlConversionFailure(Box::new(error)) +} + +fn database_error(error: rusqlite::Error) -> MusicDhtError { + MusicDhtError::Database(error.to_string()) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs new file mode 100644 index 0000000..faf6159 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -0,0 +1,845 @@ +//! Unit tests of the podcast-discovery overlay: records, ordering, cache and +//! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`. + +use federation_net::{NetworkId, SecretKey}; + +use super::card::supersedes_everywhere; +use super::search::{aggregate_hits, rank_hits}; +use super::*; +use crate::podcast::{FeedUrl, PodcastMeta}; +use crate::record::{ItemId, ItemKind, LibraryItem}; + +const NOW: u64 = 1_750_000_000_000; + +fn network() -> NetworkId { + NetworkId::from_name("podcast-dht-unit") +} + +fn secret(seed: u8) -> SecretKey { + SecretKey::from_bytes(&[seed; 32]) +} + +fn feed(path: &str) -> FeedUrl { + FeedUrl::parse(&format!("https://feeds.example.test/{path}")).unwrap() +} + +fn spec(path: &str, title: &str) -> PodcastCardSpec { + let feed_url = feed(path); + PodcastCardSpec { + podcast_id: PodcastId::from_feed_url(&feed_url), + feed_url, + id_feed_url: None, + title: title.to_string(), + author: Some("Jane Doe".to_string()), + description: Some("A show about\nthings.".to_string()), + image_url: Some("https://cdn.example.test/art.jpg".to_string()), + link: Some("https://example.test/show".to_string()), + language: Some("en".to_string()), + categories: vec!["Technology".to_string()], + episode_count: Some(12), + latest_episode_at_ms: Some(1_700_000_000_000), + } +} + +fn card_with( + spec: PodcastCardSpec, + key: &SecretKey, + revision: u64, + issued: u64, +) -> SignedPodcastRecord { + let card = PodcastCard::from_spec(spec, network(), key.public(), revision, issued); + SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)).unwrap() +} + +fn card(key: &SecretKey, revision: u64) -> SignedPodcastRecord { + card_with(spec("show.xml", "Rust Radio"), key, revision, NOW) +} + +fn tombstone(key: &SecretKey, revision: u64, issued: u64) -> SignedPodcastRecord { + let retracted = card(key, 1); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + retracted.podcast_id().clone(), + revision, + issued, + retracted.dht_keys(), + ); + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap() +} + +fn resign(record: &mut SignedPodcastRecord, key: &SecretKey) { + record.signature = key.sign(&record.signing_bytes()); +} + +fn subscription(spec: &PodcastCardSpec) -> SubscriptionRecord { + SubscriptionRecord { + podcast_id: spec.podcast_id.clone(), + subscribed: true, + podcast: PodcastMeta { + feed_url: spec.feed_url.clone(), + title: spec.title.clone(), + author: spec.author.clone(), + description: spec.description.clone(), + link: spec.link.clone(), + image_url: spec.image_url.clone(), + }, + feed_aliases: Vec::new(), + generation: 1, + hlc_ms: 100, + op_id: "private-op-id-123".to_string(), + } +} + +// ---------------------------------------------------------------- signatures + +#[test] +fn signed_card_verifies_and_rejects_tampering() { + let key = secret(1); + let record = card(&key, 1); + record.verify_at(&network(), NOW).unwrap(); + + let mut tampered = record.clone(); + if let PodcastRecordBody::Card(card) = &mut tampered.body { + card.title = "Evil Radio".to_string(); + } + assert!(tampered.verify(&network()).is_err()); + + let mut moved = record.clone(); + if let PodcastRecordBody::Card(card) = &mut moved.body { + card.revision += 1; + } + assert!(moved.verify(&network()).is_err()); + + assert!(record.verify(&NetworkId::from_name("other")).is_err()); +} + +#[test] +fn record_signed_by_another_key_is_rejected() { + let publisher = secret(1); + let forger = secret(2); + let card = PodcastCard::from_spec( + spec("show.xml", "Rust Radio"), + network(), + publisher.public(), + 1, + NOW, + ); + let forged = SignedPodcastRecord::sign_card(card, |bytes| forger.sign(bytes)).unwrap(); + assert!(forged.verify(&network()).is_err()); + + let mut forged_tombstone = tombstone(&forger, 2, NOW); + if let PodcastRecordBody::Tombstone(tombstone) = &mut forged_tombstone.body { + tombstone.publisher = publisher.public(); + } + resign(&mut forged_tombstone, &forger); + assert!(forged_tombstone.verify(&network()).is_err()); +} + +#[test] +fn canonical_encoding_is_stable() { + let key = secret(9); + let record = card_with(spec("vector.xml", "Vector Show"), &key, 7, NOW); + let bytes = record.signing_bytes(); + assert!(bytes.starts_with(b"furumi-podcast-card:v1\0\x01\x00")); + assert_eq!( + blake3::hash(&bytes).to_hex().as_str(), + "cbbc512b077ad05baf01728fb5d179a1d89301f19ea15c66058df284a716e9b5", + "card signing input changed: this breaks every published card" + ); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + record.podcast_id().clone(), + 8, + NOW, + record.dht_keys(), + ); + let bytes = tombstone.signing_bytes(); + assert!(bytes.starts_with(b"furumi-podcast-card-tombstone:v1\0\x01\x00")); + assert_eq!( + blake3::hash(&bytes).to_hex().as_str(), + "032ccef911bff5f2f37367137d428d60c111efb65529c8d75754434434dbd083", + "tombstone signing input changed" + ); + assert_eq!( + hex(PodcastDhtKey::token(&network(), "vector").as_bytes()), + "4367d03f8ea2a541ebec43b73d767b42ad733e3f26f90372cff9f92cf19ccbf1" + ); +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[test] +fn optional_fields_are_unambiguous_in_the_signing_input() { + let key = secret(1); + let mut a = spec("show.xml", "Rust Radio"); + a.author = Some(String::new()); + let mut b = spec("show.xml", "Rust Radio"); + b.author = None; + let a = PodcastCard::from_spec(a, network(), key.public(), 1, NOW); + let b = PodcastCard::from_spec(b, network(), key.public(), 1, NOW); + assert_ne!(a.signing_bytes(), b.signing_bytes()); +} + +// -------------------------------------------------------------------- bounds + +#[test] +fn card_text_bounds_accept_the_limit_and_reject_one_more() { + let key = secret(1); + let sign = |spec: PodcastCardSpec| { + let card = PodcastCard::from_spec(spec, network(), key.public(), 1, NOW); + SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)) + }; + let mut at = spec("show.xml", &"я".repeat(MAX_CARD_TITLE_CHARS)); + at.author = Some("a".repeat(MAX_CARD_AUTHOR_CHARS)); + at.description = Some("d".repeat(MAX_CARD_DESCRIPTION_CHARS)); + at.categories = (0..MAX_CARD_CATEGORIES) + .map(|index| format!("{index}{}", "c".repeat(MAX_CARD_CATEGORY_CHARS - 1))) + .collect(); + at.language = Some("x".repeat(MAX_CARD_LANGUAGE_BYTES)); + let record = sign(at.clone()).unwrap(); + assert!(record.encoded_len() <= MAX_PODCAST_RECORD_BYTES); + + let mut over = at.clone(); + over.title.push('я'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.author.as_mut().unwrap().push('a'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.description.as_mut().unwrap().push('d'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.categories.push("extra".into()); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.categories[0].push('c'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.language.as_mut().unwrap().push('x'); + assert!(sign(over).is_err()); + let mut over = at; + over.title = " ".into(); + assert!(sign(over).is_err()); +} + +#[test] +fn card_rejects_bad_urls_ids_and_control_characters() { + let key = secret(1); + let sign = |spec: PodcastCardSpec| { + let card = PodcastCard::from_spec(spec, network(), key.public(), 1, NOW); + SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)) + }; + let mut bad = spec("show.xml", "Rust Radio"); + bad.image_url = Some("file:///etc/passwd".into()); + assert!(sign(bad).is_err()); + let mut bad = spec("show.xml", "Rust Radio"); + bad.podcast_id = PodcastId::from_feed_url(&feed("other.xml")); + assert!(sign(bad).is_err(), "id must hash from a URL in the card"); + let mut bad = spec("show.xml", "Rust Radio"); + bad.title = "Rust\u{7}Radio".into(); + assert!(sign(bad).is_err()); + let mut bad = spec("show.xml", "Rust Radio"); + bad.id_feed_url = Some(bad.feed_url.clone()); + assert!(sign(bad).is_err()); + + let mut moved = spec("new.xml", "Rust Radio"); + moved.id_feed_url = Some(feed("show.xml")); + moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml")); + sign(moved).unwrap(); +} + +#[test] +fn bounded_spec_fits_the_card() { + let mut raw = spec("show.xml", &format!(" {}\n", "t".repeat(2_000))); + raw.author = Some("\u{1}".into()); + raw.description = Some(format!("{}\u{0}", "d".repeat(5_000))); + raw.image_url = Some("not a url".into()); + raw.language = Some("en US".into()); + raw.categories = (0..20).map(|index| format!("Cat {}", index % 10)).collect(); + raw.categories.push("cat 1".into()); + raw.latest_episode_at_ms = Some(-5); + let bounded = raw.bounded(); + bounded.validate().unwrap(); + assert_eq!(bounded.title.chars().count(), MAX_CARD_TITLE_CHARS); + assert_eq!(bounded.author, None); + assert_eq!(bounded.image_url, None); + assert_eq!(bounded.language, None); + assert_eq!(bounded.categories.len(), MAX_CARD_CATEGORIES); + assert_eq!(bounded.latest_episode_at_ms, None); + assert_eq!(bounded.bounded(), bounded); +} + +#[test] +fn index_tokens_are_distinct_bounded_and_skip_long_tokens() { + let long = "x".repeat(MAX_INDEX_TOKEN_BYTES + 1); + let title = format!("The Rust the {long} radio"); + assert_eq!( + card_index_tokens(&title, Some("Rust Jane")), + vec!["the", "rust", "radio", "jane"] + ); + let many: Vec = (0..50).map(|index| format!("w{index}")).collect(); + let tokens = card_index_tokens(&many.join(" "), Some("author")); + assert_eq!(tokens.len(), MAX_CARD_INDEX_TOKENS); + assert!(!tokens.contains(&"author".to_string())); + let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW); + assert_eq!(record.dht_keys().len(), MAX_RECORD_DHT_KEYS); +} + +#[test] +fn oversized_tombstone_key_lists_are_rejected() { + let key = secret(1); + let id = PodcastId::from_feed_url(&feed("show.xml")); + let keys: Vec<_> = (0..=MAX_CARD_INDEX_TOKENS) + .map(|index| PodcastDhtKey::token(&network(), &format!("t{index}"))) + .collect(); + let mut tombstone = PodcastTombstone::new(network(), key.public(), id.clone(), 2, NOW, keys); + assert_eq!(tombstone.index_keys.len(), MAX_CARD_INDEX_TOKENS); + SignedPodcastRecord::sign_tombstone(tombstone.clone(), |bytes| key.sign(bytes)).unwrap(); + tombstone + .index_keys + .push(PodcastDhtKey::from_bytes([0xff; 32])); + assert!( + SignedPodcastRecord::sign_tombstone(tombstone.clone(), |bytes| key.sign(bytes)).is_err() + ); + tombstone.index_keys.pop(); + tombstone.index_keys.reverse(); + assert!( + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).is_err(), + "unsorted keys are not canonical" + ); +} + +// ------------------------------------------------------------------ lifetime + +#[test] +fn expiry_future_issue_and_overlong_lifetimes_are_rejected() { + let key = secret(1); + let record = card(&key, 1); + let ttl = PODCAST_CARD_TTL.as_millis() as u64; + record.verify_at(&network(), NOW + ttl - 1).unwrap(); + assert!(record.verify_at(&network(), NOW + ttl).is_err()); + + let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64; + assert!(record.verify_at(&network(), NOW - skew).is_ok()); + assert!(record.verify_at(&network(), NOW - skew - 1).is_err()); + + let mut overlong = record.clone(); + if let PodcastRecordBody::Card(card) = &mut overlong.body { + card.expires_at_ms += 1; + } + resign(&mut overlong, &key); + assert!(overlong.verify(&network()).is_err()); + + let grave = tombstone(&key, 2, NOW); + let tomb_ttl = PODCAST_TOMBSTONE_TTL.as_millis() as u64; + assert!(tomb_ttl > ttl); + grave.verify_at(&network(), NOW + tomb_ttl - 1).unwrap(); + assert!(grave.verify_at(&network(), NOW + tomb_ttl).is_err()); +} + +// ------------------------------------------------------------------ ordering + +#[test] +fn record_order_is_total_and_tombstones_win_ties() { + let key = secret(1); + let older = card(&key, 1); + let newer = card(&key, 2); + let refresh = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + 1); + let edited = card_with(spec("show.xml", "Rust Radio!"), &key, 1, NOW); + let grave = tombstone(&key, 1, NOW); + + assert!(compare_records(&newer, &older).is_gt()); + assert!(compare_records(&refresh, &older).is_gt()); + assert!(compare_records(&grave, &newer).is_lt()); + assert!(compare_records(&grave, &older).is_gt()); + assert!(compare_records(&grave, &refresh).is_gt()); + assert_ne!(compare_records(&edited, &older), std::cmp::Ordering::Equal); + assert_eq!( + compare_records(&edited, &older), + compare_records(&older, &edited).reverse() + ); + assert!(supersedes_everywhere(&grave, &older)); + assert!(supersedes_everywhere(&newer, &older)); + assert!(!supersedes_everywhere(&refresh, &older)); + assert!(!supersedes_everywhere(&older, &grave)); +} + +// --------------------------------------------------------------------- cache + +async fn open_db(temp: &tempfile::TempDir) -> store::PodcastDhtDatabase { + store::PodcastDhtDatabase::open(&temp.path().join("podcast-dht.sqlite3"), network()) + .await + .unwrap() +} + +fn entry(record: &SignedPodcastRecord) -> (Vec, SignedPodcastRecord) { + (record.dht_keys(), record.clone()) +} + +#[tokio::test] +async fn cache_replay_of_an_older_revision_is_ignored() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let newer = card(&key, 2); + let older = card(&key, 1); + let id_key = PodcastDhtKey::podcast(&network(), newer.podcast_id()); + assert!(db.store(vec![entry(&newer)], NOW).await.unwrap() > 0); + assert_eq!(db.store(vec![entry(&older)], NOW).await.unwrap(), 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![newer.clone()] + ); + // Storing the same record again changes nothing. + assert_eq!(db.store(vec![entry(&newer)], NOW).await.unwrap(), 0); +} + +#[tokio::test] +async fn cache_converges_regardless_of_delivery_order() { + let key = secret(1); + let records = [ + card(&key, 1), + card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + 5), + card_with(spec("show.xml", "Rust Radio Weekly"), &key, 2, NOW), + tombstone(&key, 2, NOW + 1), + card_with(spec("show.xml", "Rust Radio"), &key, 3, NOW + 2), + ]; + let id_key = PodcastDhtKey::podcast(&network(), records[0].podcast_id()); + let mut results = Vec::new(); + for order in [ + [0, 1, 2, 3, 4], + [4, 3, 2, 1, 0], + [3, 0, 4, 2, 1], + [2, 4, 0, 3, 1], + ] { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + for index in order { + db.store(vec![entry(&records[index])], NOW).await.unwrap(); + } + results.push( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + ); + } + assert!( + results + .iter() + .all(|result| *result == vec![records[4].clone()]) + ); +} + +#[tokio::test] +async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let active = card(&key, 1); + db.store(vec![entry(&active)], NOW).await.unwrap(); + let token_key = PodcastDhtKey::token(&network(), "rust"); + assert_eq!( + db.records_by_key(token_key, vec!["rust".into()], NOW, 10) + .await + .unwrap(), + vec![active.clone()] + ); + + // A tombstone stored only under the id key still removes the card under + // the text keys of this node. + let grave = tombstone(&key, 2, NOW); + let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id()); + db.store(vec![(vec![id_key], grave.clone())], NOW) + .await + .unwrap(); + assert_eq!( + db.records_by_key(token_key, vec!["rust".into()], NOW, 10) + .await + .unwrap(), + Vec::::new() + ); + // An equal-revision card arriving later loses, under any key. + let same = card(&key, 2); + assert_eq!(db.store(vec![entry(&same)], NOW).await.unwrap(), 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![grave.clone()] + ); + // The tombstone is retained (and served) until it expires. + let card_ttl = PODCAST_CARD_TTL.as_millis() as u64; + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW + card_ttl + 1, 10) + .await + .unwrap(), + vec![grave.clone()] + ); + // Republishing with a higher revision wins over the tombstone. + let again = card(&key, 3); + assert!(db.store(vec![entry(&again)], NOW).await.unwrap() > 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![again] + ); +} + +#[tokio::test] +async fn cache_drops_stale_text_keys_of_older_revisions() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let old = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW); + let renamed = card_with(spec("show.xml", "Ferris Hour"), &key, 2, NOW); + db.store(vec![entry(&old), entry(&renamed)], NOW) + .await + .unwrap(); + let rust = PodcastDhtKey::token(&network(), "rust"); + let ferris = PodcastDhtKey::token(&network(), "ferris"); + assert!( + db.records_by_key(rust, Vec::new(), NOW, 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + db.records_by_key(ferris, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![renamed] + ); +} + +#[tokio::test] +async fn cache_read_path_filters_tokens_expiry_and_corruption() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let record = card(&key, 1); + db.store(vec![entry(&record)], NOW).await.unwrap(); + let rust = PodcastDhtKey::token(&network(), "rust"); + assert_eq!( + db.records_by_key(rust, vec!["rust".into(), "radio".into()], NOW, 10) + .await + .unwrap() + .len(), + 1 + ); + assert!( + db.records_by_key(rust, vec!["rust".into(), "jazz".into()], NOW, 10) + .await + .unwrap() + .is_empty() + ); + let ttl = PODCAST_CARD_TTL.as_millis() as u64; + assert!( + db.records_by_key(rust, Vec::new(), NOW + ttl, 10) + .await + .unwrap() + .is_empty() + ); + + // A row whose payload no longer verifies is never served. + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let mut tampered = record.clone(); + if let PodcastRecordBody::Card(card) = &mut tampered.body { + card.title = "Rust Radio Evil".into(); + } + conn.execute( + "UPDATE podcast_dht_records SET payload = ?1", + [postcard::to_stdvec(&tampered).unwrap()], + ) + .unwrap(); + assert!( + db.records_by_key(rust, Vec::new(), NOW, 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + db.delete_expired(NOW + ttl).await.unwrap(), + MAX_RECORD_DHT_KEYS.min(record.dht_keys().len()) + ); +} + +#[tokio::test] +async fn cache_bounds_records_per_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let token_key = PodcastDhtKey::token(&network(), "rust"); + let mut entries = Vec::new(); + for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { + let key = SecretKey::from_bytes(&{ + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes()); + bytes + }); + // Later publishers expire later, so the first one is evicted. + let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64); + entries.push((vec![token_key], record)); + } + let first = entries[0].1.clone(); + for chunk in entries.chunks(64) { + db.store(chunk.to_vec(), NOW).await.unwrap(); + } + let cached = db + .records_by_key(token_key, Vec::new(), NOW, usize::MAX) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(!cached.contains(&first)); + // A record expiring before every cached one is not admitted. + assert_eq!( + db.store(vec![(vec![token_key], first)], NOW).await.unwrap(), + 0 + ); +} + +#[tokio::test] +async fn cache_refuses_a_newer_schema_and_keeps_local_publications() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("podcast-dht.sqlite3"); + let key = secret(1); + let record = card(&key, 5); + { + let db = store::PodcastDhtDatabase::open(&path, network()) + .await + .unwrap(); + db.put_local_publication( + store::LocalPublication { + podcast_id: record.podcast_id().clone(), + revision: 5, + spec: Some(record.card().unwrap().spec()), + record: record.clone(), + }, + NOW, + ) + .await + .unwrap(); + } + let db = store::PodcastDhtDatabase::open(&path, network()) + .await + .unwrap(); + let local = db.local_publications().await.unwrap(); + assert_eq!(local.len(), 1); + assert_eq!(local[0].revision, 5); + assert_eq!(local[0].record, record); + drop(db); + + rusqlite::Connection::open(&path) + .unwrap() + .pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1) + .unwrap(); + assert!( + store::PodcastDhtDatabase::open(&path, network()) + .await + .is_err() + ); +} + +// ------------------------------------------------------------------ batches + +#[test] +fn store_batches_are_bounded_and_keys_must_belong() { + let key = secret(1); + let record = card(&key, 1); + let good = PodcastStoreEntry { + keys: record.dht_keys(), + record: record.clone(), + }; + validate_store_batch(vec![good.clone()], &network(), NOW).unwrap(); + assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err()); + assert!( + validate_store_batch( + vec![good.clone(); MAX_PODCAST_STORE_RECORDS], + &network(), + NOW + ) + .is_ok() + ); + assert!( + validate_store_batch( + vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1], + &network(), + NOW + ) + .is_err() + ); + let mut foreign = good.clone(); + foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")]; + assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err()); + let mut duplicated = good.clone(); + duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]]; + assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err()); + let mut forged = good; + resign(&mut forged.record, &secret(2)); + assert!(validate_store_batch(vec![forged], &network(), NOW).is_err()); +} + +#[test] +fn filter_tokens_are_bounded() { + let at: Vec = (0..MAX_PODCAST_QUERY_TOKENS) + .map(|i| format!("t{i}")) + .collect(); + validate_filter_tokens(&at).unwrap(); + let mut over = at.clone(); + over.push("x".into()); + assert!(validate_filter_tokens(&over).is_err()); + assert!(validate_filter_tokens(&["x".repeat(MAX_INDEX_TOKEN_BYTES + 1)]).is_err()); + assert!(validate_filter_tokens(&[String::new()]).is_err()); +} + +// ------------------------------------------------------------------ privacy + +#[test] +fn card_from_subscription_is_a_public_projection_only() { + let base = spec("show.xml", "Rust Radio"); + let mut subscription = subscription(&base); + let spec = PodcastCardSpec::from_subscription(&subscription).unwrap(); + assert_eq!(spec.feed_url, base.feed_url); + assert_eq!(spec.id_feed_url, None); + let record = card_with(spec, &secret(1), 1, NOW); + let bytes = postcard::to_stdvec(&record).unwrap(); + let private = b"private-op-id-123"; + assert!(!bytes.windows(private.len()).any(|window| window == private)); + assert!( + !record + .signing_bytes() + .windows(private.len()) + .any(|window| window == private) + ); + + // After a feed move, the card binds the id to the original URL. + subscription.feed_aliases = vec![base.feed_url.clone()]; + subscription.podcast.feed_url = feed("moved.xml"); + let moved = PodcastCardSpec::from_subscription(&subscription).unwrap(); + assert_eq!(moved.feed_url, feed("moved.xml")); + assert_eq!(moved.id_feed_url, Some(base.feed_url.clone())); + moved.validate().unwrap(); + + // Without the original URL the id cannot be proven and nothing is built. + subscription.feed_aliases.clear(); + assert!(PodcastCardSpec::from_subscription(&subscription).is_err()); +} + +#[test] +fn privacy_warnings_cover_both_feed_urls() { + let mut card = spec("show.xml", "Rust Radio"); + assert!(card.privacy_warnings().is_empty()); + card.id_feed_url = + Some(FeedUrl::parse("https://feeds.example.test/show.xml?token=abc").unwrap()); + assert_eq!(card.privacy_warnings().len(), 1); +} + +// ------------------------------------------------------------------- search + +fn music(name: &str, kind: ItemKind) -> LibraryItem { + LibraryItem { + id: ItemId::from_bytes([1; 32]), + owner: secret(5).public(), + kind, + name: name.to_string(), + normalized_name: crate::normalize_name(name), + artist_names: Vec::new(), + featured_artist_names: Vec::new(), + year: None, + release_type: None, + release_title: None, + track_number: None, + disc_number: None, + duration_seconds: None, + content_id: None, + revision: 1, + deleted: false, + updated_at_ms: NOW, + } +} + +#[test] +fn hits_merge_publishers_and_drop_tombstoned_ones() { + let a = secret(1); + let b = secret(2); + let c = secret(3); + let mut moved = spec("moved.xml", "Rust Radio"); + moved.id_feed_url = Some(feed("show.xml")); + moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml")); + let records = vec![ + card(&a, 1), + card(&a, 2), + card_with(moved, &b, 1, NOW + 10), + card(&c, 1), + tombstone(&c, 2, NOW), + card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW), + ]; + let hits = rank_hits( + aggregate_hits(records), + "rust radio", + &podcast_query_tokens("rust radio"), + ); + assert_eq!(hits.len(), 1, "Rust Hour lacks the token radio"); + let hit = &hits[0]; + assert_eq!(hit.kind(), SearchHitKind::Podcast); + let mut expected = vec![a.public(), b.public()]; + expected.sort_by_key(|key| *key.as_bytes()); + assert_eq!(hit.publishers, expected); + // Equal votes per feed URL: the most recently issued card represents. + assert_eq!(hit.card.publisher, b.public()); +} + +#[test] +fn federated_results_are_typed_and_exact_matches_lead() { + let podcast = aggregate_hits(vec![card(&secret(1), 1)]).remove(0); + let merged = merge_search_results( + "rust radio", + vec![ + music("Rust Radio Theme", ItemKind::Track), + music("Rust", ItemKind::Artist), + ], + vec![podcast], + ); + let kinds: Vec<_> = merged.iter().map(FederatedSearchHit::kind).collect(); + assert_eq!( + kinds, + vec![ + SearchHitKind::Podcast, + SearchHitKind::Track, + SearchHitKind::Artist + ] + ); + let json = serde_json::to_value(&merged[0]).unwrap(); + assert_eq!(json["type"], "podcast"); + assert_eq!(json["hit"]["card"]["title"], "Rust Radio"); + let json = serde_json::to_value(&merged[1]).unwrap(); + assert_eq!(json["type"], "music"); + assert_eq!(json["hit"]["kind"], "track"); + assert_eq!(SearchHitKind::Podcast.as_str(), "podcast"); +} + +#[test] +fn query_tokens_are_normalized_distinct_and_bounded() { + assert_eq!( + podcast_query_tokens(" Rust, RUST radio!"), + vec!["rust", "radio"] + ); + let long: Vec = (0..20).map(|i| format!("w{i}")).collect(); + assert_eq!( + podcast_query_tokens(&long.join(" ")).len(), + MAX_PODCAST_QUERY_TOKENS + ); + assert!(podcast_query_tokens("!!!").is_empty()); +} + +#[test] +fn error_responses_are_bounded() { + let response = PodcastDhtResponse::error("é".repeat(MAX_ERROR_BYTES)); + assert!(response.error.as_ref().unwrap().len() <= MAX_ERROR_BYTES); + validate_response(&response).unwrap(); +} -- 2.54.0 From 2fcb5f1822f7d6e30912af289b2d57133b6deeaf Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:45:52 +0200 Subject: [PATCH 2/9] Test podcast discovery across three loopback nodes A publishes, an unrelated peer C finds the card by text and id while A's trusted device B is offline, a never-shared podcast stays invisible, a tombstone hides the card, and forged, tampered, replayed, malformed, oversized and wrong-version requests are rejected. A peer without the overlay never joins its routing table. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/tests/podcast_dht.rs | 421 ++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 crates/music-dht/tests/podcast_dht.rs diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs new file mode 100644 index 0000000..548f869 --- /dev/null +++ b/crates/music-dht/tests/podcast_dht.rs @@ -0,0 +1,421 @@ +//! Integration test: public podcast discovery on three real loopback iroh +//! nodes. +//! +//! * A publishes a podcast its user shared; B is one of A's trusted devices +//! (in this test merely another overlay node, then offline); C is an +//! unrelated peer outside A's trusted-device group, with no device-sync +//! pairing. +//! * C finds the card by text and by id; a podcast A never shared is never +//! discoverable; after A unpublishes, the tombstone hides the card from C. +//! * Forged, tampered, replayed, malformed, oversized and wrong-version +//! requests are rejected by a real peer. +//! * D, a music-DHT peer without the overlay, is never added to the overlay. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use music_dht::capabilities::{CapabilityManifest, PODCAST_DHT_ID}; +use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta, ShareRecord, SubscriptionRecord}; +use music_dht::podcast_dht::{ + MAX_PODCAST_DHT_REQUEST_BYTES, PODCAST_DHT_ALPN, PODCAST_DHT_PROTOCOL_VERSION, PodcastCard, + PodcastCardSpec, PodcastDht, PodcastDhtOperation, PodcastDhtRequest, PodcastDhtResponse, + PodcastRecordBody, PodcastSearchHit, PodcastShareState, PodcastStoreEntry, SignedPodcastRecord, +}; +use music_dht::{MusicDhtConfig, MusicDhtError, MusicDhtService, NetworkId, NodeContact, NodeId}; +use tokio::io::AsyncReadExt; + +const TEST_TIMEOUT: Duration = Duration::from_secs(240); + +struct Node { + service: Arc, + dht: Arc, + tasks: Vec>, +} + +impl Node { + async fn start(directory: &Path, network: NetworkId) -> Self { + let (service, events) = start_service(directory, network, true).await; + let acceptor = service.stream_acceptor(PODCAST_DHT_ALPN).unwrap(); + let dht = PodcastDht::open(Arc::clone(&service), directory.join("podcast-dht.sqlite3")) + .await + .unwrap(); + let tasks = vec![events, tokio::spawn(Arc::clone(&dht).serve(acceptor))]; + Self { + service, + dht, + tasks, + } + } + + async fn stop(self) { + for task in &self.tasks { + task.abort(); + } + self.service.shutdown().await.unwrap(); + } + + async fn contact(&self) -> NodeContact { + let peer_id = self.service.endpoint_id(); + NodeContact { + node_id: NodeId::from_endpoint(&peer_id), + peer_id, + ticket: self.service.ticket().await.unwrap().to_string(), + last_seen_ms: 1, + } + } +} + +async fn start_service( + directory: &Path, + network: NetworkId, + overlay: bool, +) -> (Arc, tokio::task::JoinHandle<()>) { + std::fs::create_dir_all(directory).unwrap(); + let mut config = MusicDhtConfig::builder() + .data_dir(directory) + .network_id(network) + .request_timeout(Duration::from_secs(2)) + .lookup_timeout(Duration::from_secs(5)) + .transport_timeout(Duration::from_secs(5)) + .dial_timeout(Duration::from_secs(2)); + if overlay { + config = config.schema_independent_stream_protocol(PODCAST_DHT_ALPN); + } + let (service, mut events) = MusicDhtService::start(config.build().unwrap()) + .await + .unwrap(); + let events = tokio::spawn(async move { while events.recv().await.is_some() {} }); + (Arc::new(service), events) +} + +fn feed(path: &str) -> FeedUrl { + FeedUrl::parse(&format!("https://feeds.example.test/{path}")).unwrap() +} + +/// A subscription of A's user (generation 1). +fn subscription(path: &str, title: &str) -> SubscriptionRecord { + let feed_url = feed(path); + SubscriptionRecord { + podcast_id: PodcastId::from_feed_url(&feed_url), + subscribed: true, + podcast: PodcastMeta { + feed_url, + title: title.to_string(), + author: Some("Ferris Crab".to_string()), + description: Some("Weekly news.".to_string()), + link: None, + image_url: None, + }, + feed_aliases: Vec::new(), + generation: 1, + hlc_ms: 1_000, + op_id: "device-a-op-1".to_string(), + } +} + +fn share(subscription: &SubscriptionRecord, shared: bool, generation: u64) -> ShareRecord { + ShareRecord { + podcast_id: subscription.podcast_id.clone(), + shared, + generation, + hlc_ms: 2_000 + generation as i64, + op_id: format!("device-a-share-{generation}"), + } +} + +async fn wait_for(what: &str, mut check: impl AsyncFnMut() -> Option) -> T { + let started = std::time::Instant::now(); + loop { + if let Some(value) = check().await { + return value; + } + assert!( + started.elapsed() < Duration::from_secs(60), + "timed out waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(300)).await; + } +} + +/// Sends raw bytes as one overlay request and decodes the response. +async fn raw_request(from: &Node, to: &Node, payload: &[u8]) -> PodcastDhtResponse { + let ticket = to.service.ticket().await.unwrap(); + let mut stream = from + .service + .open_stream_to(&ticket, PODCAST_DHT_ALPN) + .await + .unwrap(); + stream.send.write_all(payload).await.unwrap(); + stream.send.finish().unwrap(); + let mut bytes = Vec::new(); + AsyncReadExt::read_to_end(&mut stream.recv, &mut bytes) + .await + .unwrap(); + postcard::from_bytes(&bytes).unwrap() +} + +async fn store_request(from: &Node, to: &Node, record: SignedPodcastRecord) -> PodcastDhtResponse { + let request = PodcastDhtRequest { + version: PODCAST_DHT_PROTOCOL_VERSION, + requester: from.contact().await, + operation: PodcastDhtOperation::StoreBatch { + records: vec![PodcastStoreEntry { + keys: record.dht_keys(), + record, + }], + }, + }; + raw_request(from, to, &postcard::to_stdvec(&request).unwrap()).await +} + +async fn search(node: &Node, query: &str) -> Vec { + node.dht.search(query, 10).await.unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn shared_podcast_is_discoverable_by_strangers_and_retractable() { + tokio::time::timeout(TEST_TIMEOUT, scenario()) + .await + .unwrap(); +} + +async fn scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-integration"); + let a = Node::start(&temp.path().join("a"), network).await; + let b = Node::start(&temp.path().join("b"), network).await; + let c = Node::start(&temp.path().join("c"), network).await; + // D runs the music DHT only. + let (d, d_events) = start_service(&temp.path().join("d"), network, false).await; + + // B and C reach A through the ordinary music-DHT transport; C has no + // device-sync pairing with A or B. + b.service + .connect(a.service.ticket().await.unwrap()) + .await + .unwrap(); + c.service + .connect(a.service.ticket().await.unwrap()) + .await + .unwrap(); + d.connect(a.service.ticket().await.unwrap()).await.unwrap(); + wait_for("music DHT contacts", async || { + (a.service.known_peers().len() >= 3 + && !b.service.known_peers().is_empty() + && !c.service.known_peers().is_empty()) + .then_some(()) + }) + .await; + for node in [&a, &b, &c] { + node.dht.bootstrap().await.unwrap(); + } + wait_for("podcast overlay contacts", async || { + (a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(()) + }) + .await; + // D never answers on the overlay ALPN, so it never becomes an overlay + // contact and is never queried or asked to store records. + a.dht.bootstrap().await.unwrap(); + assert_eq!(a.dht.known_peers(), 2); + + // A capability manifest without podcast_dht evicts a peer; one with it + // lets the peer be learned again. + let manifest = CapabilityManifest::frid("furumi", "test"); + a.dht + .note_peer_capabilities( + c.service.endpoint_id(), + &manifest.clone().without_protocol(PODCAST_DHT_ID), + ) + .await + .unwrap(); + assert_eq!(a.dht.known_peers(), 1); + a.dht.bootstrap().await.unwrap(); + assert_eq!( + a.dht.known_peers(), + 1, + "the evicted peer is not bootstrapped" + ); + a.dht + .note_peer_capabilities(c.service.endpoint_id(), &manifest) + .await + .unwrap(); + a.dht.bootstrap().await.unwrap(); + assert_eq!(a.dht.known_peers(), 2); + + // A request with another overlay version is refused. + let request = PodcastDhtRequest { + version: PODCAST_DHT_PROTOCOL_VERSION + 1, + requester: c.contact().await, + operation: PodcastDhtOperation::FindNode { target: [0; 32] }, + }; + let response = raw_request(&c, &a, &postcard::to_stdvec(&request).unwrap()).await; + assert!(!response.ok); + + // A's user shares "Rust Radio Weekly" but never shares "Secret Garden". + let shared = subscription("rust-radio.xml", "Rust Radio Weekly"); + let private = subscription("secret-garden.xml", "Secret Garden Diaries"); + let states = vec![ + PodcastShareState { + subscription: shared.clone(), + share: Some(share(&shared, true, 2)), + card: None, + }, + PodcastShareState { + subscription: private.clone(), + share: None, + card: None, + }, + ]; + let stats = a.dht.sync_shares(states.clone()).await.unwrap(); + assert_eq!( + (stats.published, stats.unchanged, stats.unpublished), + (1, 0, 0) + ); + assert!(stats.push.local_replica); + assert!(stats.push.remote_nodes >= 1); + let error = a + .dht + .publish( + &private, + &share(&private, false, 2), + PodcastCardSpec::from_subscription(&private).unwrap(), + ) + .await + .unwrap_err(); + assert!(matches!(error, MusicDhtError::PodcastPublication(_))); + // Idempotent: nothing new is published. + let again = a.dht.sync_shares(states).await.unwrap(); + assert_eq!( + (again.published, again.unchanged, again.unpublished), + (0, 1, 0) + ); + let published = a.dht.published().await.unwrap(); + assert_eq!(published.len(), 1); + let first_card = published[0].record.clone(); + + // B (A's trusted device) is not needed for discovery. + b.stop().await; + + let hits = wait_for("C finds the card by text", async || { + let hits = search(&c, "rust radio").await; + (!hits.is_empty()).then_some(hits) + }) + .await; + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].podcast_id, shared.podcast_id); + assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]); + assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url); + assert_eq!(hits[0].card.title, "Rust Radio Weekly"); + assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens"); + assert!( + search(&c, "rust jazz").await.is_empty(), + "every token must match" + ); + let by_id = c.dht.lookup_podcast(&shared.podcast_id).await.unwrap(); + assert_eq!(by_id.unwrap().card.podcast_id, shared.podcast_id); + + // The never-shared podcast leaves no trace anywhere. + assert!(search(&c, "secret garden").await.is_empty()); + assert!(search(&a, "secret garden").await.is_empty()); + assert!( + c.dht + .lookup_podcast(&private.podcast_id) + .await + .unwrap() + .is_none() + ); + assert!( + a.dht + .lookup_podcast(&private.podcast_id) + .await + .unwrap() + .is_none() + ); + + // Forged: C signs a card that claims A as publisher. + let forged_spec = + PodcastCardSpec::from_subscription(&subscription("forged.xml", "Forged Show")).unwrap(); + let forged_card = PodcastCard::from_spec( + forged_spec.clone(), + network, + a.service.endpoint_id(), + u64::MAX - 1, + first_card.issued_at_ms(), + ); + let forged = SignedPodcastRecord { + signature: c.service.sign_identity(&forged_card.signing_bytes()), + body: PodcastRecordBody::Card(Box::new(forged_card)), + }; + let response = store_request(&c, &a, forged).await; + assert!(!response.ok && response.stored == 0, "{response:?}"); + + // Tampered: A's genuine card with a different feed URL. + let mut tampered = first_card.clone(); + if let PodcastRecordBody::Card(card) = &mut tampered.body { + card.description = Some("Now with malware".into()); + } + let response = store_request(&c, &a, tampered).await; + assert!(!response.ok, "{response:?}"); + + // Malformed and oversized requests. + let response = raw_request(&c, &a, b"definitely not postcard").await; + assert!(!response.ok); + let response = raw_request(&c, &a, &vec![0u8; MAX_PODCAST_DHT_REQUEST_BYTES + 1]).await; + assert!(!response.ok); + assert!(response.error.unwrap().contains("size")); + + // A valid card signed by C is accepted: the overlay is open to anyone + // who signs their own cards. + let own_card = PodcastCard::from_spec( + forged_spec, + network, + c.service.endpoint_id(), + 1, + first_card.issued_at_ms(), + ); + let own = + SignedPodcastRecord::sign_card(own_card, |bytes| c.service.sign_identity(bytes)).unwrap(); + let response = store_request(&c, &a, own).await; + assert!(response.ok && response.stored > 0, "{response:?}"); + let forged_hits = search(&a, "forged show").await; + assert_eq!(forged_hits.len(), 1); + assert_eq!(forged_hits[0].publishers, vec![c.service.endpoint_id()]); + + // A's user stops sharing: a tombstone with the next revision replaces + // the card and C stops seeing it. + let unshare = vec![PodcastShareState { + subscription: shared.clone(), + share: Some(share(&shared, false, 3)), + card: None, + }]; + let stats = a.dht.sync_shares(unshare).await.unwrap(); + assert_eq!(stats.unpublished, 1); + let published = a.dht.published().await.unwrap(); + assert!(!published[0].active); + assert!(published[0].record.is_tombstone()); + assert_eq!(published[0].revision, first_card.revision() + 1); + wait_for("tombstone hides the card from C", async || { + search(&c, "rust radio").await.is_empty().then_some(()) + }) + .await; + assert!( + c.dht + .lookup_podcast(&shared.podcast_id) + .await + .unwrap() + .is_none() + ); + + // Replaying the old card, even to the node that stores the tombstone, + // does not bring it back. + let response = store_request(&c, &a, first_card).await; + assert!(response.ok && response.stored == 0, "{response:?}"); + assert!(search(&c, "rust radio").await.is_empty()); + assert!(search(&a, "rust radio").await.is_empty()); + + a.stop().await; + c.stop().await; + d_events.abort(); + d.shutdown().await.unwrap(); +} -- 2.54.0 From fef155d6ef1972c0ab6acea1826eec3e5677b06e Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:45:52 +0200 Subject: [PATCH 3/9] Document public podcast discovery Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 24 +++- CHANGELOG.md | 20 ++++ PODCASTS.md | 222 ++++++++++++++++++++++++++++++++++++- crates/music-dht/README.md | 15 +++ 4 files changed, 274 insertions(+), 7 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0553b05..23a53ef 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -232,8 +232,28 @@ refused, and their playback commands are ignored while the queue contains episodes. Jam snapshots replace episodes with opaque placeholders. Progress, subscription and share registers order by a Lamport generation before wall-clock `hlc_ms`, so a device with a fast clock cannot undo an observed -seek, restart, unsubscribe or share decision. Public podcast discovery is a future -separate overlay; the music DHT schema and `federation-net` are unchanged. +seek, restart, unsubscribe or share decision. The music DHT schema and +`federation-net` are unchanged. + +Public podcast discovery is a third schema-independent Kademlia overlay, +`music_dht::podcast_dht` (ALPN `furumi-fd/podcast-dht/1`, version 1, +capability `podcast_dht`), built like the similarity-routing overlay. A +device publishes a card only for podcasts whose merged registers satisfy +`effective_share`; `sync_shares` reconciles publications declaratively and +nothing is sent for a podcast the device never published. A card is the +public projection of a podcast (feed URL and channel metadata, no personal +state), signed by the device's iroh identity over a domain-separated +canonical encoding and bound to its id by the URL the id hashes from. Cards +live under the podcast-id key and one key per normalized title/author token; +unpublishing writes a signed tombstone of the next revision that supersedes +lower and equal revisions under every key and outlives every older card. +Records use the music DHT's active/tombstone TTLs and republish cadence, are +verified on every receive and cache read, and are kept in a separate, +replaceable SQLite cache. Search filters on every replica by all query tokens, +merges cards of one podcast across publisher devices, and returns typed +`podcast` hits that clients merge with music results. Capability data only +helps skip peers; it never authorizes. See the "Public discovery" section of +[PODCASTS.md](PODCASTS.md). Music discovery identifies an owner and content; it does not force all domain traffic through DHT messages. `music-dht` exposes the underlying authenticated diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f9f284..cb64153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## Unreleased — podcast discovery (`podcast_dht`) + +- `music-dht 0.6.0`: new `music_dht::podcast_dht` overlay for public podcast + discovery on its own ALPN `furumi-fd/podcast-dht/1` (version 1) and its own + SQLite cache: signed `PodcastCard`s and `PodcastTombstone`s + (`furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1` canonical + signing input), monotonic revisions with a total tie-break, text and id + keys, `PodcastDht::{publish, unpublish, sync_shares, republish, search, + lookup_podcast}` gated by `effective_share`, typed search hits + (`PodcastSearchHit`, `FederatedSearchHit`, `SearchHitKind`, + `merge_search_results`) and the advisory `feed_url_privacy_warning`. +- `music-dht 0.6.0`: capability manifests advertise `podcast_dht = 1` + (`capabilities::PODCAST_DHT_ID`, `CapabilityManifest::protocol_version`, + `CapabilityManifest::supports_podcast_dht`); applications that do not run + the overlay must remove it with `without_protocol`. +- `music-dht 0.6.0`: new `MusicDhtError::PodcastPublication` variant (the enum + is `#[non_exhaustive]`). +- The music DHT (`music-dht-v5`), `federation-net` and device-sync protocol v4 + are unchanged. See "Public discovery (podcast_dht)" in `PODCASTS.md`. + ## Unreleased — podcast contract - `music-dht 0.6.0` (breaking): new `music_dht::podcast` module with the shared diff --git a/PODCASTS.md b/PODCASTS.md index 010a50d..dcd9653 100644 --- a/PODCASTS.md +++ b/PODCASTS.md @@ -10,8 +10,9 @@ live in `music_dht::podcast` (pure types and functions) and `music_dht::device_sync` (wire DTOs and gating helpers). Clients must use them instead of local copies. -Public discovery (signed podcast cards in a separate DHT overlay) is not part -of this version of the contract; it will be specified with the overlay. +Public discovery of podcasts a user explicitly shared uses signed podcast +cards in a separate DHT overlay, `music_dht::podcast_dht`; see +[Public discovery (podcast_dht)](#public-discovery-podcast_dht). ## Responsibilities @@ -19,6 +20,7 @@ of this version of the contract; it will be specified with the overlay. | --- | --- | | `music_dht::podcast` | ids, URL canonicalization, no-GUID rule, metadata DTOs and bounds, merge functions, RSS/Atom parser (feature `podcast-feed`) | | `music_dht::device_sync` | sync operation kinds, snapshot sections, `PlaybackSource`, version gating, Jam redaction | +| `music_dht::podcast_dht` | public podcast cards and tombstones, their signatures, replication, cache and search, privacy warnings for feed URLs | | Client storage (`furumi_library::podcasts::PodcastStore`, web PostgreSQL) | persistence and migrations implementing the rules below (SQLite reference: see "SQLite reference store") | | Client runtime | HTTP fetching, conditional requests, scheduling, audio buffering and decoding, UI | @@ -32,7 +34,7 @@ offered for P2P transfer. RSS refreshes fetch metadata only. | --- | --- | --- | --- | | Subscription state, channel metadata | `podcast_subscription_set` + snapshot | never | | | Feed aliases | `podcast_feed_aliases_added`, `podcast_subscription_set` + snapshot | never | | -| Share decision | `podcast_share_set` + snapshot | only the published card (future overlay) | | +| Share decision | `podcast_share_set` + snapshot | only the signed card while `effective_share` holds (`podcast_dht`) | | | Episode progress | `podcast_progress_set` + snapshot | never | | | Listening sessions, play counts | `podcast_listen_recorded` + snapshot | never | | | Episode catalog | each device parses RSS itself; `PodcastEpisodeRef` travels with progress and queue items | never | | @@ -498,6 +500,213 @@ Jam participants are outside the trusted-device group. private item. States that drop, duplicate or inject private items are rejected. Participants may still pause, seek and reorder music. +## Public discovery (podcast_dht) + +`music_dht::podcast_dht` publishes podcasts the user explicitly shared so +that peers outside the user's trusted-device group can find and subscribe to +them. It is a separate, schema-independent Kademlia overlay built like +`similarity_dht`: its own ALPN, version, routing contacts and SQLite cache. +The music DHT (`music-dht-v5`) and `federation-net` are unchanged. Every +device of a user publishes under its own identity; search merges the cards +of one podcast by `podcast_id`. + +| Item | Value | +| --- | --- | +| ALPN | `furumi-fd/podcast-dht/1` (schema-independent stream protocol) | +| Wire and record version | `PODCAST_DHT_PROTOCOL_VERSION = 1` | +| Capability manifest | `podcast_dht = 1` (`CapabilityManifest::supports_podcast_dht`) | +| Cache | separate SQLite file, `PRAGMA user_version = 1`; a newer schema is refused | +| Card TTL | `ACTIVE_RECORD_TTL` of the music DHT (12 h) | +| Tombstone TTL | `TOMBSTONE_TTL` of the music DHT (24 h) | +| Republish | `DEFAULT_REPUBLISH_INTERVAL` (10 min), expiry sweep `DEFAULT_EXPIRE_INTERVAL` (1 min) | +| Clock skew | `issued_at_ms` at most 5 min in the future | + +### Card + +A card is the public projection of one podcast by one publisher device. It +is built from the subscription with `PodcastCardSpec::from_subscription`, +which reads only the podcast id, the feed URLs and the channel metadata. It +never contains progress, listening history, play counts, subscription or +share registers (`generation`, `hlc_ms`, `op_id`), device names, local paths, +HTTP validators or episodes. + +| Field | Bound | +| --- | --- | +| `version`, `network_id`, `publisher` | 1, the federation network, the device's iroh endpoint key | +| `podcast_id` | must equal `PodcastId::from_feed_url(id_feed_url or feed_url)` | +| `feed_url` | canonical `FeedUrl` the feed is fetched from | +| `id_feed_url` | the URL the id was derived from, present only when it differs from `feed_url` (feed moved) | +| `title` | 1..300 characters, no control characters | +| `author` | optional, ≤ 300 characters, no control characters | +| `description` | optional, ≤ 1 000 characters, only `\n`/`\t` controls | +| `image_url`, `link` | optional absolute `http(s)` URLs ≤ 2 048 bytes | +| `language` | optional, ≤ 35 bytes of ASCII letters, digits and `-` | +| `categories` | ≤ 8, each 1..64 characters, no control characters | +| `episode_count`, `latest_episode_at_ms` | optional; the time is ≥ 0 | +| `revision` | ≥ 1, monotonic per `(podcast_id, publisher)` | +| `issued_at_ms`, `expires_at_ms` | `0 < issued < expires ≤ issued + 12 h` | + +`PodcastCardSpec::bounded` truncates publisher data to these bounds. The +whole signed record is at most 24 KiB postcard-encoded; the field bounds keep +every valid card below it. + +Binding the id to a URL in the card prevents claiming an existing podcast id +for an arbitrary feed. When a feed moved, `feed_url` is the publisher's +claim; clients should prefer their own subscription's URLs for a podcast +they already know. + +### Tombstone + +`(version, network_id, publisher, podcast_id, revision, issued_at_ms, +expires_at_ms, index_keys)` with `expires_at_ms ≤ issued_at_ms + 24 h` and +`index_keys` the sorted, distinct text keys of the retracted card (at most +32, without the podcast-id key). It carries key hashes, not the title. A +tombstone supersedes every card of the same `(podcast_id, publisher)` with a +lower or equal revision and is itself retained and served until it expires. +Because the tombstone TTL exceeds the card TTL, every card issued before a +tombstone expires before it. + +### Signing input + +Both records are signed with the publisher's Ed25519 iroh identity over a +canonical, language-independent encoding: the domain string, a `0x00` byte, +then the fields in the order listed. Integers are little-endian (`u16`, +`u32`, `u64`, `i64`); strings are a `u32` byte length followed by UTF-8; +`Option` is `0x00`, or `0x01` followed by the value; lists are a `u32` count +followed by the items; `network_id`, `publisher` and keys are their raw 32 +bytes. + +* Card, domain `furumi-podcast-card:v1`: `version u16, network_id, publisher, + podcast_id str, feed_url str, id_feed_url opt str, title str, author opt + str, description opt str, image_url opt str, link opt str, language opt + str, categories list of str, episode_count opt u32, latest_episode_at_ms + opt i64, revision u64, issued_at_ms u64, expires_at_ms u64`. +* Tombstone, domain `furumi-podcast-card-tombstone:v1`: `version u16, + network_id, publisher, podcast_id str, revision u64, issued_at_ms u64, + expires_at_ms u64, index_keys list of 32-byte keys`. + +Unit tests pin BLAKE3 vectors of both encodings. On the wire, records are +postcard `SignedPodcastRecord { body: Card | Tombstone, signature }`. + +### Keys and placement + +`key = BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind || +material)` with kind `0x01` and the podcast id string for the id key, or +kind `0x02` and a token for a text key. Tokens are the distinct +`normalize_name` tokens of the title, then of the author, skipping tokens +longer than 64 bytes, at most 32. A card is stored under the id key and its +text keys (at most 33); a tombstone under the id key and its `index_keys`. +Replicas accept a record only under a key derived from it. + +### Ordering + +For one `(podcast_id, publisher)` the greater record wins: higher +`revision`; at an equal revision a tombstone beats a card; then later +`issued_at_ms` (a republish); then later `expires_at_ms`; then the +lexicographically greater signing input. This is a total order, so every +replica converges whatever the delivery order. A replica that holds a +tombstone, or a card of a higher revision, under any key drops older cards of +that publisher under every key and refuses them later; a replayed old +revision is therefore ignored. + +Revisions: a changed card gets `max(previous + 1, now_ms)` (time-seeded, so +revisions stay monotonic even if the local cache is lost); an unchanged card +is re-signed with the same revision and a fresh lifetime on every republish; +unpublishing writes a tombstone with `previous + 1`; publishing again later +gets `max(tombstone + 1, now_ms)`. + +### Publication + +* `PodcastDht::publish(subscription, share, card)` refuses + (`MusicDhtError::PodcastPublication`) unless `effective_share` holds, so a + new or resumed subscription is never published without a fresh share + decision. +* `PodcastDht::sync_shares(states)` is declarative and idempotent: podcasts + for which `effective_share` holds are published (when changed, new or past + half their lifetime), every other podcast this device published is + unpublished. Clients call it after applying share or subscription + operations, so unsharing or unsubscribing on any trusted device revokes + every device's own card once the registers converge, and a stale device + cannot republish (its registers lose to the newer share decision). +* `PodcastDht::unpublish(podcast_id)` writes the tombstone. Nothing is ever + sent for a podcast this device never published: no tombstone, no id. +* `PodcastDht::republish` (run by `maintenance`) re-signs active cards and + re-sends unexpired tombstones every 10 minutes. +* Before publishing, the UI must call `feed_url_privacy_warning` (or + `PodcastCardSpec::privacy_warnings`, which also checks `id_feed_url`) and + ask the user when it returns a warning. The heuristics are advisory: query + parameter names containing words such as `token`, `key`, `auth`, `secret`, + `sig`/`signature`, `uid`, `user`, `pass`, `session`, `access`, `private`; + known member-feed hosts; and long high-entropy query values (≥ 16 + characters) or path segments (≥ 32 characters). False positives and misses + are possible. + +### Search + +* `PodcastDht::search(query, limit)` normalizes the query + (`podcast_query_tokens`: distinct tokens, at most 8), looks up the text + keys of the three longest tokens and asks every replica to return only + cards indexed under **all** query tokens (tombstones are always returned). + Results are verified again, the winning record per publisher is kept, + tombstoned publishers are dropped and cards are merged by `podcast_id` + into `PodcastSearchHit { podcast_id, card, publishers }`. Ranking: exact + normalized title first, then more publishers. +* `PodcastDht::lookup_podcast(podcast_id)` reads the id key. +* `publishers` lists distinct publisher **devices**; it is not a subscriber + count (one user may publish from several devices, and subscribers who did + not share are invisible), so no subscriber count is offered. +* The representative card is the one whose `feed_url` most publishers agree + on, then the most recently issued, then the greatest publisher key. +* `FederatedSearchHit` (`{"type": "music" | "podcast", "hit": ...}`), + `SearchHitKind` (`artist`, `release`, `track`, `podcast`) and + `merge_search_results` merge podcast hits with `SearchOutcome` music + results; `SearchOutcome` itself is unchanged. +* Lookups query the 8 closest overlay nodes (α = 3, at most 32 requests, + 15 s deadline) and return partial results when peers fail. A peer that + fails a request is skipped for 30 s. + +Subscribing from a hit creates the user's own private subscription: resolve +the id with `resolve_podcast_id` on the card's `id_feed_url` (or `feed_url`), +which yields the card's `podcast_id` unless the user already knows that URL, +fetch the feed from `feed_url` and record it as an alias when it differs. The +new subscription is private (see `effective_share`), the publisher's +availability is not needed afterwards, and the episode catalog comes from RSS. + +### Limits + +| Limit | Value | +| --- | --- | +| Request / response | 512 KiB / 4 MiB postcard | +| Store batch | 1..16 entries, each 1..33 distinct keys | +| Records per `FindValue` response | 128 | +| Filter tokens | ≤ 8, each 1..64 bytes | +| Records cached per key | 256; the record expiring first is evicted for a longer-lived one | +| Concurrent inbound requests | 64 (more are dropped); 10 s per request | +| Error message | ≤ 1 KiB | +| Contact ticket | ≤ 16 KiB | + +### Privacy guarantees + +* Only podcasts with `effective_share` are published; never-shared podcasts + produce no network traffic at all. +* Cards carry the public projection only (see Card); tombstones carry key + hashes only. +* Records are signed by the publisher device and verified on every receive + and on every cache read; a forwarder cannot alter or impersonate them, and + records from another network are rejected. +* Cards do not carry a connection ticket: discovering a card does not reveal + how to reach the publisher, and subscribing never contacts it. +* The publisher's device key is public in the card. Devices of one user are + not linked by the overlay, but publishing the same podcasts from several + devices makes them correlatable. +* Unpublishing is eventually consistent: replicas that missed the tombstone + keep serving the old card until it expires (at most 12 h). A replica that + only held text keys of an older revision (after a title change) may serve + that revision until it expires. +* Capability manifests are informational. A peer without `podcast_dht` never + enters the overlay routing table (it cannot answer on the ALPN), and + `PodcastDht::note_peer_capabilities` evicts a peer whose manifest lacks it. + ## Feed parser With the `podcast-feed` feature, `music_dht::podcast::feed` parses RSS 2.0, @@ -667,5 +876,8 @@ must keep the same invariants. * Feed URL canonicalization, the identity bounds (`MAX_URL_BYTES`, `MAX_GUID_BYTES`) and the no-GUID rule are frozen with the `v1` domains: receivers reject non-canonical ids and URLs while parsing. -* Public podcast cards will use a separate signed overlay with its own ALPN - and version; the music DHT schema and `federation-net` do not change. +* Public podcast cards use the separate signed overlay `podcast_dht` with + its own ALPN (`furumi-fd/podcast-dht/1`), wire/record version 1, signing + domains `furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1`, key + domain `furumi-podcast-dht:key:v1` and cache schema 1, all frozen; the music + DHT schema and `federation-net` do not change. diff --git a/crates/music-dht/README.md b/crates/music-dht/README.md index e2f16dd..f5cb318 100644 --- a/crates/music-dht/README.md +++ b/crates/music-dht/README.md @@ -250,6 +250,21 @@ reported itself (`profile_protocol_version`; relayed profiles go through `jam::redact_jam_snapshot`, `jam_command_allowed` and `restore_jam_private_items`. +### Public podcast discovery + +`music_dht::podcast_dht::PodcastDht` is a separate signed overlay (ALPN +`furumi-fd/podcast-dht/1`, capability `podcast_dht`) for podcasts the user +explicitly shared. Declare the ALPN with +`schema_independent_stream_protocol`, open the overlay with its own cache file, +and spawn `serve` and `maintenance`, as for `SimilarityDht`. Call +`sync_shares` with the merged subscription and share registers after applying +sync operations: it publishes cards only where `effective_share` holds and +tombstones every other own publication. Ask the user before publishing when +`PodcastCardSpec::privacy_warnings` (or `feed_url_privacy_warning`) reports a +URL that looks private. `search` and `lookup_podcast` return +`PodcastSearchHit`s; `merge_search_results` combines them with +`SearchOutcome::network_results` as typed `FederatedSearchHit`s. + ## Verification ```bash -- 2.54.0 From 81ccd924d0d60310e0a05ee48fdce8ee227bb6c3 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 17:11:50 +0200 Subject: [PATCH 4/9] Harden podcast discovery against sybil and hijack records Replicas now accept records only from their publisher, keep one payload per (podcast, publisher), store placements only for keys they are close to, and admit records within per-key, per-tombstone, per-publisher and total budgets without ever evicting existing records. Tombstones cannot claim text keys without their owner's card. Search offers only the feed URL the podcast id hashes from and reports moved-feed URLs as claims; publisher counts no longer rank. sync_shares handles each podcast on its own, always revokes, and holds back cards with privacy warnings. Lookups and inbound requests per peer are bounded, persisted contacts are capped, and the podcast_dht capability is opt-in. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/capabilities.rs | 32 +- crates/music-dht/src/podcast_dht/card.rs | 25 +- crates/music-dht/src/podcast_dht/mod.rs | 280 ++++++++++-- crates/music-dht/src/podcast_dht/search.rs | 96 +++- crates/music-dht/src/podcast_dht/store.rs | 493 ++++++++++++++------- crates/music-dht/src/podcast_dht/tests.rs | 450 +++++++++++++++++-- crates/music-dht/tests/podcast_dht.rs | 181 +++++++- 7 files changed, 1276 insertions(+), 281 deletions(-) diff --git a/crates/music-dht/src/capabilities.rs b/crates/music-dht/src/capabilities.rs index f4525f8..06249d7 100644 --- a/crates/music-dht/src/capabilities.rs +++ b/crates/music-dht/src/capabilities.rs @@ -56,7 +56,13 @@ pub struct CapabilityManifest { } impl CapabilityManifest { - /// Creates a manifest containing every protocol owned by Frid. + /// Creates a manifest containing every protocol owned by Frid that a + /// music-DHT application serves. + /// + /// The public podcast-discovery overlay is opt-in: applications that run + /// [`crate::podcast_dht::PodcastDht`] add it with + /// [`Self::with_podcast_dht`], so an application that does not serve the + /// overlay never advertises it. pub fn frid(application: impl Into, application_version: impl Into) -> Self { let mut protocols = BTreeMap::new(); protocols.insert( @@ -81,10 +87,6 @@ impl CapabilityManifest { SIMILARITY_DHT_ID.to_string(), crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION, ); - protocols.insert( - PODCAST_DHT_ID.to_string(), - crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION, - ); protocols.insert( DEVICE_SYNC_ID.to_string(), crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION, @@ -111,6 +113,17 @@ impl CapabilityManifest { self } + /// Advertises the public podcast-discovery overlay + /// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]). + /// Call it only when the application serves + /// [`crate::podcast_dht::PODCAST_DHT_ALPN`]. + pub fn with_podcast_dht(self) -> Self { + self.with_protocol( + PODCAST_DHT_ID, + crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION, + ) + } + /// Version of protocol `id` advertised by the peer, if any. pub fn protocol_version(&self, id: &str) -> Option { self.protocols.get(id).copied() @@ -237,7 +250,6 @@ mod tests { CATALOG_ID, SIMILARITY_ID, SIMILARITY_DHT_ID, - PODCAST_DHT_ID, DEVICE_SYNC_ID, JAM_ID, ] { @@ -247,13 +259,17 @@ mod tests { } #[test] - fn podcast_dht_capability_is_advertised_and_removable() { - let manifest = CapabilityManifest::frid("furumi", "1.2.3"); + fn podcast_dht_capability_is_opt_in_and_removable() { + let plain = CapabilityManifest::frid("furumi", "1.2.3"); + assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None); + assert!(!plain.supports_podcast_dht()); + let manifest = plain.with_podcast_dht(); assert_eq!( manifest.protocol_version(PODCAST_DHT_ID), Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION) ); assert!(manifest.supports_podcast_dht()); + manifest.validate().unwrap(); let without = manifest.clone().without_protocol(PODCAST_DHT_ID); assert!(!without.supports_podcast_dht()); let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2); diff --git a/crates/music-dht/src/podcast_dht/card.rs b/crates/music-dht/src/podcast_dht/card.rs index 2bcfd75..0b549da 100644 --- a/crates/music-dht/src/podcast_dht/card.rs +++ b/crates/music-dht/src/podcast_dht/card.rs @@ -396,6 +396,19 @@ impl PodcastCard { card_index_tokens(&self.title, self.author.as_deref()) } + /// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url` + /// when the feed has not moved). A valid card's id hashes from it, so it + /// is the only URL of the card bound to the podcast. + pub fn origin_feed_url(&self) -> &FeedUrl { + self.id_feed_url.as_ref().unwrap_or(&self.feed_url) + } + + /// Whether `feed_url` is the URL the id is derived from. Otherwise + /// `feed_url` is an unverified claim of the publisher that the feed moved. + pub fn feed_url_is_origin(&self) -> bool { + self.id_feed_url.is_none() + } + /// Canonical signing input (see `PODCASTS.md`). pub fn signing_bytes(&self) -> Vec { let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN); @@ -735,18 +748,6 @@ pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) .then_with(|| left.signing_bytes().cmp(&right.signing_bytes())) } -/// Whether `winner` makes `other` obsolete everywhere, not just under one key: -/// `other` is a card of a lower revision, or `winner` is a tombstone of an -/// equal or higher revision. -pub(crate) fn supersedes_everywhere( - winner: &SignedPodcastRecord, - other: &SignedPodcastRecord, -) -> bool { - !other.is_tombstone() - && (other.revision() < winner.revision() - || (winner.is_tombstone() && other.revision() <= winner.revision())) -} - struct Canonical(Vec); impl Canonical { diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs index 276b897..f71554f 100644 --- a/crates/music-dht/src/podcast_dht/mod.rs +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -27,6 +27,13 @@ //! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with //! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take //! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`]. +//! Advertise the overlay with [`CapabilityManifest::with_podcast_dht`] (it is +//! opt-in) and pass peer manifests to [`PodcastDht::note_peer_capabilities`]. +//! +//! Only the URL a podcast id hashes from is bound to the podcast; search hits +//! offer that URL ([`PodcastSearchHit::feed_url`]) and report moved-feed URLs +//! as unverified claims. The publisher key is the device's transport identity +//! and therefore linkable to its network address and music-DHT records. mod card; mod privacy; @@ -36,7 +43,7 @@ mod store; #[cfg(test)] mod tests; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -69,10 +76,16 @@ pub use privacy::{ FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning, }; pub use search::{ - FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, - SearchHitKind, merge_search_results, podcast_query_tokens, + ClaimedFeedUrl, FederatedSearchHit, MAX_HIT_FEED_URL_CLAIMS, MAX_HIT_PUBLISHERS, + MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, SearchHitKind, merge_search_results, + podcast_query_tokens, +}; +pub use store::{ + MAX_PERSISTED_PODCAST_DHT_PEERS, MAX_PODCAST_CACHE_BYTES, MAX_PODCAST_CACHE_RECORDS, + MAX_PODCAST_RECORDS_PER_KEY, MAX_PODCAST_RECORDS_PER_PUBLISHER, + MAX_PODCAST_RESPONSE_TOMBSTONES, MAX_PODCAST_TOMBSTONES_PER_KEY, + PODCAST_DHT_CACHE_SCHEMA_VERSION, }; -pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION}; use card::publication_error; use search::{aggregate_hits, rank_hits}; @@ -105,6 +118,16 @@ pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60); /// Maximum inbound requests processed concurrently; further streams are /// dropped. pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64; +/// Maximum inbound requests of one peer processed concurrently; further +/// streams of that peer are dropped. +pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8; +/// Maximum distinct `(podcast_id, publisher)` records one lookup keeps; +/// records of further owners are ignored. +pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512; +/// A node stores a placement only when fewer than this many of its known +/// overlay contacts are closer to the key than itself (`2 * K`: the `K` +/// closest with slack for differing routing views). +pub const PODCAST_STORE_RESPONSIBILITY_WINDOW: usize = 2 * K; /// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is /// not contacted. pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60); @@ -228,7 +251,7 @@ pub struct PodcastDhtPublishStats { } /// Result of [`PodcastDht::sync_shares`]. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct PodcastDhtSyncStats { /// Cards published with a new revision, or refreshed because they used /// half their lifetime. @@ -237,10 +260,38 @@ pub struct PodcastDhtSyncStats { pub unchanged: usize, /// Podcasts unpublished with a tombstone. pub unpublished: usize, + /// Shared podcasts not published because their card has privacy + /// warnings the user has not confirmed. The UI asks, then calls + /// [`PodcastDht::publish`] or `sync_shares` with + /// [`PodcastShareState::privacy_confirmed`]. An earlier publication of + /// the podcast stays as it was. + pub held_back: Vec, + /// Podcasts whose card could not be built or signed. They are not + /// published, and an earlier publication of them is unpublished. + pub failed: Vec, /// Network statistics of the pushed records. pub push: PodcastDhtPublishStats, } +/// A shared podcast held back by [`PodcastDht::sync_shares`] until the user +/// confirms publishing URLs that look private. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastPrivacyHold { + /// Podcast. + pub podcast_id: PodcastId, + /// Warnings of [`PodcastCardSpec::privacy_warnings`]. + pub warnings: Vec, +} + +/// A podcast [`PodcastDht::sync_shares`] could not publish or unpublish. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastSyncFailure { + /// Podcast. + pub podcast_id: PodcastId, + /// Diagnostic. + pub error: String, +} + /// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PodcastShareState { @@ -251,6 +302,11 @@ pub struct PodcastShareState { /// Card to publish when shared; `None` derives it with /// [`PodcastCardSpec::from_subscription`]. pub card: Option, + /// Whether the user confirmed publishing this card's feed URLs despite + /// its [`PodcastCardSpec::privacy_warnings`]. Without it, a card with + /// warnings is only published when this device already publishes the + /// same feed URLs. + pub privacy_confirmed: bool, } impl PodcastShareState { @@ -258,6 +314,22 @@ impl PodcastShareState { pub fn is_shared(&self) -> bool { effective_share(Some(&self.subscription), self.share.as_ref()) } + + /// The bounded, validated card [`PodcastDht::sync_shares`] would + /// publish: [`Self::card`] or one derived from the subscription. + pub fn card_spec(&self) -> Result { + let card = match &self.card { + Some(card) => card.bounded(), + None => PodcastCardSpec::from_subscription(&self.subscription)?, + }; + if card.podcast_id != self.subscription.podcast_id { + return Err(publication_error( + "card and subscription describe different podcasts", + )); + } + card.validate()?; + Ok(card) + } } /// This device's publication state of one podcast. @@ -284,6 +356,7 @@ pub struct PodcastDht { failed: Mutex>, publish_gate: Semaphore, serve_limit: Arc, + inflight: Arc>>, } impl std::fmt::Debug for PodcastDht { @@ -330,6 +403,7 @@ impl PodcastDht { failed: Mutex::new(HashMap::new()), publish_gate: Semaphore::new(1), serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), + inflight: Arc::new(Mutex::new(HashMap::new())), })) } @@ -341,6 +415,10 @@ impl PodcastDht { /// Serves authenticated overlay requests until `acceptor` closes. pub async fn serve(self: Arc, mut acceptor: StreamAcceptor) { while let Some(stream) = acceptor.accept().await { + let Some(slot) = PeerSlot::acquire(&self.inflight, stream.peer_id) else { + debug!(peer = %stream.peer_id, "podcast DHT request dropped: peer limit"); + continue; + }; let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else { debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded"); continue; @@ -348,6 +426,7 @@ impl PodcastDht { let this = Arc::clone(&self); tokio::spawn(async move { let _permit = permit; + let _slot = slot; let peer = stream.peer_id; match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await { Ok(Ok(())) => {} @@ -421,7 +500,10 @@ impl PodcastDht { } /// Discovers overlay peers among the routing table and the music DHT - /// contacts. + /// contacts with one `FindNode` each. Music-DHT contacts whose capability + /// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are + /// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become + /// overlay contacts when they answer. pub async fn bootstrap(&self) -> Result<()> { let known_feature: HashSet = lock(&self.routing) .contacts() @@ -521,44 +603,92 @@ impl PodcastDht { /// [`Self::republish`]). Every other podcast this device published, /// including podcasts absent from `states`, is unpublished. The call is /// idempotent: repeating it publishes nothing new. + /// + /// Each podcast is handled on its own, so revocation is fail-safe: a + /// shared podcast whose card cannot be built is reported in + /// [`PodcastDhtSyncStats::failed`] and unpublished, and never prevents + /// other podcasts from being unpublished. A card with + /// [`PodcastCardSpec::privacy_warnings`] is held back + /// ([`PodcastDhtSyncStats::held_back`]) unless + /// [`PodcastShareState::privacy_confirmed`] is set or this device already + /// publishes the same feed URLs. pub async fn sync_shares(&self, states: Vec) -> Result { - let mut desired: HashMap = HashMap::new(); - for state in states { + let _permit = self.publish_permit().await?; + let local = self.database.local_publications().await?; + let active: HashMap<&PodcastId, &PodcastCardSpec> = local + .iter() + .filter_map(|publication| { + publication + .spec + .as_ref() + .map(|spec| (&publication.podcast_id, spec)) + }) + .collect(); + let mut stats = PodcastDhtSyncStats::default(); + let mut desired: BTreeMap = BTreeMap::new(); + let mut held: HashSet = HashSet::new(); + for state in &states { if !state.is_shared() { continue; } - let card = match state.card { - Some(card) => card.bounded(), - None => PodcastCardSpec::from_subscription(&state.subscription)?, + let podcast_id = state.subscription.podcast_id.clone(); + let card = match state.card_spec() { + Ok(card) => card, + Err(error) => { + debug!(%podcast_id, %error, "podcast card cannot be published"); + stats.failed.push(PodcastSyncFailure { + podcast_id, + error: error.to_string(), + }); + continue; + } }; - if card.podcast_id != state.subscription.podcast_id { - return Err(publication_error( - "card and subscription describe different podcasts", - )); + let warnings = card.privacy_warnings(); + let already_published = active.get(&podcast_id).is_some_and(|published| { + published.feed_url == card.feed_url && published.id_feed_url == card.id_feed_url + }); + if !warnings.is_empty() && !state.privacy_confirmed && !already_published { + held.insert(podcast_id.clone()); + stats.held_back.push(PodcastPrivacyHold { + podcast_id, + warnings, + }); + continue; } - card.validate()?; - desired.insert(card.podcast_id.clone(), card); + desired.insert(podcast_id, card); } - let _permit = self.publish_permit().await?; - let local = self.database.local_publications().await?; - let mut stats = PodcastDhtSyncStats::default(); let mut records = Vec::new(); for publication in &local { - if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) { - records.extend(self.unpublish_locked(&publication.podcast_id).await?); - stats.unpublished += 1; + let podcast_id = &publication.podcast_id; + if publication.spec.is_none() + || desired.contains_key(podcast_id) + || held.contains(podcast_id) + { + continue; + } + match self.unpublish_locked(podcast_id).await { + Ok(record) => { + records.extend(record); + stats.unpublished += 1; + } + Err(error) => stats.failed.push(PodcastSyncFailure { + podcast_id: podcast_id.clone(), + error: error.to_string(), + }), } } - let mut desired: Vec<_> = desired.into_values().collect(); - desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id)); - for card in desired { - match self.sign_card_locked(card, false).await? { - Some(record) => { + for (podcast_id, card) in desired { + match self.sign_card_locked(card, false).await { + Ok(Some(record)) => { records.push(record); stats.published += 1; } - None => stats.unchanged += 1, + Ok(None) => stats.unchanged += 1, + Err(error) => stats.failed.push(PodcastSyncFailure { + podcast_id, + error: error.to_string(), + }), } } stats.push = self.push(records).await?; @@ -613,8 +743,8 @@ impl PodcastDht { /// normalized token of `query` ([`podcast_query_tokens`]). /// /// Hits are merged by `podcast_id` over publishers and ranked: exact - /// normalized title first, then more publishers. Partial results are - /// returned when peers fail. + /// normalized title first, then hits with a card under the origin feed + /// URL. Partial results are returned when peers fail. pub async fn search(&self, query: &str, limit: usize) -> Result> { let tokens = podcast_query_tokens(query); if tokens.is_empty() || limit == 0 { @@ -966,8 +1096,25 @@ impl PodcastDht { )) } PodcastDhtOperation::StoreBatch { records } => { - let entries = validate_store_batch(records, &self.network_id, now_ms())?; - let stored = self.database.store(entries, now_ms()).await?; + let entries = + validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?; + let nodes: Vec = lock(&self.routing) + .contacts() + .into_iter() + .map(|contact| contact.node_id) + .collect(); + let entries: Vec<_> = entries + .into_iter() + .filter_map(|(mut keys, record)| { + keys.retain(|key| is_responsible(&self.own_contact.node_id, &nodes, key)); + (!keys.is_empty()).then_some((keys, record)) + }) + .collect(); + let stored = if entries.is_empty() { + 0 + } else { + self.database.store(entries, now_ms()).await? + }; Ok(PodcastDhtResponse::success( Vec::new(), Vec::new(), @@ -1099,11 +1246,17 @@ impl PodcastDht { } } -/// Validates a store batch received from a peer: bounds, every record's -/// signature and lifetime, and that every key belongs to its record. +/// Validates a store batch received from `requester`: bounds, that every +/// record is published by the requester itself, every record's signature and +/// lifetime, and that every key belongs to its record. +/// +/// Replicas never forward records, so only a record's publisher stores it. +/// Nobody can therefore replay or inject another device's records, and +/// per-publisher quotas apply to the storing peer. fn validate_store_batch( entries: Vec, network_id: &NetworkId, + requester: EndpointId, now_ms: u64, ) -> Result, SignedPodcastRecord)>> { if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS { @@ -1117,6 +1270,11 @@ fn validate_store_batch( if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() { return Err(protocol_error("invalid podcast DHT store keys")); } + if entry.record.publisher() != requester { + return Err(protocol_error( + "podcast records must be stored by their publisher", + )); + } entry.record.verify_at(network_id, now_ms)?; let valid_keys = entry.record.dht_keys(); if keys.iter().any(|key| !valid_keys.contains(key)) { @@ -1128,6 +1286,18 @@ fn validate_store_batch( .collect() } +/// Whether a node is among the nodes that should store `key`: fewer than +/// [`PODCAST_STORE_RESPONSIBILITY_WINDOW`] known contacts are closer. +fn is_responsible(own: &NodeId, contacts: &[NodeId], key: &PodcastDhtKey) -> bool { + let own_distance = distance(own.as_bytes(), key.as_bytes()); + contacts + .iter() + .filter(|node| distance(node.as_bytes(), key.as_bytes()) < own_distance) + .take(PODCAST_STORE_RESPONSIBILITY_WINDOW) + .count() + < PODCAST_STORE_RESPONSIBILITY_WINDOW +} + fn validate_filter_tokens(tokens: &[String]) -> Result<()> { if tokens.len() > MAX_PODCAST_QUERY_TOKENS || tokens @@ -1139,7 +1309,8 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> { Ok(()) } -/// Adds verified records found under `key` to the per-owner winners. +/// Adds verified records found under `key` to the per-owner winners, keeping +/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners. fn merge_records( winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, records: Vec, @@ -1161,6 +1332,7 @@ fn merge_records( let owner = (record.podcast_id().clone(), record.publisher()); match winners.get(&owner) { Some(current) if compare_records(current, &record).is_ge() => {} + None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {} _ => { winners.insert(owner, record); } @@ -1168,6 +1340,42 @@ fn merge_records( } } +/// One in-flight inbound request of a peer; released on drop. +struct PeerSlot { + inflight: Arc>>, + peer: EndpointId, +} + +impl PeerSlot { + fn acquire( + inflight: &Arc>>, + peer: EndpointId, + ) -> Option { + let mut map = lock(inflight); + let count = map.entry(peer).or_default(); + if *count >= MAX_PODCAST_DHT_REQUESTS_PER_PEER { + return None; + } + *count += 1; + Some(Self { + inflight: Arc::clone(inflight), + peer, + }) + } +} + +impl Drop for PeerSlot { + fn drop(&mut self) { + let mut map = lock(&self.inflight); + if let Some(count) = map.get_mut(&self.peer) { + *count = count.saturating_sub(1); + if *count == 0 { + map.remove(&self.peer); + } + } + } +} + fn validate_contact( contact: &NodeContact, network_id: &NetworkId, diff --git a/crates/music-dht/src/podcast_dht/search.rs b/crates/music-dht/src/podcast_dht/search.rs index 8b9e3b9..2ab8781 100644 --- a/crates/music-dht/src/podcast_dht/search.rs +++ b/crates/music-dht/src/podcast_dht/search.rs @@ -8,15 +8,17 @@ use federation_net::EndpointId; use serde::{Deserialize, Serialize}; use crate::normalization::{normalize_name, tokenize}; -use crate::podcast::PodcastId; +use crate::podcast::{FeedUrl, PodcastId}; use crate::record::{ItemKind, LibraryItem}; use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records}; /// Maximum distinct query tokens used by a podcast text search. pub const MAX_PODCAST_QUERY_TOKENS: usize = 8; -/// Maximum publishers reported per hit. +/// Maximum publishers reported per hit and per claimed feed URL. pub const MAX_HIT_PUBLISHERS: usize = 64; +/// Maximum claimed moved feed URLs reported per hit. +pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8; /// Kind of a federated search hit. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -64,18 +66,35 @@ impl From for SearchHitKind { /// /// Cards of the same podcast published by several devices are merged by /// `podcast_id`. The hit carries no personal state of any publisher. +/// +/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every +/// other field is a claim of some publisher device, and device identities are +/// free, so neither the metadata nor the number of publishers is a trust +/// signal. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PodcastSearchHit { /// Podcast. pub podcast_id: PodcastId, - /// Representative card: the most common `feed_url` among the - /// publishers' current cards, then the most recently issued card, then - /// the greatest publisher key. + /// The URL `podcast_id` is derived from + /// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with + /// this URL; follow the feed's own HTTP redirects and + /// `itunes:new-feed-url` from there. + pub feed_url: FeedUrl, + /// Representative card for display: a card whose `feed_url` is the + /// origin URL when one exists, then the title most publishers agree on, + /// then the smallest publisher key. Its metadata is a publisher's claim. pub card: PodcastCard, /// Sorted distinct publisher devices with a current valid card, at most /// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers: - /// one user may publish from several devices. + /// one user may publish from several devices, and anyone can mint + /// devices. pub publishers: Vec, + /// Feed URLs other than [`Self::feed_url`] that publishers claim the + /// podcast moved to, most claimed first, at most + /// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this + /// podcast or record it as an alias unless the feed at + /// [`Self::feed_url`] itself redirects there. + pub claimed_feed_urls: Vec, } impl PodcastSearchHit { @@ -85,6 +104,16 @@ impl PodcastSearchHit { } } +/// An unverified moved-feed URL claimed by some publishers of a podcast. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClaimedFeedUrl { + /// Claimed current feed URL. + pub feed_url: FeedUrl, + /// Sorted distinct publisher devices claiming it, at most + /// [`MAX_HIT_PUBLISHERS`]. + pub publishers: Vec, +} + /// A music or podcast result of a federated search. /// /// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use @@ -184,37 +213,66 @@ pub(crate) fn aggregate_hits( by_podcast .into_iter() .map(|(podcast_id, cards)| { - let mut feed_votes: HashMap<&str, usize> = HashMap::new(); + let mut title_votes: HashMap = HashMap::new(); for card in &cards { - *feed_votes.entry(card.feed_url.as_str()).or_default() += 1; + *title_votes.entry(normalize_name(&card.title)).or_default() += 1; } + // Publisher-controlled times and revisions never break ties. let card = cards .iter() .max_by_key(|card| { ( - feed_votes[card.feed_url.as_str()], - card.issued_at_ms, - *card.publisher.as_bytes(), + card.feed_url_is_origin(), + title_votes[&normalize_name(&card.title)], + Reverse(*card.publisher.as_bytes()), ) }) .cloned() .expect("a group has at least one card"); - let mut publishers: Vec = cards.iter().map(|card| card.publisher).collect(); - publishers.sort_by_key(|publisher| *publisher.as_bytes()); - publishers.dedup(); - publishers.truncate(MAX_HIT_PUBLISHERS); + let mut claims: BTreeMap<&FeedUrl, Vec> = BTreeMap::new(); + for card in cards.iter().filter(|card| !card.feed_url_is_origin()) { + claims + .entry(&card.feed_url) + .or_default() + .push(card.publisher); + } + let mut claimed_feed_urls: Vec = claims + .into_iter() + .map(|(feed_url, publishers)| ClaimedFeedUrl { + feed_url: feed_url.clone(), + publishers: sorted_publishers(publishers), + }) + .collect(); + claimed_feed_urls.sort_by(|left, right| { + right + .publishers + .len() + .cmp(&left.publishers.len()) + .then_with(|| left.feed_url.cmp(&right.feed_url)) + }); + claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS); PodcastSearchHit { podcast_id, + feed_url: card.origin_feed_url().clone(), + publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()), card, - publishers, + claimed_feed_urls, } }) .collect() } +fn sorted_publishers(mut publishers: Vec) -> Vec { + publishers.sort_by_key(|publisher| *publisher.as_bytes()); + publishers.dedup(); + publishers.truncate(MAX_HIT_PUBLISHERS); + publishers +} + /// Keeps hits whose card is indexed under every query token and ranks -/// them: exact normalized title first, then more publishers, then title and -/// id. +/// them: exact normalized title first, then hits with a card published under +/// the origin feed URL, then title and id. The publisher count is not used: +/// anyone can inflate it with new device identities. pub(crate) fn rank_hits( mut hits: Vec, query: &str, @@ -229,7 +287,7 @@ pub(crate) fn rank_hits( let title = normalize_name(&hit.card.title); ( title != normalized_query, - Reverse(hit.publishers.len()), + !hit.card.feed_url_is_origin(), title, hit.podcast_id.clone(), ) diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs index 9f27104..acaedbc 100644 --- a/crates/music-dht/src/podcast_dht/store.rs +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -5,7 +5,7 @@ //! (like the similarity-routing cache). Every record read from it is decoded //! and verified again before it is used or served. -use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -17,32 +17,66 @@ use crate::error::{MusicDhtError, Result}; use crate::podcast::PodcastId; use crate::routing::{NodeContact, NodeId}; -use super::card::{ - PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere, -}; +use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records}; -/// `PRAGMA user_version` of the podcast-discovery cache schema. -pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1; -/// Maximum distinct `(podcast_id, publisher)` records cached under one key. -/// When full, the record expiring first is evicted if the new one outlives it. +/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1 +/// (pre-release) stored one payload per key; its replica table is dropped on +/// open because the cache is replaceable. Own publications are kept. +pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2; +/// Maximum records (cards and tombstones) indexed under one key. A full key +/// admits no new `(podcast_id, publisher)` until a record expires: existing +/// records are never evicted for newcomers, and their owners keep refreshing +/// them in place. pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; +/// Maximum tombstones newly indexed under one key. A tombstone is admitted +/// under a text key only in place of a record of the same +/// `(podcast_id, publisher)` already indexed there, so this budget applies to +/// the podcast-id key, where tombstones of unknown cards land. +pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32; +/// Maximum tombstones in one `FindValue` response; cards fill the rest. +pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32; +/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a +/// replica. +pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256; +/// Maximum `(podcast_id, publisher)` records in a replica cache. +pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096; +/// Maximum total encoded size of the records in a replica cache. +pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024; +/// Maximum persisted overlay contacts; the least recently seen are dropped. +pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024; const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS podcast_dht_records ( - dht_key BLOB NOT NULL, - podcast_id TEXT NOT NULL, - publisher TEXT NOT NULL, - tombstone INTEGER NOT NULL, - expires_at_ms INTEGER NOT NULL, - payload BLOB NOT NULL, - PRIMARY KEY (dht_key, podcast_id, publisher) + podcast_id TEXT NOT NULL, + publisher TEXT NOT NULL, + tombstone INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + payload_bytes INTEGER NOT NULL, + payload BLOB NOT NULL, + PRIMARY KEY (podcast_id, publisher) ); CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry ON podcast_dht_records(expires_at_ms); -CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner - ON podcast_dht_records(podcast_id, publisher); +CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher + ON podcast_dht_records(publisher); + +CREATE TABLE IF NOT EXISTS podcast_dht_keys ( + dht_key BLOB NOT NULL, + podcast_id TEXT NOT NULL, + publisher TEXT NOT NULL, + tombstone INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + first_stored_at_ms INTEGER NOT NULL, + PRIMARY KEY (dht_key, podcast_id, publisher) +); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner + ON podcast_dht_keys(podcast_id, publisher); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry + ON podcast_dht_keys(expires_at_ms); CREATE TABLE IF NOT EXISTS podcast_dht_peers ( peer_id TEXT PRIMARY KEY, @@ -60,6 +94,28 @@ CREATE TABLE IF NOT EXISTS podcast_dht_local ( ); "; +/// Admission limits of a replica cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StoreLimits { + pub per_key: usize, + pub tombstones_per_key: usize, + pub per_publisher: usize, + pub records: usize, + pub bytes: usize, +} + +impl Default for StoreLimits { + fn default() -> Self { + Self { + per_key: MAX_PODCAST_RECORDS_PER_KEY, + tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY, + per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER, + records: MAX_PODCAST_CACHE_RECORDS, + bytes: MAX_PODCAST_CACHE_BYTES, + } + } +} + /// This device's latest publication of one podcast. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LocalPublication { @@ -76,6 +132,7 @@ pub(crate) struct LocalPublication { pub(crate) struct PodcastDhtDatabase { conn: Arc>, network_id: NetworkId, + limits: StoreLimits, } impl std::fmt::Debug for PodcastDhtDatabase { @@ -88,6 +145,14 @@ impl std::fmt::Debug for PodcastDhtDatabase { impl PodcastDhtDatabase { pub async fn open(path: &Path, network_id: NetworkId) -> Result { + Self::open_with_limits(path, network_id, StoreLimits::default()).await + } + + pub async fn open_with_limits( + path: &Path, + network_id: NetworkId, + limits: StoreLimits, + ) -> Result { let path = path.to_path_buf(); let conn = tokio::task::spawn_blocking(move || -> Result { let conn = Connection::open(&path).map_err(|error| { @@ -101,6 +166,11 @@ impl PodcastDhtDatabase { "podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}" ))); } + if version == 1 { + // Replicas are replaceable; the old layout is dropped. + conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;") + .map_err(database_error)?; + } conn.execute_batch(SCHEMA).map_err(|error| { MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}")) })?; @@ -113,37 +183,39 @@ impl PodcastDhtDatabase { Ok(Self { conn: Arc::new(Mutex::new(conn)), network_id, + limits, }) } async fn call(&self, operation: F) -> Result where - F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result + Send + 'static, + F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result + Send + 'static, T: Send + 'static, { let conn = Arc::clone(&self.conn); let network_id = self.network_id; + let limits = self.limits; tokio::task::spawn_blocking(move || { let guard = lock(&conn); - operation(&guard, &network_id).map_err(database_error) + operation(&guard, &network_id, &limits).map_err(database_error) }) .await .map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))? } /// Stores records under the given keys. The caller has verified every - /// record and checked that each key belongs to it. Returns the number of - /// `(key, record)` rows written. + /// record, checked that each key belongs to it and that the sender is its + /// publisher. Returns the number of `(key, record)` placements written. pub async fn store( &self, entries: Vec<(Vec, SignedPodcastRecord)>, now_ms: u64, ) -> Result { - self.call(move |conn, network_id| { + self.call(move |conn, network_id, limits| { let transaction = conn.unchecked_transaction()?; let mut written = 0; for (keys, record) in entries { - written += store_one(&transaction, network_id, &keys, &record, now_ms)?; + written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?; } transaction.commit()?; Ok(written) @@ -151,9 +223,11 @@ impl PodcastDhtDatabase { .await } - /// Valid, unexpired records under `key`, newest expiry first. Cards must - /// contain every token in `filter_tokens`; tombstones are always - /// returned so stale replicas elsewhere can be overruled. + /// Valid, unexpired records under `key`: at most + /// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale + /// replicas elsewhere can be overruled) and cards containing every token + /// in `filter_tokens` up to `limit` records in all. Records indexed longest + /// come first. Cards precede tombstones. pub async fn records_by_key( &self, key: PodcastDhtKey, @@ -161,52 +235,70 @@ impl PodcastDhtDatabase { now_ms: u64, limit: usize, ) -> Result> { - self.call(move |conn, network_id| { + self.call(move |conn, network_id, _| { let mut statement = conn.prepare( - "SELECT payload FROM podcast_dht_records - WHERE dht_key = ?1 AND expires_at_ms > ?2 - ORDER BY expires_at_ms DESC, podcast_id, publisher", + "SELECT r.payload FROM podcast_dht_keys k + JOIN podcast_dht_records r + ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher + WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3 + ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher", )?; - let rows = statement.query_map( - params![key.as_bytes().as_slice(), clamp_i64(now_ms)], - |row| row.get::<_, Vec>(0), - )?; - let mut records = Vec::new(); - for row in rows { - let Some(record) = decode_valid(&row?, network_id, now_ms) else { - continue; - }; - if !record.belongs_under(&key) { - continue; + let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result> { + let mut records = Vec::new(); + if limit == 0 { + return Ok(records); } - let matches = record.card().is_none_or(|card| { - let tokens = card.index_tokens(); - filter_tokens.iter().all(|token| tokens.contains(token)) - }); - if matches { - records.push(record); - if records.len() == limit { - break; + let rows = statement.query_map( + params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)], + |row| row.get::<_, Vec>(0), + )?; + for row in rows { + let Some(record) = decode_valid(&row?, network_id, now_ms) else { + continue; + }; + if record.is_tombstone() != tombstone || !record.belongs_under(&key) { + continue; + } + let matches = record.card().is_none_or(|card| { + let tokens = card.index_tokens(); + filter_tokens.iter().all(|token| tokens.contains(token)) + }); + if matches { + records.push(record); + if records.len() == limit { + break; + } } } - } + Ok(records) + }; + let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?; + let mut records = select(false, limit - tombstones.len())?; + records.extend(tombstones); Ok(records) }) .await } + /// Removes expired records and their key placements. Returns the number + /// of records removed. pub async fn delete_expired(&self, now_ms: u64) -> Result { - self.call(move |conn, _| { + self.call(move |conn, _, _| { + let now = clamp_i64(now_ms); + conn.execute( + "DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1", + [now], + )?; conn.execute( "DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1", - [clamp_i64(now_ms)], + [now], ) }) .await } pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { conn.execute( "INSERT OR REPLACE INTO podcast_dht_peers (peer_id, node_id, ticket, last_seen_ms) @@ -218,13 +310,20 @@ impl PodcastDhtDatabase { clamp_i64(contact.last_seen_ms), ], )?; + conn.execute( + "DELETE FROM podcast_dht_peers WHERE peer_id IN ( + SELECT peer_id FROM podcast_dht_peers + ORDER BY last_seen_ms DESC, peer_id + LIMIT -1 OFFSET ?1)", + [MAX_PERSISTED_PODCAST_DHT_PEERS as i64], + )?; Ok(()) }) .await } pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { conn.execute( "DELETE FROM podcast_dht_peers WHERE peer_id = ?1", [peer.to_string()], @@ -235,7 +334,7 @@ impl PodcastDhtDatabase { } pub async fn load_peers(&self) -> Result> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { let mut statement = conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?; let rows = statement.query_map([], |row| { @@ -267,7 +366,7 @@ impl PodcastDhtDatabase { &self, podcast_id: PodcastId, ) -> Result> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { conn.query_row( "SELECT podcast_id, revision, spec, record FROM podcast_dht_local WHERE podcast_id = ?1", @@ -281,7 +380,7 @@ impl PodcastDhtDatabase { } pub async fn local_publications(&self) -> Result> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { let mut statement = conn.prepare( "SELECT podcast_id, revision, spec, record FROM podcast_dht_local ORDER BY podcast_id", @@ -303,7 +402,7 @@ impl PodcastDhtDatabase { publication: LocalPublication, now_ms: u64, ) -> Result<()> { - self.call(move |conn, _| { + self.call(move |conn, _, _| { let spec = publication .spec .as_ref() @@ -332,141 +431,235 @@ impl PodcastDhtDatabase { fn store_one( conn: &Connection, network_id: &NetworkId, + limits: &StoreLimits, keys: &[PodcastDhtKey], record: &SignedPodcastRecord, now_ms: u64, ) -> rusqlite::Result { let podcast_id = record.podcast_id().as_str().to_string(); let publisher = record.publisher().to_string(); + let now = clamp_i64(now_ms); - // Every cached version of this (podcast, publisher), re-verified. - let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new(); - let mut invalid_keys = Vec::new(); - { - let mut statement = conn.prepare( - "SELECT dht_key, payload FROM podcast_dht_records + // The cached record of this (podcast, publisher), re-verified. + let cached: Option<(Vec, i64)> = conn + .query_row( + "SELECT payload, payload_bytes FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2", - )?; - let rows = statement.query_map(params![podcast_id, publisher], |row| { - Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (key, payload) = row?; - let decoded = decode_valid(&payload, network_id, now_ms); - match (<[u8; 32]>::try_from(key.as_slice()), decoded) { - (Ok(key_bytes), Some(cached)) => { - existing.insert(key_bytes, cached); - } - _ => invalid_keys.push(key), + params![podcast_id, publisher], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + let (existing, existing_bytes) = match cached { + Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) { + Some(existing) => (Some(existing), bytes.max(0) as usize), + None => { + delete_owner(conn, &podcast_id, &publisher)?; + (None, 0) } + }, + None => { + // Stray placements without a record. + delete_owner(conn, &podcast_id, &publisher)?; + (None, 0) } - } - for key in invalid_keys { - conn.execute( - "DELETE FROM podcast_dht_records - WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", - params![key, podcast_id, publisher], - )?; - } + }; - // A tombstone or a newer revision cached under any key wins everywhere. - if existing - .values() - .any(|cached| supersedes_everywhere(cached, record)) + // Higher revision, tombstone at an equal revision, later republish... + let replace = match &existing { + Some(existing) => match compare_records(existing, record) { + std::cmp::Ordering::Greater => return Ok(0), + std::cmp::Ordering::Equal => false, + std::cmp::Ordering::Less => true, + }, + None => true, + }; + let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; + if replace + && !admits_record( + conn, + limits, + &publisher, + existing.is_none(), + existing_bytes, + payload.len(), + now, + )? { return Ok(0); } - let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; + let mut indexed: HashSet<[u8; 32]> = HashSet::new(); + { + let mut statement = conn.prepare( + "SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2", + )?; + let rows = statement.query_map(params![podcast_id, publisher], |row| { + row.get::<_, Vec>(0) + })?; + for row in rows { + if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) { + indexed.insert(key); + } + } + } + let mut written = 0; + if replace { + // Placements the new version no longer belongs under are dropped (a + // renamed card, or a tombstone that does not list the key); the others + // now serve the new version. + let belongs: HashSet<[u8; 32]> = record + .dht_keys() + .iter() + .map(|key| *key.as_bytes()) + .collect(); + let stale: Vec<[u8; 32]> = indexed + .iter() + .filter(|key| !belongs.contains(*key)) + .copied() + .collect(); + for key in stale { + conn.execute( + "DELETE FROM podcast_dht_keys + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_slice(), podcast_id, publisher], + )?; + indexed.remove(&key); + } + written += conn.execute( + "UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4 + WHERE podcast_id = ?1 AND publisher = ?2", + params![ + podcast_id, + publisher, + record.is_tombstone(), + clamp_i64(record.expires_at_ms()), + ], + )?; + } + + // New placements. A tombstone only takes a new place under the podcast-id + // key: under a text key it may only replace a record of its owner, so + // nobody can index tombstones under someone else's search terms. + let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id()); for key in keys { - if let Some(cached) = existing.get(key.as_bytes()) - && compare_records(cached, record).is_ge() - { + if indexed.contains(key.as_bytes()) { continue; } - if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? { + if record.is_tombstone() && *key != id_key { + continue; + } + if !admits_placement(conn, limits, key, record.is_tombstone(), now)? { continue; } conn.execute( - "INSERT INTO podcast_dht_records - (dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET - tombstone = excluded.tombstone, - expires_at_ms = excluded.expires_at_ms, - payload = excluded.payload", + "INSERT INTO podcast_dht_keys + (dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", params![ key.as_bytes().as_slice(), podcast_id, publisher, record.is_tombstone(), clamp_i64(record.expires_at_ms()), - payload, + now, ], )?; + indexed.insert(*key.as_bytes()); written += 1; } - // Drop versions the new record makes obsolete under other keys too. - if written > 0 { - for (key, cached) in &existing { - if !keys.iter().any(|stored| stored.as_bytes() == key) - && supersedes_everywhere(record, cached) - { - conn.execute( - "DELETE FROM podcast_dht_records - WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", - params![key.as_slice(), podcast_id, publisher], - )?; - } - } + if indexed.is_empty() { + delete_owner(conn, &podcast_id, &publisher)?; + return Ok(0); + } + if replace { + conn.execute( + "INSERT INTO podcast_dht_records + (podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(podcast_id, publisher) DO UPDATE SET + tombstone = excluded.tombstone, + expires_at_ms = excluded.expires_at_ms, + payload_bytes = excluded.payload_bytes, + payload = excluded.payload", + params![ + podcast_id, + publisher, + record.is_tombstone(), + clamp_i64(record.expires_at_ms()), + payload.len() as i64, + payload, + ], + )?; } Ok(written) } -/// Ensures there is room for one more record under `key`. -fn make_room( +/// Whether the cache may hold the new version of a record: per-publisher and +/// global record counts for a new owner, and the global byte budget. +fn admits_record( conn: &Connection, - key: &PodcastDhtKey, - record: &SignedPodcastRecord, - now_ms: u64, + limits: &StoreLimits, + publisher: &str, + new_owner: bool, + previous_bytes: usize, + new_bytes: usize, + now: i64, ) -> rusqlite::Result { - conn.execute( - "DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2", - params![key.as_bytes().as_slice(), clamp_i64(now_ms)], + let (records, bytes): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records + WHERE expires_at_ms > ?1", + [now], + |row| Ok((row.get(0)?, row.get(1)?)), )?; - let count: i64 = conn.query_row( - "SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1", - [key.as_bytes().as_slice()], - |row| row.get(0), - )?; - if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY { + let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); + if bytes.saturating_add(new_bytes) > limits.bytes { + return Ok(false); + } + if !new_owner { return Ok(true); } - let victim: Option<(String, String, i64)> = conn - .query_row( - "SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records - WHERE dht_key = ?1 - ORDER BY expires_at_ms ASC, podcast_id, publisher - LIMIT 1", - [key.as_bytes().as_slice()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .optional()?; - match victim { - Some((podcast_id, publisher, expires_at_ms)) - if expires_at_ms < clamp_i64(record.expires_at_ms()) => - { - conn.execute( - "DELETE FROM podcast_dht_records - WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", - params![key.as_bytes().as_slice(), podcast_id, publisher], - )?; - Ok(true) - } - _ => Ok(false), + if records.max(0) as usize >= limits.records { + return Ok(false); } + let own: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", + params![publisher, now], + |row| row.get(0), + )?; + Ok((own.max(0) as usize) < limits.per_publisher) +} + +/// Whether `key` has room for one more placement. Existing placements are +/// never evicted. +fn admits_placement( + conn: &Connection, + limits: &StoreLimits, + key: &PodcastDhtKey, + tombstone: bool, + now: i64, +) -> rusqlite::Result { + let (total, tombstones): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys + WHERE dht_key = ?1 AND expires_at_ms > ?2", + params![key.as_bytes().as_slice(), now], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + Ok((total.max(0) as usize) < limits.per_key + && (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key)) +} + +fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> { + conn.execute( + "DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2", + params![podcast_id, publisher], + )?; + conn.execute( + "DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2", + params![podcast_id, publisher], + )?; + Ok(()) } fn decode_valid( diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs index faf6159..034f6c3 100644 --- a/crates/music-dht/src/podcast_dht/tests.rs +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -3,7 +3,8 @@ use federation_net::{NetworkId, SecretKey}; -use super::card::supersedes_everywhere; +use crate::routing::NodeId; + use super::search::{aggregate_hits, rank_hits}; use super::*; use crate::podcast::{FeedUrl, PodcastMeta}; @@ -362,10 +363,6 @@ fn record_order_is_total_and_tombstones_win_ties() { compare_records(&edited, &older), compare_records(&older, &edited).reverse() ); - assert!(supersedes_everywhere(&grave, &older)); - assert!(supersedes_everywhere(&newer, &older)); - assert!(!supersedes_everywhere(&refresh, &older)); - assert!(!supersedes_everywhere(&older, &grave)); } // --------------------------------------------------------------------- cache @@ -437,7 +434,7 @@ async fn cache_converges_regardless_of_delivery_order() { } #[tokio::test] -async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() { +async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key_it_holds() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; let key = secret(1); @@ -451,7 +448,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() vec![active.clone()] ); - // A tombstone stored only under the id key still removes the card under + // A tombstone stored only under the id key still replaces the card under // the text keys of this node. let grave = tombstone(&key, 2, NOW); let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id()); @@ -462,7 +459,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() db.records_by_key(token_key, vec!["rust".into()], NOW, 10) .await .unwrap(), - Vec::::new() + vec![grave.clone()] ); // An equal-revision card arriving later loses, under any key. let same = card(&key, 2); @@ -564,43 +561,247 @@ async fn cache_read_path_filters_tokens_expiry_and_corruption() { .unwrap() .is_empty() ); - assert_eq!( - db.delete_expired(NOW + ttl).await.unwrap(), - MAX_RECORD_DHT_KEYS.min(record.dht_keys().len()) - ); + assert_eq!(db.delete_expired(NOW + ttl).await.unwrap(), 1); +} + +fn numbered_secret(index: usize) -> SecretKey { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes()); + bytes[31] = 0x5a; + SecretKey::from_bytes(&bytes) +} + +async fn open_db_with( + temp: &tempfile::TempDir, + limits: store::StoreLimits, +) -> store::PodcastDhtDatabase { + store::PodcastDhtDatabase::open_with_limits( + &temp.path().join("podcast-dht.sqlite3"), + network(), + limits, + ) + .await + .unwrap() } #[tokio::test] -async fn cache_bounds_records_per_key() { +async fn cache_full_key_keeps_its_records_and_refuses_newcomers() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; let token_key = PodcastDhtKey::token(&network(), "rust"); let mut entries = Vec::new(); for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { - let key = SecretKey::from_bytes(&{ - let mut bytes = [0u8; 32]; - bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes()); - bytes - }); - // Later publishers expire later, so the first one is evicted. - let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64); + // Later publishers expire later; that buys them nothing. + let record = card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + NOW + index as u64, + ); entries.push((vec![token_key], record)); } let first = entries[0].1.clone(); - for chunk in entries.chunks(64) { - db.store(chunk.to_vec(), NOW).await.unwrap(); + let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone(); + // The first publisher arrived first; the rest a moment later. + db.store(entries[..1].to_vec(), NOW).await.unwrap(); + for chunk in entries[1..].chunks(64) { + db.store(chunk.to_vec(), NOW + 1).await.unwrap(); } let cached = db .records_by_key(token_key, Vec::new(), NOW, usize::MAX) .await .unwrap(); assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); - assert!(!cached.contains(&first)); - // A record expiring before every cached one is not admitted. - assert_eq!( - db.store(vec![(vec![token_key], first)], NOW).await.unwrap(), - 0 + assert!(cached.contains(&first)); + assert!(!cached.contains(&last)); + assert_eq!(cached[0], first, "records indexed longest come first"); + // The first publisher keeps refreshing its place. + let refresh = card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(0), + 1, + NOW + 1_000, ); + assert_eq!( + db.store(vec![(vec![token_key], refresh)], NOW) + .await + .unwrap(), + 1 + ); +} + +#[tokio::test] +async fn sybil_tombstones_cannot_suppress_a_card() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let victim = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id()); + let rust = PodcastDhtKey::token(&network(), "rust"); + db.store(vec![entry(&victim)], NOW).await.unwrap(); + + // 256 throwaway identities sign tombstones of the victim's podcast that + // list the victim card's text keys, and issue them as late as allowed. + let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64; + let sybils: Vec<_> = (0..256) + .map(|index| { + let key = numbered_secret(index); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + victim.podcast_id().clone(), + u64::MAX, + NOW + skew, + victim.dht_keys(), + ); + let record = + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap(); + (record.dht_keys(), record) + }) + .collect(); + let mut written = 0; + for chunk in sybils.chunks(MAX_PODCAST_STORE_RECORDS) { + written += db.store(chunk.to_vec(), NOW).await.unwrap(); + } + assert_eq!( + written, MAX_PODCAST_TOMBSTONES_PER_KEY, + "only the id key's tombstone budget is used" + ); + + let by_id = db + .records_by_key(id_key, Vec::new(), NOW, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(by_id.contains(&victim)); + assert_eq!( + by_id.iter().filter(|record| record.is_tombstone()).count(), + MAX_PODCAST_RESPONSE_TOMBSTONES + ); + assert_eq!( + db.records_by_key(rust, vec!["rust".into()], NOW, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(), + vec![victim.clone()], + "no tombstone is indexed under a text key without its owner's card" + ); + // Eleven hours later the victim's republish still lands. + let later = NOW + 11 * 3_600_000; + let republished = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later); + assert_eq!( + db.store(vec![entry(&republished)], later).await.unwrap(), + republished.dht_keys().len() + ); + // Sybil cards issued in the future cannot displace it either. + let sybil_cards: Vec<_> = (300..600) + .map(|index| { + entry(&card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + u64::MAX, + later + skew, + )) + }) + .collect(); + for chunk in sybil_cards.chunks(MAX_PODCAST_STORE_RECORDS) { + db.store(chunk.to_vec(), later).await.unwrap(); + } + for key in [id_key, rust] { + let records = db + .records_by_key(key, vec![], later, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(records.contains(&republished)); + assert!(records.len() <= MAX_PODCAST_DHT_RECORDS); + } +} + +#[tokio::test] +async fn cache_enforces_publisher_record_and_byte_quotas() { + let limits = store::StoreLimits { + per_key: 100, + tombstones_per_key: 10, + per_publisher: 3, + records: 5, + bytes: usize::MAX, + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + let show = |index: usize, key: &SecretKey| { + entry(&card_with( + spec(&format!("show-{index}.xml"), "Rust Radio"), + key, + 1, + NOW, + )) + }; + let heavy = secret(1); + let stored: Vec = { + let mut out = Vec::new(); + for index in 0..4 { + out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap()); + } + out + }; + assert!(stored[..3].iter().all(|written| *written > 0)); + assert_eq!( + stored[3], 0, + "the fourth podcast of one publisher is refused" + ); + // A refresh of an admitted record is always accepted. + let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1); + assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0); + assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0); + assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0); + assert_eq!( + db.store(vec![show(12, &secret(4))], NOW).await.unwrap(), + 0, + "the cache holds at most five records" + ); + + let one = card(&secret(1), 1); + let limits = store::StoreLimits { + bytes: one.encoded_len() * 2, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + assert!( + db.store(vec![entry(&card(&secret(1), 1))], NOW) + .await + .unwrap() + > 0 + ); + assert!( + db.store(vec![entry(&card(&secret(2), 1))], NOW) + .await + .unwrap() + > 0 + ); + assert_eq!( + db.store(vec![entry(&card(&secret(3), 1))], NOW) + .await + .unwrap(), + 0, + "the byte budget is full" + ); +} + +#[tokio::test] +async fn cache_stores_each_payload_once() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let many: Vec = (0..50).map(|index| format!("w{index}")).collect(); + let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW); + assert_eq!( + db.store(vec![entry(&record)], NOW).await.unwrap(), + MAX_RECORD_DHT_KEYS + ); + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let payloads: i64 = conn + .query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(payloads, 1); } #[tokio::test] @@ -634,6 +835,25 @@ async fn cache_refuses_a_newer_schema_and_keeps_local_publications() { assert_eq!(local[0].record, record); drop(db); + // A pre-release version-1 replica table is dropped; own publications + // survive. + { + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute_batch( + "DROP TABLE podcast_dht_records; + CREATE TABLE podcast_dht_records (dht_key BLOB, payload BLOB); + INSERT INTO podcast_dht_records VALUES (x'00', x'00'); + PRAGMA user_version = 1;", + ) + .unwrap(); + } + let db = store::PodcastDhtDatabase::open(&path, network()) + .await + .unwrap(); + assert_eq!(db.local_publications().await.unwrap().len(), 1); + assert!(db.store(vec![entry(&record)], NOW).await.unwrap() > 0); + drop(db); + rusqlite::Connection::open(&path) .unwrap() .pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1) @@ -655,12 +875,14 @@ fn store_batches_are_bounded_and_keys_must_belong() { keys: record.dht_keys(), record: record.clone(), }; - validate_store_batch(vec![good.clone()], &network(), NOW).unwrap(); - assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err()); + let publisher = key.public(); + validate_store_batch(vec![good.clone()], &network(), publisher, NOW).unwrap(); + assert!(validate_store_batch(Vec::new(), &network(), publisher, NOW).is_err()); assert!( validate_store_batch( vec![good.clone(); MAX_PODCAST_STORE_RECORDS], &network(), + publisher, NOW ) .is_ok() @@ -669,19 +891,87 @@ fn store_batches_are_bounded_and_keys_must_belong() { validate_store_batch( vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1], &network(), + publisher, NOW ) .is_err() ); let mut foreign = good.clone(); foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")]; - assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err()); + assert!(validate_store_batch(vec![foreign], &network(), publisher, NOW).is_err()); let mut duplicated = good.clone(); duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]]; - assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err()); - let mut forged = good; + assert!(validate_store_batch(vec![duplicated], &network(), publisher, NOW).is_err()); + let mut forged = good.clone(); resign(&mut forged.record, &secret(2)); - assert!(validate_store_batch(vec![forged], &network(), NOW).is_err()); + assert!(validate_store_batch(vec![forged], &network(), publisher, NOW).is_err()); + // A valid record of another publisher may not be stored (or replayed) + // by anyone else. + let error = validate_store_batch(vec![good], &network(), secret(2).public(), NOW).unwrap_err(); + assert!(error.to_string().contains("publisher"), "{error}"); +} + +#[test] +fn replicas_only_accept_keys_they_are_close_to() { + let own = NodeId::from_bytes([0; 32]); + let key = PodcastDhtKey::from_bytes([0; 32]); + let far: Vec = (0..100) + .map(|index| NodeId::from_bytes([0x80 | index as u8; 32])) + .collect(); + assert!(is_responsible(&own, &far, &key)); + let near: Vec = (1..=PODCAST_STORE_RESPONSIBILITY_WINDOW) + .map(|index| { + let mut bytes = [0; 32]; + bytes[31] = index as u8; + NodeId::from_bytes(bytes) + }) + .collect(); + let own = NodeId::from_bytes([0xff; 32]); + assert!(!is_responsible(&own, &near, &key)); + assert!(is_responsible(&own, &near[1..], &key)); +} + +#[test] +fn inbound_requests_are_limited_per_peer() { + let inflight = Arc::new(Mutex::new(HashMap::new())); + let peer = secret(1).public(); + let slots: Vec<_> = (0..MAX_PODCAST_DHT_REQUESTS_PER_PEER) + .map(|_| PeerSlot::acquire(&inflight, peer).unwrap()) + .collect(); + assert!(PeerSlot::acquire(&inflight, peer).is_none()); + assert!(PeerSlot::acquire(&inflight, secret(2).public()).is_some()); + drop(slots); + assert!(PeerSlot::acquire(&inflight, peer).is_some()); + assert!(lock(&inflight).is_empty()); +} + +#[test] +fn lookups_keep_a_bounded_number_of_owners() { + let mut winners = HashMap::new(); + let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id()); + let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS) + .map(|index| { + card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + now_ms(), + ) + }) + .collect(); + let first = records[0].clone(); + merge_records(&mut winners, records, &network(), &id_key, &[]); + assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS); + // A newer record of a known owner still replaces its winner. + let newer = card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(0), + 2, + now_ms(), + ); + merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]); + let owner = (first.podcast_id().clone(), first.publisher()); + assert_eq!(winners[&owner], newer); } #[test] @@ -763,18 +1053,27 @@ fn music(name: &str, kind: ItemKind) -> LibraryItem { } } +fn moved_spec(path: &str, origin: &str, title: &str) -> PodcastCardSpec { + let mut moved = spec(path, title); + moved.id_feed_url = Some(feed(origin)); + moved.podcast_id = PodcastId::from_feed_url(&feed(origin)); + moved +} + #[test] fn hits_merge_publishers_and_drop_tombstoned_ones() { let a = secret(1); let b = secret(2); let c = secret(3); - let mut moved = spec("moved.xml", "Rust Radio"); - moved.id_feed_url = Some(feed("show.xml")); - moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml")); let records = vec![ card(&a, 1), card(&a, 2), - card_with(moved, &b, 1, NOW + 10), + card_with( + moved_spec("moved.xml", "show.xml", "Rust Radio"), + &b, + 1, + NOW + 10, + ), card(&c, 1), tombstone(&c, 2, NOW), card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW), @@ -790,8 +1089,81 @@ fn hits_merge_publishers_and_drop_tombstoned_ones() { let mut expected = vec![a.public(), b.public()]; expected.sort_by_key(|key| *key.as_bytes()); assert_eq!(hit.publishers, expected); - // Equal votes per feed URL: the most recently issued card represents. - assert_eq!(hit.card.publisher, b.public()); + // The card under the origin URL represents, whoever issued later. + assert_eq!(hit.card.publisher, a.public()); + assert_eq!(hit.feed_url, feed("show.xml")); + assert_eq!( + hit.claimed_feed_urls, + vec![ClaimedFeedUrl { + feed_url: feed("moved.xml"), + publishers: vec![b.public()], + }] + ); +} + +#[test] +fn sybil_cards_cannot_redirect_an_existing_podcast() { + let genuine = card(&secret(1), 1); + let id = genuine.podcast_id().clone(); + let evil_feed = FeedUrl::parse("https://evil.example/feed.xml").unwrap(); + let hijack = |seed: usize| { + let mut spec = spec("show.xml", "Rust Radio (official)"); + spec.id_feed_url = Some(spec.feed_url.clone()); + spec.feed_url = evil_feed.clone(); + spec.validate().unwrap(); + assert_eq!(spec.podcast_id, id, "a hijack card claims the genuine id"); + let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64; + card_with(spec, &numbered_secret(seed), u64::MAX, NOW + skew) + }; + let mut records = vec![genuine.clone()]; + records.extend((0..10).map(hijack)); + let hit = aggregate_hits(records).remove(0); + assert_eq!( + hit.feed_url, + feed("show.xml"), + "only the origin URL is offered" + ); + assert_eq!( + hit.card, + *genuine.card().unwrap(), + "the origin card represents" + ); + assert_eq!(hit.claimed_feed_urls.len(), 1); + assert_eq!(hit.claimed_feed_urls[0].feed_url, evil_feed); + assert_eq!(hit.claimed_feed_urls[0].publishers.len(), 10); + + // Without any origin card the hit still offers the origin URL, never the + // claimed one, and ranks below podcasts that have an origin card. + let only_claims = aggregate_hits((0..3).map(hijack)).remove(0); + assert_eq!(only_claims.feed_url, feed("show.xml")); + assert!(!only_claims.card.feed_url_is_origin()); + let honest = aggregate_hits(vec![card_with( + spec("b.xml", "Rust Radio Weekly"), + &secret(2), + 1, + NOW, + )]) + .remove(0); + let ranked = rank_hits( + vec![only_claims.clone(), honest.clone()], + "rust", + &podcast_query_tokens("rust"), + ); + assert_eq!(ranked[0].podcast_id, honest.podcast_id); + // Publisher counts do not rank. + let crowded = aggregate_hits( + (20..60).map(|seed| card_with(spec("c.xml", "Rust Zone"), &numbered_secret(seed), 1, NOW)), + ) + .remove(0); + let ranked = rank_hits( + vec![crowded, honest.clone()], + "rust", + &podcast_query_tokens("rust"), + ); + assert_eq!( + ranked[0].podcast_id, honest.podcast_id, + "ranked by title, not crowd" + ); } #[test] diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs index 548f869..b3ba983 100644 --- a/crates/music-dht/tests/podcast_dht.rs +++ b/crates/music-dht/tests/podcast_dht.rs @@ -1,15 +1,23 @@ //! Integration test: public podcast discovery on three real loopback iroh //! nodes. //! -//! * A publishes a podcast its user shared; B is one of A's trusted devices -//! (in this test merely another overlay node, then offline); C is an -//! unrelated peer outside A's trusted-device group, with no device-sync -//! pairing. -//! * C finds the card by text and by id; a podcast A never shared is never -//! discoverable; after A unpublishes, the tombstone hides the card from C. -//! * Forged, tampered, replayed, malformed, oversized and wrong-version -//! requests are rejected by a real peer. -//! * D, a music-DHT peer without the overlay, is never added to the overlay. +//! * A publishes a podcast its user shared. B is another device of the same +//! user: it applies the same merged registers (as device sync would deliver +//! them) and publishes the same podcast under its own key. C is an +//! unrelated peer outside the user's trusted-device group, with no +//! device-sync pairing. +//! * C finds the podcast by text and by id, merged over publishers {A, B}; +//! once B unshares and goes offline, C still finds A's card, so B is +//! irrelevant to discovery. A podcast the user never shared (held by both +//! A and B) is never discoverable; after A unpublishes, the tombstone hides +//! the card from C. +//! * Forged, foreign, tampered, replayed, malformed, oversized and +//! wrong-version requests are rejected by a real peer. +//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but +//! never answers on the overlay ALPN, so it never becomes an overlay contact +//! and is never asked for records or to store them. +//! * `sync_shares` stays fail-safe for revocation and holds back URLs that +//! look private (single node). use std::path::Path; use std::sync::Arc; @@ -214,14 +222,15 @@ async fn scenario() { (a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(()) }) .await; - // D never answers on the overlay ALPN, so it never becomes an overlay - // contact and is never queried or asked to store records. + // D is dialed by bootstrap but never answers on the overlay ALPN, so it + // never becomes an overlay contact: it is never asked for records or to + // store them. a.dht.bootstrap().await.unwrap(); assert_eq!(a.dht.known_peers(), 2); // A capability manifest without podcast_dht evicts a peer; one with it // lets the peer be learned again. - let manifest = CapabilityManifest::frid("furumi", "test"); + let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht(); a.dht .note_peer_capabilities( c.service.endpoint_id(), @@ -260,11 +269,13 @@ async fn scenario() { subscription: shared.clone(), share: Some(share(&shared, true, 2)), card: None, + privacy_confirmed: false, }, PodcastShareState { subscription: private.clone(), share: None, card: None, + privacy_confirmed: false, }, ]; let stats = a.dht.sync_shares(states.clone()).await.unwrap(); @@ -294,7 +305,54 @@ async fn scenario() { assert_eq!(published.len(), 1); let first_card = published[0].record.clone(); - // B (A's trusted device) is not needed for discovery. + // B, the same user's other device, holds the same merged registers and + // publishes the same podcast under its own key; the private podcast + // stays private there too. + let b_states = vec![ + PodcastShareState { + subscription: shared.clone(), + share: Some(share(&shared, true, 2)), + card: None, + privacy_confirmed: false, + }, + PodcastShareState { + subscription: private.clone(), + share: None, + card: None, + privacy_confirmed: false, + }, + ]; + let stats = b.dht.sync_shares(b_states).await.unwrap(); + assert_eq!((stats.published, stats.unpublished), (1, 0)); + let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()]; + both.sort_by_key(|key| *key.as_bytes()); + let hits = wait_for("C sees both publishers", async || { + let hits = search(&c, "rust radio").await; + (hits.len() == 1 && hits[0].publishers == both).then_some(hits) + }) + .await; + assert_eq!(hits[0].feed_url, shared.podcast.feed_url); + assert!(hits[0].claimed_feed_urls.is_empty()); + + // B unshares (for example a stale register on B) and goes offline: B is + // not needed for discovery, A's card is still found. + let stats = b + .dht + .sync_shares(vec![PodcastShareState { + subscription: shared.clone(), + share: Some(share(&shared, false, 3)), + card: None, + privacy_confirmed: false, + }]) + .await + .unwrap(); + assert_eq!(stats.unpublished, 1); + let a_only = vec![a.service.endpoint_id()]; + wait_for("B's tombstone reaches C's lookup", async || { + let hits = search(&c, "rust radio").await; + (hits.len() == 1 && hits[0].publishers == a_only).then_some(()) + }) + .await; b.stop().await; let hits = wait_for("C finds the card by text", async || { @@ -305,6 +363,7 @@ async fn scenario() { assert_eq!(hits.len(), 1); assert_eq!(hits[0].podcast_id, shared.podcast_id); assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]); + assert_eq!(hits[0].feed_url, shared.podcast.feed_url); assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url); assert_eq!(hits[0].card.title, "Rust Radio Weekly"); assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens"); @@ -350,7 +409,11 @@ async fn scenario() { let response = store_request(&c, &a, forged).await; assert!(!response.ok && response.stored == 0, "{response:?}"); - // Tampered: A's genuine card with a different feed URL. + // Foreign: A's genuine card stored by C. Only publishers store records. + let response = store_request(&c, &a, first_card.clone()).await; + assert!(!response.ok && response.stored == 0, "{response:?}"); + + // Tampered: A's genuine card with a different description. let mut tampered = first_card.clone(); if let PodcastRecordBody::Card(card) = &mut tampered.body { card.description = Some("Now with malware".into()); @@ -388,6 +451,7 @@ async fn scenario() { subscription: shared.clone(), share: Some(share(&shared, false, 3)), card: None, + privacy_confirmed: false, }]; let stats = a.dht.sync_shares(unshare).await.unwrap(); assert_eq!(stats.unpublished, 1); @@ -407,10 +471,12 @@ async fn scenario() { .is_none() ); - // Replaying the old card, even to the node that stores the tombstone, - // does not bring it back. - let response = store_request(&c, &a, first_card).await; + // A replaying its own old card to a replica that stores the tombstone + // does not bring it back; nobody else may replay it at all. + let response = store_request(&a, &c, first_card.clone()).await; assert!(response.ok && response.stored == 0, "{response:?}"); + let response = store_request(&c, &a, first_card).await; + assert!(!response.ok, "{response:?}"); assert!(search(&c, "rust radio").await.is_empty()); assert!(search(&a, "rust radio").await.is_empty()); @@ -419,3 +485,84 @@ async fn scenario() { d_events.abort(); d.shutdown().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sync_shares_is_fail_safe_and_holds_back_private_urls() { + tokio::time::timeout(TEST_TIMEOUT, single_node_scenario()) + .await + .unwrap(); +} + +async fn single_node_scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-sync-shares"); + let a = Node::start(&temp.path().join("a"), network).await; + let state = + |subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState { + subscription: subscription.clone(), + share: Some(share(subscription, shared, generation)), + card: None, + privacy_confirmed: false, + }; + let first = subscription("first.xml", "First Show"); + let second = subscription("second.xml", "Second Show"); + let stats = a + .dht + .sync_shares(vec![state(&first, true, 2), state(&second, true, 2)]) + .await + .unwrap(); + assert_eq!(stats.published, 2); + assert!(stats.failed.is_empty() && stats.held_back.is_empty()); + + // In one call the user unshares the first podcast while the second + // one's card cannot be built (the URL its id derives from is unknown). + let mut broken = second.clone(); + broken.podcast.feed_url = feed("elsewhere.xml"); + let stats = a + .dht + .sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)]) + .await + .unwrap(); + assert_eq!(stats.failed.len(), 1); + assert_eq!(stats.failed[0].podcast_id, second.podcast_id); + assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast"); + let published = a.dht.published().await.unwrap(); + assert!(published.iter().all(|publication| !publication.active)); + assert!( + published + .iter() + .all(|publication| publication.record.is_tombstone()) + ); + // Republishing re-sends tombstones only; nothing comes back. + a.dht.republish().await.unwrap(); + assert!(search(&a, "first show").await.is_empty()); + assert!(search(&a, "second show").await.is_empty()); + + // A feed URL that looks private is held back until the user confirms. + let mut member = subscription("member.xml?token=abc", "Member Show"); + member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url); + let stats = a + .dht + .sync_shares(vec![state(&member, true, 2)]) + .await + .unwrap(); + assert_eq!(stats.published, 0); + assert_eq!(stats.held_back.len(), 1); + assert_eq!(stats.held_back[0].podcast_id, member.podcast_id); + assert!(!stats.held_back[0].warnings.is_empty()); + assert!(search(&a, "member show").await.is_empty()); + let mut confirmed = state(&member, true, 2); + confirmed.privacy_confirmed = true; + let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap(); + assert_eq!(stats.published, 1); + assert_eq!(search(&a, "member show").await.len(), 1); + // Once published with the user's consent, the same URLs are kept. + let stats = a + .dht + .sync_shares(vec![state(&member, true, 2)]) + .await + .unwrap(); + assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0)); + + a.stop().await; +} -- 2.54.0 From 5a4d418d2b3585a990c3174dd1c31607a38d38c1 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 17:11:50 +0200 Subject: [PATCH 5/9] Document podcast discovery wire protocol and replica limits Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 29 ++++-- CHANGELOG.md | 24 +++-- PODCASTS.md | 206 ++++++++++++++++++++++++++++--------- crates/music-dht/README.md | 30 ++++-- 4 files changed, 215 insertions(+), 74 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 23a53ef..d508928 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -243,16 +243,25 @@ device publishes a card only for podcasts whose merged registers satisfy nothing is sent for a podcast the device never published. A card is the public projection of a podcast (feed URL and channel metadata, no personal state), signed by the device's iroh identity over a domain-separated -canonical encoding and bound to its id by the URL the id hashes from. Cards -live under the podcast-id key and one key per normalized title/author token; -unpublishing writes a signed tombstone of the next revision that supersedes -lower and equal revisions under every key and outlives every older card. -Records use the music DHT's active/tombstone TTLs and republish cadence, are -verified on every receive and cache read, and are kept in a separate, -replaceable SQLite cache. Search filters on every replica by all query tokens, -merges cards of one podcast across publisher devices, and returns typed -`podcast` hits that clients merge with music results. Capability data only -helps skip peers; it never authorizes. See the "Public discovery" section of +canonical encoding. Only the origin URL the id hashes from is bound to the +podcast; a moved `feed_url` is an unverified publisher claim that search never +offers as the podcast's feed. Cards live under the podcast-id key and one key +per normalized title/author token; unpublishing writes a signed tombstone of +the next revision that supersedes lower and equal revisions on replicas that +hold it, for the keys it lists, and outlives every older card. Only a record's +publisher may store it; replicas keep one payload per publisher and podcast, +admit placements only for keys they are close to, within per-key, +per-publisher and total budgets, and never evict existing records for +newcomers (tombstones cannot displace cards). Records use the music DHT's +active/tombstone TTLs and republish cadence, are verified on every receive and +cache read, and are kept in a separate, replaceable SQLite cache. Search +filters on every replica by all query tokens, merges cards of one podcast +across publisher devices, and returns typed `podcast` hits that clients merge +with music results; publisher counts are not a trust signal. The publisher key +is the device's transport identity, so a card is linkable to the device's +network address and music-DHT records. Applications advertise the +`podcast_dht` capability only when they run the overlay; capability data only +helps skip peers, it never authorizes. See the "Public discovery" section of [PODCASTS.md](PODCASTS.md). Music discovery identifies an owner and content; it does not force all domain diff --git a/CHANGELOG.md b/CHANGELOG.md index cb64153..4b14174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,27 @@ - `music-dht 0.6.0`: new `music_dht::podcast_dht` overlay for public podcast discovery on its own ALPN `furumi-fd/podcast-dht/1` (version 1) and its own - SQLite cache: signed `PodcastCard`s and `PodcastTombstone`s + SQLite cache (schema 2): signed `PodcastCard`s and `PodcastTombstone`s (`furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1` canonical signing input), monotonic revisions with a total tie-break, text and id keys, `PodcastDht::{publish, unpublish, sync_shares, republish, search, lookup_podcast}` gated by `effective_share`, typed search hits - (`PodcastSearchHit`, `FederatedSearchHit`, `SearchHitKind`, - `merge_search_results`) and the advisory `feed_url_privacy_warning`. -- `music-dht 0.6.0`: capability manifests advertise `podcast_dht = 1` - (`capabilities::PODCAST_DHT_ID`, `CapabilityManifest::protocol_version`, - `CapabilityManifest::supports_podcast_dht`); applications that do not run - the overlay must remove it with `without_protocol`. + (`PodcastSearchHit`, `ClaimedFeedUrl`, `FederatedSearchHit`, + `SearchHitKind`, `merge_search_results`) and the advisory + `feed_url_privacy_warning`. +- Replicas accept records only from their publisher, store one payload per + `(podcast_id, publisher)`, admit placements only for keys they are close to + and within per-key, per-publisher and total budgets, and never evict + existing records for newcomers. Search offers only the URL the podcast id + hashes from; moved-feed URLs are reported as unverified claims. +- `sync_shares` is fail-safe for revocation (per-podcast `failed`) and holds + back cards with privacy warnings until confirmed (`held_back`, + `PodcastShareState::privacy_confirmed`). +- `music-dht 0.6.0`: capability manifests can advertise `podcast_dht = 1` + (`capabilities::PODCAST_DHT_ID`, `CapabilityManifest::with_podcast_dht`, + `CapabilityManifest::protocol_version`, + `CapabilityManifest::supports_podcast_dht`). It is opt-in: + `CapabilityManifest::frid()` does not include it. - `music-dht 0.6.0`: new `MusicDhtError::PodcastPublication` variant (the enum is `#[non_exhaustive]`). - The music DHT (`music-dht-v5`), `federation-net` and device-sync protocol v4 diff --git a/PODCASTS.md b/PODCASTS.md index dcd9653..4d4b9bd 100644 --- a/PODCASTS.md +++ b/PODCASTS.md @@ -514,8 +514,8 @@ of one podcast by `podcast_id`. | --- | --- | | ALPN | `furumi-fd/podcast-dht/1` (schema-independent stream protocol) | | Wire and record version | `PODCAST_DHT_PROTOCOL_VERSION = 1` | -| Capability manifest | `podcast_dht = 1` (`CapabilityManifest::supports_podcast_dht`) | -| Cache | separate SQLite file, `PRAGMA user_version = 1`; a newer schema is refused | +| Capability manifest | `podcast_dht = 1`, opt-in (`CapabilityManifest::with_podcast_dht`, `supports_podcast_dht`) | +| Cache | separate SQLite file, `PRAGMA user_version = 2`; a newer schema is refused, the pre-release version 1 replica table is dropped | | Card TTL | `ACTIVE_RECORD_TTL` of the music DHT (12 h) | | Tombstone TTL | `TOMBSTONE_TTL` of the music DHT (24 h) | | Republish | `DEFAULT_REPUBLISH_INTERVAL` (10 min), expiry sweep `DEFAULT_EXPIRE_INTERVAL` (1 min) | @@ -534,7 +534,7 @@ HTTP validators or episodes. | --- | --- | | `version`, `network_id`, `publisher` | 1, the federation network, the device's iroh endpoint key | | `podcast_id` | must equal `PodcastId::from_feed_url(id_feed_url or feed_url)` | -| `feed_url` | canonical `FeedUrl` the feed is fetched from | +| `feed_url` | canonical `FeedUrl` the publisher fetches the feed from | | `id_feed_url` | the URL the id was derived from, present only when it differs from `feed_url` (feed moved) | | `title` | 1..300 characters, no control characters | | `author` | optional, ≤ 300 characters, no control characters | @@ -550,10 +550,14 @@ HTTP validators or episodes. whole signed record is at most 24 KiB postcard-encoded; the field bounds keep every valid card below it. -Binding the id to a URL in the card prevents claiming an existing podcast id -for an arbitrary feed. When a feed moved, `feed_url` is the publisher's -claim; clients should prefer their own subscription's URLs for a podcast -they already know. +Only the **origin URL** (`PodcastCard::origin_feed_url`: `id_feed_url`, or +`feed_url` when the feed has not moved) is bound to the podcast, because the +id is its hash. When `id_feed_url` is present, `feed_url` is an unverified +claim of the publisher that the feed moved: anyone can sign a card that +carries an existing podcast's origin URL and an arbitrary `feed_url`. Search +therefore never offers a claimed URL as the podcast's feed (see Search), and +clients must not fetch it as that podcast or record it as an alias unless the +feed at the origin URL itself redirects there. ### Tombstone @@ -566,6 +570,10 @@ lower or equal revision and is itself retained and served until it expires. Because the tombstone TTL exceeds the card TTL, every card issued before a tombstone expires before it. +`index_keys` is self-asserted, so it never creates a placement by itself: a +replica indexes a tombstone under a text key only in place of a record of the +same `(podcast_id, publisher)` it already holds there (see Replica storage). + ### Signing input Both records are signed with the publisher's Ed25519 iroh identity over a @@ -594,9 +602,10 @@ postcard `SignedPodcastRecord { body: Card | Tombstone, signature }`. material)` with kind `0x01` and the podcast id string for the id key, or kind `0x02` and a token for a text key. Tokens are the distinct `normalize_name` tokens of the title, then of the author, skipping tokens -longer than 64 bytes, at most 32. A card is stored under the id key and its +longer than 64 bytes, at most 32. A card belongs under the id key and its text keys (at most 33); a tombstone under the id key and its `index_keys`. -Replicas accept a record only under a key derived from it. +The publisher sends each record to the `K` (8) overlay nodes closest to each +of its keys, one `StoreBatch` per node listing the keys that node should hold. ### Ordering @@ -604,10 +613,10 @@ For one `(podcast_id, publisher)` the greater record wins: higher `revision`; at an equal revision a tombstone beats a card; then later `issued_at_ms` (a republish); then later `expires_at_ms`; then the lexicographically greater signing input. This is a total order, so every -replica converges whatever the delivery order. A replica that holds a -tombstone, or a card of a higher revision, under any key drops older cards of -that publisher under every key and refuses them later; a replayed old -revision is therefore ignored. +replica and every lookup converge whatever the delivery order. A replica +keeps only the winning record of each `(podcast_id, publisher)`; a replayed +old revision is ignored. These fields only order one publisher's own records: +they never rank or select records of different publishers. Revisions: a changed card gets `max(previous + 1, now_ms)` (time-seeded, so revisions stay monotonic even if the local cache is lost); an unchanged card @@ -615,12 +624,80 @@ is re-signed with the same revision and a fresh lifetime on every republish; unpublishing writes a tombstone with `previous + 1`; publishing again later gets `max(tombstone + 1, now_ms)`. +### Wire protocol + +One request per bidirectional stream on `furumi-fd/podcast-dht/1`: the +requester writes one postcard `PodcastDhtRequest` and finishes its send side; +the responder writes one postcard `PodcastDhtResponse` and finishes. Messages +are not length-prefixed; the stream end delimits them. Requests above +512 KiB and responses above 4 MiB are refused. + +```text +PodcastDhtRequest { version: u16, requester: NodeContact, operation: PodcastDhtOperation } +PodcastDhtOperation = 0: FindNode { target: [u8; 32] } + | 1: FindValue { key: PodcastDhtKey, filter_tokens: Vec } + | 2: StoreBatch { records: Vec } +PodcastStoreEntry { keys: Vec, record: SignedPodcastRecord } +PodcastDhtResponse { version: u16, ok: bool, error: Option, + nodes: Vec, records: Vec, stored: u32 } +NodeContact { node_id: [u8; 32], peer_id: EndpointId, ticket: String, last_seen_ms: u64 } +PodcastRecordBody = 0: Card(PodcastCard) | 1: Tombstone(PodcastTombstone) +``` + +Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. + +* `version` must be 1 in both directions. `requester.peer_id` must equal the + authenticated transport peer, `node_id` must be `NodeId::from_endpoint` + of it and its ticket must name the same endpoint and network; otherwise the + request is refused. The responder learns the requester as an overlay + contact. +* `FindNode` returns the `K` closest overlay contacts (the responder + included). +* `FindValue` returns the records under `key` (see Replica storage) and the + `K` closest contacts. `filter_tokens` (≤ 8, each 1..64 bytes) are + normalized tokens every returned card must be indexed under; tombstones are + returned regardless. Requesters verify every record again and drop records + that do not belong under `key` or miss a filter token. +* `StoreBatch` carries 1..16 entries, each with 1..33 distinct keys. Every + record must be published by the requester itself (replicas never forward + records), pass `verify_at` (bounds, network, lifetime, clock skew, + signature, size) and belong under each listed key; otherwise the whole + batch is refused. Keys the responder is not responsible for are skipped. + `stored` counts the `(key, record)` placements written. +* A refused request gets `ok = false` with an `error` of at most 1 KiB and + empty `nodes`, `records` and `stored = 0`; an accepted one has `ok = true` + and no `error`. Requesters reject responses that violate this or exceed + `K` nodes or 128 records. + +### Replica storage + +* A replica stores one payload per `(podcast_id, publisher)` (the winning + record) and indexes it under the keys it holds. A newer version replaces + it in place; placements the new version no longer belongs under (an old + title's tokens, keys a tombstone does not list) are dropped, so on one + replica a tombstone supersedes the card under every key it held. +* A node accepts a placement only when fewer than `2 * K` of its known + overlay contacts are closer to the key than itself. +* Admission, never eviction: a key holds at most 256 records. A full key + admits no new `(podcast_id, publisher)` until one expires; existing records + keep their place and their publishers refresh them in place. A tombstone + is indexed under a text key only in place of its owner's record there; new + tombstones under a key (in practice the id key, for cards this replica + never saw) are limited to 32. +* A replica holds at most 256 records per publisher, 4 096 records and + 32 MiB of payload; records beyond that are refused, not evicted. At most + 1 024 overlay contacts are persisted. +* `FindValue` answers with at most 128 records: at most 32 tombstones and + the cards indexed longest first. Every record is verified again on every + read. + ### Publication * `PodcastDht::publish(subscription, share, card)` refuses (`MusicDhtError::PodcastPublication`) unless `effective_share` holds, so a new or resumed subscription is never published without a fresh share - decision. + decision. The UI calls it only after asking the user about any privacy + warning, so the call is the user's consent. * `PodcastDht::sync_shares(states)` is declarative and idempotent: podcasts for which `effective_share` holds are published (when changed, new or past half their lifetime), every other podcast this device published is @@ -628,17 +705,27 @@ gets `max(tombstone + 1, now_ms)`. operations, so unsharing or unsubscribing on any trusted device revokes every device's own card once the registers converge, and a stale device cannot republish (its registers lose to the newer share decision). +* Revocation is fail-safe: each podcast is handled on its own. A shared + podcast whose card cannot be built or signed is reported in + `PodcastDhtSyncStats::failed` and treated as not shared (an earlier + publication is tombstoned); it never prevents other podcasts from being + unpublished. +* A card whose `PodcastCardSpec::privacy_warnings` is not empty is held back + (`PodcastDhtSyncStats::held_back`, with the warnings) unless + `PodcastShareState::privacy_confirmed` is set or this device already + publishes the same `feed_url` and `id_feed_url`. A feed move to a URL that + looks private therefore asks again; an earlier publication of a held-back + podcast stays as it was. * `PodcastDht::unpublish(podcast_id)` writes the tombstone. Nothing is ever sent for a podcast this device never published: no tombstone, no id. * `PodcastDht::republish` (run by `maintenance`) re-signs active cards and re-sends unexpired tombstones every 10 minutes. -* Before publishing, the UI must call `feed_url_privacy_warning` (or - `PodcastCardSpec::privacy_warnings`, which also checks `id_feed_url`) and - ask the user when it returns a warning. The heuristics are advisory: query +* The privacy heuristics (`feed_url_privacy_warning`) are advisory: query parameter names containing words such as `token`, `key`, `auth`, `secret`, `sig`/`signature`, `uid`, `user`, `pass`, `session`, `access`, `private`; known member-feed hosts; and long high-entropy query values (≥ 16 - characters) or path segments (≥ 32 characters). False positives and misses + characters) or path segments (≥ 32 characters). They check `feed_url` and + `id_feed_url`, not artwork or web-page links. False positives and misses are possible. ### Search @@ -649,39 +736,54 @@ gets `max(tombstone + 1, now_ms)`. cards indexed under **all** query tokens (tombstones are always returned). Results are verified again, the winning record per publisher is kept, tombstoned publishers are dropped and cards are merged by `podcast_id` - into `PodcastSearchHit { podcast_id, card, publishers }`. Ranking: exact - normalized title first, then more publishers. + into `PodcastSearchHit { podcast_id, feed_url, card, publishers, + claimed_feed_urls }`. * `PodcastDht::lookup_podcast(podcast_id)` reads the id key. -* `publishers` lists distinct publisher **devices**; it is not a subscriber +* `feed_url` is the origin URL; it is the only URL a client subscribes with. + `claimed_feed_urls` lists other `feed_url`s that publishers claim (at most + 8, with their publishers), for display only. +* The representative `card` is a card whose `feed_url` is the origin URL + when one exists, then the title most publishers agree on, then the + smallest publisher key. Publisher-controlled times and revisions never + break ties. Its metadata remains a publisher's claim. +* `publishers` lists distinct publisher **devices**. It is not a subscriber count (one user may publish from several devices, and subscribers who did - not share are invisible), so no subscriber count is offered. -* The representative card is the one whose `feed_url` most publishers agree - on, then the most recently issued, then the greatest publisher key. + not share are invisible) and not a trust signal: device identities are + free. No subscriber count is offered and ranking ignores it. +* Ranking: exact normalized title first, then hits that have a card under + the origin URL, then title and id. * `FederatedSearchHit` (`{"type": "music" | "podcast", "hit": ...}`), `SearchHitKind` (`artist`, `release`, `track`, `podcast`) and `merge_search_results` merge podcast hits with `SearchOutcome` music results; `SearchOutcome` itself is unchanged. * Lookups query the 8 closest overlay nodes (α = 3, at most 32 requests, - 15 s deadline) and return partial results when peers fail. A peer that - fails a request is skipped for 30 s. + 15 s deadline), keep at most 512 `(podcast_id, publisher)` records each and + return partial results when peers fail. A peer that fails a request is + skipped for 30 s. -Subscribing from a hit creates the user's own private subscription: resolve -the id with `resolve_podcast_id` on the card's `id_feed_url` (or `feed_url`), -which yields the card's `podcast_id` unless the user already knows that URL, -fetch the feed from `feed_url` and record it as an alias when it differs. The -new subscription is private (see `effective_share`), the publisher's -availability is not needed afterwards, and the episode catalog comes from RSS. +Subscribing from a hit creates the user's own private subscription with the +hit's `feed_url` (the origin URL): resolve the id with `resolve_podcast_id`, +which yields the hit's `podcast_id` unless the user already knows that URL, +fetch the feed from that URL and follow its own HTTP redirects and +`itunes:new-feed-url`, recording those as aliases as usual. A URL from +`claimed_feed_urls` becomes an alias only when that fetch leads there. The new +subscription is private (see `effective_share`), the publisher's availability +is not needed afterwards, and the episode catalog comes from RSS. ### Limits | Limit | Value | | --- | --- | | Request / response | 512 KiB / 4 MiB postcard | -| Store batch | 1..16 entries, each 1..33 distinct keys | -| Records per `FindValue` response | 128 | +| Store batch | 1..16 entries, each 1..33 distinct keys, records of the requester only | +| Records per `FindValue` response | 128, at most 32 of them tombstones | | Filter tokens | ≤ 8, each 1..64 bytes | -| Records cached per key | 256; the record expiring first is evicted for a longer-lived one | -| Concurrent inbound requests | 64 (more are dropped); 10 s per request | +| Records per key | 256 (new tombstones 32); admission only, no eviction | +| Records per replica | 256 per publisher, 4 096 in all, 32 MiB of payload | +| Store responsibility | fewer than `2 * K` = 16 known contacts closer to the key | +| Records kept per lookup | 512 `(podcast_id, publisher)` | +| Persisted overlay contacts | 1 024 | +| Inbound requests | 64 concurrent, 8 per peer (more are dropped); 10 s each | | Error message | ≤ 1 KiB | | Contact ticket | ≤ 16 KiB | @@ -692,20 +794,28 @@ availability is not needed afterwards, and the episode catalog comes from RSS. * Cards carry the public projection only (see Card); tombstones carry key hashes only. * Records are signed by the publisher device and verified on every receive - and on every cache read; a forwarder cannot alter or impersonate them, and - records from another network are rejected. -* Cards do not carry a connection ticket: discovering a card does not reveal - how to reach the publisher, and subscribing never contacts it. -* The publisher's device key is public in the card. Devices of one user are - not linked by the overlay, but publishing the same podcasts from several - devices makes them correlatable. + and on every cache read; a forwarder cannot alter or impersonate them, + only the publisher can store them, and records from another network are + rejected. +* The card's `publisher` is the device's persistent iroh endpoint key, which + is also its overlay and music-DHT identity. It is **linkable**: anyone can + look the key up in the overlay (`FindNode` for + `NodeId::from_endpoint(publisher)`) and obtain the device's contact ticket + with its relay and direct IP addresses, because every overlay request + carries the requester's ticket. The same key owns the device's music-DHT + records, so a device's shared podcasts can be linked to its public music + library and its network address. Subscribing from a card never contacts + the publisher. +* Devices of one user are not linked by the overlay, but publishing the same + podcasts from several devices makes them correlatable. * Unpublishing is eventually consistent: replicas that missed the tombstone - keep serving the old card until it expires (at most 12 h). A replica that - only held text keys of an older revision (after a title change) may serve - that revision until it expires. + keep serving the old card until it expires (at most 12 h), including + replicas that only held an older title's text keys. * Capability manifests are informational. A peer without `podcast_dht` never enters the overlay routing table (it cannot answer on the ALPN), and - `PodcastDht::note_peer_capabilities` evicts a peer whose manifest lacks it. + `PodcastDht::note_peer_capabilities` evicts a peer whose manifest lacks it + and stops dialing it. Applications advertise the overlay only when they + run it (`with_podcast_dht`). ## Feed parser @@ -879,5 +989,5 @@ must keep the same invariants. * Public podcast cards use the separate signed overlay `podcast_dht` with its own ALPN (`furumi-fd/podcast-dht/1`), wire/record version 1, signing domains `furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1`, key - domain `furumi-podcast-dht:key:v1` and cache schema 1, all frozen; the music + domain `furumi-podcast-dht:key:v1`, all frozen, and cache schema 2; the music DHT schema and `federation-net` do not change. diff --git a/crates/music-dht/README.md b/crates/music-dht/README.md index f5cb318..e34c778 100644 --- a/crates/music-dht/README.md +++ b/crates/music-dht/README.md @@ -254,15 +254,27 @@ reported itself (`profile_protocol_version`; relayed profiles go through `music_dht::podcast_dht::PodcastDht` is a separate signed overlay (ALPN `furumi-fd/podcast-dht/1`, capability `podcast_dht`) for podcasts the user -explicitly shared. Declare the ALPN with -`schema_independent_stream_protocol`, open the overlay with its own cache file, -and spawn `serve` and `maintenance`, as for `SimilarityDht`. Call -`sync_shares` with the merged subscription and share registers after applying -sync operations: it publishes cards only where `effective_share` holds and -tombstones every other own publication. Ask the user before publishing when -`PodcastCardSpec::privacy_warnings` (or `feed_url_privacy_warning`) reports a -URL that looks private. `search` and `lookup_podcast` return -`PodcastSearchHit`s; `merge_search_results` combines them with +explicitly shared. Wiring, as for `SimilarityDht`: + +1. declare the ALPN with `schema_independent_stream_protocol(PODCAST_DHT_ALPN)`, + open the overlay with its own cache file, take + `stream_acceptor(PODCAST_DHT_ALPN)` and spawn `serve` and `maintenance`; +2. advertise it with `CapabilityManifest::with_podcast_dht()` (it is not in + `CapabilityManifest::frid()`; an application that does not run the overlay + simply does not add it); +3. pass every peer manifest received to `note_peer_capabilities`, so peers + without the overlay are not dialed; +4. call `sync_shares` with the merged subscription and share registers after + applying sync operations: it publishes cards only where `effective_share` + holds, tombstones every other own publication, and reports cards it could + not build (`failed`) or held back for privacy (`held_back`); +5. ask the user before publishing when `PodcastCardSpec::privacy_warnings` + (or `feed_url_privacy_warning`) reports a URL that looks private, then + call `publish` or set `PodcastShareState::privacy_confirmed`. + +`search` and `lookup_podcast` return `PodcastSearchHit`s; subscribe with the +hit's `feed_url` (the URL the id hashes from), never with a +`claimed_feed_urls` entry. `merge_search_results` combines hits with `SearchOutcome::network_results` as typed `FederatedSearchHit`s. ## Verification -- 2.54.0 From 71e3f1fa0a153349fe627b21928d5b8b5259438b Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 18:09:05 +0200 Subject: [PATCH 6/9] Reconcile podcast store and discovery Cross-reference the SQLite podcast store and the podcast_dht overlay in PODCASTS.md, ARCHITECTURE.md and the furumi-library README, and test that the store's merged subscription and share registers build the share state and cards PodcastDht::sync_shares publishes, including after a feed move and after unsubscribing. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 3 +- PODCASTS.md | 13 +++- crates/furumi-library/README.md | 4 ++ crates/furumi-library/src/podcasts/tests.rs | 79 +++++++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d508928..6641fdf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -239,7 +239,8 @@ Public podcast discovery is a third schema-independent Kademlia overlay, `music_dht::podcast_dht` (ALPN `furumi-fd/podcast-dht/1`, version 1, capability `podcast_dht`), built like the similarity-routing overlay. A device publishes a card only for podcasts whose merged registers satisfy -`effective_share`; `sync_shares` reconciles publications declaratively and +`effective_share` (SQLite clients read the merged registers from +`PodcastStore`); `sync_shares` reconciles publications declaratively and nothing is sent for a podcast the device never published. A card is the public projection of a podcast (feed URL and channel metadata, no personal state), signed by the device's iroh identity over a domain-separated diff --git a/PODCASTS.md b/PODCASTS.md index 4d4b9bd..2443e0a 100644 --- a/PODCASTS.md +++ b/PODCASTS.md @@ -21,7 +21,7 @@ cards in a separate DHT overlay, `music_dht::podcast_dht`; see | `music_dht::podcast` | ids, URL canonicalization, no-GUID rule, metadata DTOs and bounds, merge functions, RSS/Atom parser (feature `podcast-feed`) | | `music_dht::device_sync` | sync operation kinds, snapshot sections, `PlaybackSource`, version gating, Jam redaction | | `music_dht::podcast_dht` | public podcast cards and tombstones, their signatures, replication, cache and search, privacy warnings for feed URLs | -| Client storage (`furumi_library::podcasts::PodcastStore`, web PostgreSQL) | persistence and migrations implementing the rules below (SQLite reference: see "SQLite reference store") | +| Client storage (`furumi_library::podcasts::PodcastStore`, web PostgreSQL) | persistence and migrations implementing the rules below (SQLite reference: [SQLite reference store](#sqlite-reference-store)) | | Client runtime | HTTP fetching, conditional requests, scheduling, audio buffering and decoding, UI | Audio is always fetched from the publisher. It is never imported into the @@ -716,6 +716,13 @@ Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. publishes the same `feed_url` and `id_feed_url`. A feed move to a URL that looks private therefore asks again; an earlier publication of a held-back podcast stays as it was. +* SQLite clients build each `PodcastShareState` from + `PodcastStore::subscription` (the merged register with its alias set, which + always contains the origin URL) and `PodcastStore::share`; its `is_shared` + equals `PodcastSummary::shared`. A card for the URL the device now fetches + sets `feed_url` from the catalog and `id_feed_url` to the origin URL. The + store does not persist `privacy_confirmed`; the client keeps that + confirmation. See [SQLite reference store](#sqlite-reference-store). * `PodcastDht::unpublish(podcast_id)` writes the tombstone. Nothing is ever sent for a podcast this device never published: no tombstone, no id. * `PodcastDht::republish` (run by `maintenance`) re-signs active cards and @@ -928,7 +935,9 @@ must keep the same invariants. `set_shared`/`unsubscribe` the share register, with `next_podcast_generation`. No-op actions draw no stamp and emit nothing. `unsubscribe` emits `subscribed = false` and `shared = false` and keeps - progress, sessions and the catalog. + progress, sessions and the catalog. The store only records the decision; + publishing and revoking the public card is `PodcastDht::sync_shares` (see + [Public discovery](#public-discovery-podcast_dht)). * Explicit progress actions use `explicit_generation(max(local, carried))`; checkpoints keep the session generation and are refused (`Superseded`) once the register holds a greater one, or after the session completed. diff --git a/crates/furumi-library/README.md b/crates/furumi-library/README.md index 6ad2aaa..1d389d1 100644 --- a/crates/furumi-library/README.md +++ b/crates/furumi-library/README.md @@ -57,6 +57,10 @@ It lives in its own file, conventionally `podcasts.sqlite3` - **Queries**: `list_podcasts`, `seasons`, `episodes` (season, played-state and text filters, date or episode-number order), `resume_position_ms`, `episode_ref` / `episode_track` for queueing, `listen_history`. +- **Public discovery**: the store records share decisions only. Build each + `music_dht::podcast_dht::PodcastShareState` for `PodcastDht::sync_shares` + from `subscription` and `share` (workspace `PODCASTS.md`, section "Public + discovery (podcast_dht)"); the client keeps the user's `privacy_confirmed` decision. The store is synchronous: methods take `&self` and lock one connection, so `Arc` can be shared; run calls on a blocking pool. diff --git a/crates/furumi-library/src/podcasts/tests.rs b/crates/furumi-library/src/podcasts/tests.rs index df2523d..975ddc3 100644 --- a/crates/furumi-library/src/podcasts/tests.rs +++ b/crates/furumi-library/src/podcasts/tests.rs @@ -2242,3 +2242,82 @@ fn concurrent_fetch_state_writers_lose_no_update() { assert_eq!(state.consecutive_failures as usize, THREADS * FAILURES); assert_eq!(state.cover_path.as_deref(), Some("/covers/c.jpg")); } + +// --------------------------------------------------------------------------- +// Public discovery (`music_dht::podcast_dht`) + +/// The share state `PodcastDht::sync_shares` takes, built from the store the +/// way clients do it. +fn discovery_state( + store: &PodcastStore, + podcast_id: &PodcastId, +) -> music_dht::podcast_dht::PodcastShareState { + music_dht::podcast_dht::PodcastShareState { + subscription: store.subscription(podcast_id).unwrap().unwrap(), + share: store.share(podcast_id).unwrap(), + card: None, + privacy_confirmed: false, + } +} + +#[test] +fn store_registers_drive_discovery_publication() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let podcast_id = store + .subscribe(&mut stamps, &meta(FEED, "Show")) + .unwrap() + .podcast_id; + + // Subscribed but private: nothing to publish. + let state = discovery_state(&store, &podcast_id); + assert!(!state.is_shared()); + assert!(!store.podcast(&podcast_id).unwrap().unwrap().shared); + + store.set_shared(&mut stamps, &podcast_id, true).unwrap(); + let state = discovery_state(&store, &podcast_id); + assert!(state.is_shared()); + assert!(store.podcast(&podcast_id).unwrap().unwrap().shared); + let card = state.card_spec().unwrap(); + assert_eq!(card.podcast_id, podcast_id); + assert_eq!(card.feed_url, url(FEED)); + assert_eq!(card.id_feed_url, None); + assert_eq!(card.title, "Show"); + assert!(card.privacy_warnings().is_empty()); + + // After a permanent move the register's alias set still holds the origin + // URL, so the card stays bound to the podcast id. + let items = [Item::new( + Some("g1"), + "One", + "https://cdn.example.test/1.mp3", + )]; + let moved = url("https://new.example.test/feed.xml"); + let feed = parse(&rss("Show", None, &items), &moved); + refresh(&store, &mut stamps, &podcast_id, &feed, &moved, true); + let state = discovery_state(&store, &podcast_id); + assert!(state.subscription.feed_urls().contains(&moved)); + let card = state.card_spec().unwrap(); + assert_eq!(card.podcast_id, podcast_id); + // A card for the URL this device now fetches names the origin URL. + let summary = store.podcast(&podcast_id).unwrap().unwrap(); + assert_eq!(summary.meta.feed_url, moved); + let fetched = music_dht::podcast_dht::PodcastShareState { + card: Some(music_dht::podcast_dht::PodcastCardSpec { + feed_url: summary.meta.feed_url.clone(), + id_feed_url: Some(url(FEED)), + ..card + }), + ..state + }; + let card = fetched.card_spec().unwrap(); + assert_eq!(card.feed_url, moved); + assert_eq!(card.id_feed_url, Some(url(FEED))); + card.validate().unwrap(); + + // Unsubscribing revokes the share decision: the card must be unpublished. + store.unsubscribe(&mut stamps, &podcast_id).unwrap(); + let state = discovery_state(&store, &podcast_id); + assert!(!state.is_shared()); + assert!(!store.podcast(&podcast_id).unwrap().unwrap().shared); +} -- 2.54.0 From c0def74115a205b67d51ce99551ae14d1c65752a Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 18:50:22 +0200 Subject: [PATCH 7/9] Evict podcast replicas fairly and trust only verified contacts Replicas no longer hand capacity to whoever stored first: a key holds at most one FindValue response worth of records (128, 32 tombstones), so every admitted record is served, and a full key or cache admits a newcomer by evicting a random record from its younger half. In-place card-to-tombstone conversions count against the tombstone budget. Contacts returned by peers are only lookup and bootstrap candidates until they answer; replica responsibility counts verified contacts that are not backed off, and contacts that keep failing are dropped. Refused store batches fall back to the next closest nodes once. Each lookup response may add at most 64 new owners. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/podcast_dht/mod.rs | 367 +++++++++++++---- crates/music-dht/src/podcast_dht/store.rs | 262 ++++++++++-- crates/music-dht/src/podcast_dht/tests.rs | 480 ++++++++++++++++++---- crates/music-dht/tests/podcast_dht.rs | 90 ++++ 4 files changed, 987 insertions(+), 212 deletions(-) diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs index f71554f..ce8d5ba 100644 --- a/crates/music-dht/src/podcast_dht/mod.rs +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -124,6 +124,11 @@ pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8; /// Maximum distinct `(podcast_id, publisher)` records one lookup keeps; /// records of further owners are ignored. pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512; +/// Most new `(podcast_id, publisher)` owners one response (or the local +/// cache) may add to a lookup: a `1 / K` share of +/// [`MAX_PODCAST_LOOKUP_RECORDS`], so the first responders cannot fill the +/// lookup and crowd out later ones. +pub const MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE: usize = MAX_PODCAST_LOOKUP_RECORDS / K; /// A node stores a placement only when fewer than this many of its known /// overlay contacts are closer to the key than itself (`2 * K`: the `K` /// closest with slack for differing routing views). @@ -141,6 +146,9 @@ const SERVE_TIMEOUT: Duration = Duration::from_secs(10); /// A peer that failed to answer (network error or timeout) is skipped by /// lookups, publication and bootstrap for this long. const FAILED_PEER_BACKOFF: Duration = Duration::from_secs(30); +/// A contact that failed this many times in a row (each after its backoff) +/// is removed from the routing table and the persisted peers. +const MAX_CONSECUTIVE_FAILURES: u32 = 3; const MAX_TICKET_BYTES: usize = 16 * 1024; const MAX_ERROR_BYTES: usize = 1024; @@ -354,6 +362,7 @@ pub struct PodcastDht { routing: Mutex, unsupported: Mutex>, failed: Mutex>, + failures: Mutex>, publish_gate: Semaphore, serve_limit: Arc, inflight: Arc>>, @@ -401,6 +410,7 @@ impl PodcastDht { routing: Mutex::new(routing), unsupported: Mutex::new(HashMap::new()), failed: Mutex::new(HashMap::new()), + failures: Mutex::new(HashMap::new()), publish_gate: Semaphore::new(1), serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), inflight: Arc::new(Mutex::new(HashMap::new())), @@ -481,9 +491,12 @@ impl PodcastDht { /// A peer whose manifest lacks [`PODCAST_DHT_ID`] at /// [`PODCAST_DHT_PROTOCOL_VERSION`] is removed from the overlay routing /// table and not contacted for [`UNSUPPORTED_PEER_BACKOFF`]. Peers only - /// enter the routing table after answering on [`PODCAST_DHT_ALPN`] or - /// being returned by an overlay peer, so a peer without the overlay is - /// never queried for records or asked to store them. + /// enter the routing table after answering a request on + /// [`PODCAST_DHT_ALPN`] or sending one as an authenticated requester, so + /// a peer without the overlay is never asked to store records and never + /// counts for replica responsibility. A contact another peer returns is + /// only a lookup candidate until it answers itself; a candidate that does + /// not answer is backed off like any failed peer. pub async fn note_peer_capabilities( &self, peer: EndpointId, @@ -500,10 +513,11 @@ impl PodcastDht { } /// Discovers overlay peers among the routing table and the music DHT - /// contacts with one `FindNode` each. Music-DHT contacts whose capability - /// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are - /// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become - /// overlay contacts when they answer. + /// contacts with one `FindNode` each, then asks the not yet known contacts + /// they return once more. Music-DHT contacts whose capability manifest + /// lacks the overlay ([`Self::note_peer_capabilities`]) are skipped; + /// others are dialed on [`PODCAST_DHT_ALPN`]. A peer only becomes an + /// overlay contact when it answers. pub async fn bootstrap(&self) -> Result<()> { let known_feature: HashSet = lock(&self.routing) .contacts() @@ -517,25 +531,19 @@ impl PodcastDht { .into_iter() .filter(|contact| !known_feature.contains(&contact.peer_id)), ); - let mut seen = HashSet::new(); + let mut seen: HashSet = known_feature; + seen.insert(self.own_contact.peer_id); + let mut queried = HashSet::new(); candidates.retain(|contact| { contact.peer_id != self.own_contact.peer_id && !self.is_unsupported(&contact.peer_id) && !self.is_backed_off(&contact.peer_id) - && seen.insert(contact.peer_id) + && queried.insert(contact.peer_id) }); + seen.extend(queried); candidates.truncate(MAX_BOOTSTRAP_PEERS); - let target = *self.own_contact.node_id.as_bytes(); - let responses = stream::iter(candidates.into_iter().map(|contact| async move { - let response = self - .exchange(&contact, PodcastDhtOperation::FindNode { target }) - .await; - (contact, response) - })) - .buffer_unordered(BOOTSTRAP_CONCURRENCY) - .collect::>() - .await; - for (contact, response) in responses { + let mut discovered = Vec::new(); + for (contact, response) in self.find_own_node(candidates).await { let response = match response { Ok(response) => response, Err(error) => { @@ -544,11 +552,46 @@ impl PodcastDht { } }; self.learn_contact(contact).await?; - self.learn_response_nodes(response.nodes).await; + for node in response.nodes { + if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)).is_ok() + && !self.is_unsupported(&node.peer_id) + && !self.is_backed_off(&node.peer_id) + && seen.insert(node.peer_id) + { + discovered.push(node); + } + } + } + // Returned contacts are unverified: they join only by answering. + discovered.truncate(MAX_BOOTSTRAP_PEERS); + for (contact, response) in self.find_own_node(discovered).await { + match response { + Ok(_) => self.learn_contact(contact).await?, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT contact did not answer"); + } + } } Ok(()) } + /// One `FindNode` for this node's id to each contact. + async fn find_own_node( + &self, + contacts: Vec, + ) -> Vec<(NodeContact, Result)> { + let target = *self.own_contact.node_id.as_bytes(); + stream::iter(contacts.into_iter().map(|contact| async move { + let response = self + .exchange(&contact, PodcastDhtOperation::FindNode { target }) + .await; + (contact, response) + })) + .buffer_unordered(BOOTSTRAP_CONCURRENCY) + .collect::>() + .await + } + /// Publishes (or refreshes) this device's card of a podcast. /// /// Refused with [`MusicDhtError::PodcastPublication`] unless @@ -892,7 +935,10 @@ impl PodcastDht { } /// Stores records on the `K` closest overlay nodes of each of their keys - /// (including this node when it is among them). + /// (including this node when it is among them). When a remote node + /// refuses a batch (`stored = 0`, for example because it does not count + /// itself responsible for the keys) or fails, its records are offered + /// once to the next closest nodes of the same keys, one per refusal. async fn push(&self, records: Vec) -> Result { let mut stats = PodcastDhtPublishStats { records: records.len(), @@ -902,69 +948,105 @@ impl PodcastDht { return Ok(stats); } let mut local = Vec::new(); - let mut remote: HashMap)> = HashMap::new(); - for record in records { + let mut plan = StorePlan::new(); + let mut fallbacks: HashMap<(usize, PodcastDhtKey), Vec> = HashMap::new(); + for (index, record) in records.iter().enumerate() { let mut local_keys = Vec::new(); - let mut remote_keys: HashMap)> = - HashMap::new(); for key in record.dht_keys() { stats.keys += 1; - let (store_local, contacts) = self.replica_targets(&key); - if store_local { + let targets = self.replica_targets(&key); + if targets.local { local_keys.push(key); } - for contact in contacts { - remote_keys - .entry(contact.peer_id) - .or_insert_with(|| (contact, Vec::new())) - .1 - .push(key); + for contact in targets.primary { + plan.add(contact, index, key); } + fallbacks.insert((index, key), targets.fallback); } if !local_keys.is_empty() { local.push((local_keys, record.clone())); } - for (peer, (contact, keys)) in remote_keys { - remote - .entry(peer) - .or_insert_with(|| (contact, Vec::new())) - .1 - .push(PodcastStoreEntry { - keys, - record: record.clone(), - }); - } } if !local.is_empty() { self.database.store(local, now_ms()).await?; stats.local_replica = true; } - let sends = stream::iter(remote.into_values().map(|(contact, entries)| async move { - let mut accepted = false; - for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { - let response = self - .exchange( - &contact, - PodcastDhtOperation::StoreBatch { - records: batch.to_vec(), + let (mut accepted, refused) = self.send_store_plan(&records, plan).await; + let retry = retry_plan(refused, &mut fallbacks); + if !retry.is_empty() { + accepted.extend(self.send_store_plan(&records, retry).await.0); + } + stats.remote_nodes = accepted.len(); + Ok(stats) + } + + /// Sends a store plan. Returns the nodes that stored at least one + /// placement and the `(record index, key)` placements offered in batches + /// that were refused or failed. + async fn send_store_plan( + &self, + records: &[SignedPodcastRecord], + plan: StorePlan, + ) -> (HashSet, Vec<(usize, PodcastDhtKey)>) { + let sends = stream::iter(plan.targets.into_values().map(|(contact, entries)| async move { + let entries: Vec<(usize, PodcastStoreEntry)> = entries + .into_iter() + .map(|(index, keys)| { + ( + index, + PodcastStoreEntry { + keys, + record: records[index].clone(), }, ) - .await; - match response { - Ok(response) => accepted |= response.stored > 0, - Err(error) => { - debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); - break; + }) + .collect(); + let mut accepted = false; + let mut refused = Vec::new(); + let mut failed = false; + for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + let stored = if failed { + false + } else { + let response = self + .exchange( + &contact, + PodcastDhtOperation::StoreBatch { + records: batch.iter().map(|(_, entry)| entry.clone()).collect(), + }, + ) + .await; + match response { + Ok(response) => response.stored > 0, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); + failed = true; + false + } + } + }; + if stored { + accepted = true; + } else { + for (index, entry) in batch { + refused.extend(entry.keys.iter().map(|key| (*index, *key))); } } } - accepted + (contact.peer_id, accepted, refused) })) .buffer_unordered(PUBLISH_CONCURRENCY) .collect::>() .await; - stats.remote_nodes = sends.into_iter().filter(|accepted| *accepted).count(); - Ok(stats) + let mut accepted = HashSet::new(); + let mut refused = Vec::new(); + for (peer, stored, keys) in sends { + if stored { + accepted.insert(peer); + } + refused.extend(keys); + } + (accepted, refused) } /// Iterative lookup of `key`: queries the `K` closest known overlay @@ -1013,16 +1095,23 @@ impl PodcastDht { filter_tokens: filter_tokens.clone(), }; let mut futures: FuturesUnordered<_> = batch - .iter() + .into_iter() .map(|contact| { let operation = operation.clone(); - async move { self.exchange(contact, operation).await } + async move { + let response = self.exchange(&contact, operation).await; + (contact, response) + } }) .collect(); - while let Ok(Some(response)) = tokio::time::timeout_at(deadline, futures.next()).await { + while let Ok(Some((contact, response))) = + tokio::time::timeout_at(deadline, futures.next()).await + { let Ok(response) = response else { continue; }; + // It answered on the overlay: now it is a verified contact. + self.learn_contact(contact).await?; merge_records( &mut winners, response.records, @@ -1031,12 +1120,12 @@ impl PodcastDht { &filter_tokens, ); for node in response.nodes { + // Returned contacts are only candidates until they answer. if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)) .is_ok() && !self.is_unsupported(&node.peer_id) && known.insert(node.peer_id) { - self.learn_contact(node.clone()).await?; candidates.push(node); } } @@ -1098,9 +1187,11 @@ impl PodcastDht { PodcastDhtOperation::StoreBatch { records } => { let entries = validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?; + // Only verified contacts that are not backed off count. let nodes: Vec = lock(&self.routing) .contacts() .into_iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) .map(|contact| contact.node_id) .collect(); let entries: Vec<_> = entries @@ -1137,10 +1228,21 @@ impl PodcastDht { match &result { Ok(_) => { lock(&self.failed).remove(&contact.peer_id); + lock(&self.failures).remove(&contact.peer_id); } Err(MusicDhtError::Network(_) | MusicDhtError::Timeout) => { let until = now_ms().saturating_add(FAILED_PEER_BACKOFF.as_millis() as u64); lock(&self.failed).insert(contact.peer_id, until); + let failures = { + let mut failures = lock(&self.failures); + let count = failures.entry(contact.peer_id).or_default(); + *count += 1; + *count + }; + if failures >= MAX_CONSECUTIVE_FAILURES { + lock(&self.failures).remove(&contact.peer_id); + self.forget_contact(contact.peer_id).await; + } } Err(_) => {} } @@ -1180,6 +1282,16 @@ impl PodcastDht { Ok(response) } + /// Removes a contact that keeps failing from routing and the cache. + async fn forget_contact(&self, peer: EndpointId) { + if lock(&self.routing).remove(&peer) { + debug!(%peer, "podcast DHT contact dropped after repeated failures"); + } + if let Err(error) = self.database.remove_peer(peer).await { + debug!(%peer, %error, "podcast DHT contact removal failed"); + } + } + fn is_unsupported(&self, peer: &EndpointId) -> bool { still_listed(&self.unsupported, peer) } @@ -1188,21 +1300,15 @@ impl PodcastDht { still_listed(&self.failed, peer) } - fn replica_targets(&self, key: &PodcastDhtKey) -> (bool, Vec) { - let mut contacts: Vec<_> = lock(&self.routing) + /// Replica targets of `key` among the contacts that are not backed off + /// ([`replica_targets`]). + fn replica_targets(&self, key: &PodcastDhtKey) -> ReplicaTargets { + let contacts: Vec<_> = lock(&self.routing) .contacts() .into_iter() .filter(|contact| !self.is_backed_off(&contact.peer_id)) .collect(); - contacts.push(self.fresh_own_contact()); - contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); - contacts.dedup_by_key(|contact| contact.peer_id); - contacts.truncate(K); - let local = contacts - .iter() - .any(|contact| contact.peer_id == self.own_contact.peer_id); - contacts.retain(|contact| contact.peer_id != self.own_contact.peer_id); - (local, contacts) + replica_targets(self.fresh_own_contact(), contacts, key) } fn closest_contacts(&self, target: &[u8; 32], count: usize) -> Vec { @@ -1220,17 +1326,6 @@ impl PodcastDht { contact } - async fn learn_response_nodes(&self, nodes: Vec) { - for contact in nodes { - if self.is_unsupported(&contact.peer_id) { - continue; - } - if let Err(error) = self.learn_contact(contact).await { - debug!(%error, "invalid podcast DHT contact ignored"); - } - } - } - async fn learn_contact(&self, mut contact: NodeContact) -> Result<()> { validate_contact(&contact, &self.network_id, Some(self.own_contact.peer_id))?; contact.node_id = NodeId::from_endpoint(&contact.peer_id); @@ -1309,8 +1404,12 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> { Ok(()) } -/// Adds verified records found under `key` to the per-owner winners, keeping -/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners. +/// Adds verified records of one response (or the local cache) found under +/// `key` to the per-owner winners. A newer record of a known owner always +/// replaces its winner; the response may add at most +/// [`MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE`] new owners, and the lookup +/// keeps at most [`MAX_PODCAST_LOOKUP_RECORDS`]. So the order in which +/// responders answer does not let the first ones take every slot. fn merge_records( winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, records: Vec, @@ -1319,6 +1418,7 @@ fn merge_records( filter_tokens: &[String], ) { let now = now_ms(); + let mut added = 0usize; for record in records { if record.verify_at(network_id, now).is_err() || !record.belongs_under(key) { continue; @@ -1332,14 +1432,99 @@ fn merge_records( let owner = (record.podcast_id().clone(), record.publisher()); match winners.get(&owner) { Some(current) if compare_records(current, &record).is_ge() => {} - None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {} - _ => { + Some(_) => { winners.insert(owner, record); } + None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS + || added >= MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE => {} + None => { + winners.insert(owner, record); + added += 1; + } } } } +/// Replica nodes of one key, from [`replica_targets`]. +#[derive(Debug)] +struct ReplicaTargets { + /// Whether this node is among the `K` closest. + local: bool, + /// The other nodes among the `K` closest. + primary: Vec, + /// The next up to `K` nodes, farthest first (popped closest first). + fallback: Vec, +} + +/// The `K` closest nodes of `key` among `contacts` and `own` (this node), and +/// the next `K` as fallbacks. +fn replica_targets( + own: NodeContact, + mut contacts: Vec, + key: &PodcastDhtKey, +) -> ReplicaTargets { + let own_peer = own.peer_id; + contacts.push(own); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + contacts.dedup_by_key(|contact| contact.peer_id); + let mut fallback = contacts.split_off(K.min(contacts.len())); + fallback.retain(|contact| contact.peer_id != own_peer); + fallback.truncate(K); + fallback.reverse(); + let local = contacts.iter().any(|contact| contact.peer_id == own_peer); + contacts.retain(|contact| contact.peer_id != own_peer); + ReplicaTargets { + local, + primary: contacts, + fallback, + } +} + +/// One fallback node per refused `(record index, key)` placement: the +/// closest one not yet offered that placement. +fn retry_plan( + refused: Vec<(usize, PodcastDhtKey)>, + fallbacks: &mut HashMap<(usize, PodcastDhtKey), Vec>, +) -> StorePlan { + let mut retry = StorePlan::new(); + for (index, key) in refused { + if let Some(contact) = fallbacks + .get_mut(&(index, key)) + .and_then(|fallback| fallback.pop()) + { + retry.add(contact, index, key); + } + } + retry +} + +/// Placements to offer per remote node: record index and keys. +struct StorePlan { + targets: HashMap>)>, +} + +impl StorePlan { + fn new() -> Self { + Self { + targets: HashMap::new(), + } + } + + fn add(&mut self, contact: NodeContact, index: usize, key: PodcastDhtKey) { + self.targets + .entry(contact.peer_id) + .or_insert_with(|| (contact, BTreeMap::new())) + .1 + .entry(index) + .or_default() + .push(key); + } + + fn is_empty(&self) -> bool { + self.targets.is_empty() + } +} + /// One in-flight inbound request of a peer; released on drop. struct PeerSlot { inflight: Arc>>, diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs index acaedbc..1a86e82 100644 --- a/crates/music-dht/src/podcast_dht/store.rs +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -17,33 +17,53 @@ use crate::error::{MusicDhtError, Result}; use crate::podcast::PodcastId; use crate::routing::{NodeContact, NodeId}; +use super::MAX_PODCAST_DHT_RECORDS; use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records}; /// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1 /// (pre-release) stored one payload per key; its replica table is dropped on /// open because the cache is replaceable. Own publications are kept. pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2; -/// Maximum records (cards and tombstones) indexed under one key. A full key -/// admits no new `(podcast_id, publisher)` until a record expires: existing -/// records are never evicted for newcomers, and their owners keep refreshing -/// them in place. -pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; -/// Maximum tombstones newly indexed under one key. A tombstone is admitted -/// under a text key only in place of a record of the same -/// `(podcast_id, publisher)` already indexed there, so this budget applies to -/// the podcast-id key, where tombstones of unknown cards land. -pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32; +/// Maximum records (cards and tombstones) indexed under one key. It equals +/// [`MAX_PODCAST_DHT_RECORDS`], so one `FindValue` response carries every +/// record of a key: nothing admitted is unreachable. +/// +/// A full key admits a newcomer by evicting a random placement from its +/// younger half (by first indexing time). The older half is never evicted +/// for newcomers, so cards that were there before a flood stay, and the +/// younger half stays contestable, so nobody who stored first holds the key +/// for good. +pub const MAX_PODCAST_RECORDS_PER_KEY: usize = MAX_PODCAST_DHT_RECORDS; +/// Maximum tombstones indexed under one key, including tombstones that +/// replaced their owner's card in place. Over budget, a random tombstone of +/// another owner from the younger half of the key's tombstones is evicted. +/// It equals [`MAX_PODCAST_RESPONSE_TOMBSTONES`], so every tombstone of a key +/// is served. +pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = MAX_PODCAST_RESPONSE_TOMBSTONES; /// Maximum tombstones in one `FindValue` response; cards fill the rest. pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32; /// Maximum `(podcast_id, publisher)` records one publisher device keeps in a -/// replica. +/// replica. A publisher over it is refused (nothing is evicted for it). pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256; -/// Maximum `(podcast_id, publisher)` records in a replica cache. +/// Maximum `(podcast_id, publisher)` records in a replica cache. A full cache +/// admits a new record by evicting random records from the younger half of +/// the cache (by first indexing time), like a full key. pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096; -/// Maximum total encoded size of the records in a replica cache. +/// Maximum total encoded size of the records in a replica cache, enforced +/// like [`MAX_PODCAST_CACHE_RECORDS`]. pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024; /// Maximum persisted overlay contacts; the least recently seen are dropped. pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024; +/// Maximum records evicted from the cache to admit one record; when that is +/// not enough room, the record is refused and nothing is evicted. +const MAX_EVICTIONS_PER_RECORD: usize = 16; + +const _: () = assert!( + MAX_PODCAST_RECORDS_PER_KEY <= MAX_PODCAST_DHT_RECORDS + && MAX_PODCAST_TOMBSTONES_PER_KEY <= MAX_PODCAST_RESPONSE_TOMBSTONES + && MAX_PODCAST_RESPONSE_TOMBSTONES <= MAX_PODCAST_DHT_RECORDS, + "every record admitted under a key must fit one FindValue response" +); const SCHEMA: &str = " CREATE TABLE IF NOT EXISTS podcast_dht_records ( @@ -227,7 +247,8 @@ impl PodcastDhtDatabase { /// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale /// replicas elsewhere can be overruled) and cards containing every token /// in `filter_tokens` up to `limit` records in all. Records indexed longest - /// come first. Cards precede tombstones. + /// come first. Cards precede tombstones. With the default limits and + /// `limit` = [`MAX_PODCAST_DHT_RECORDS`], every record of the key fits. pub async fn records_by_key( &self, key: PodcastDhtKey, @@ -475,10 +496,10 @@ fn store_one( }; let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; if replace - && !admits_record( + && !make_room_for_record( conn, limits, - &publisher, + (&podcast_id, &publisher), existing.is_none(), existing_bytes, payload.len(), @@ -536,6 +557,14 @@ fn store_one( clamp_i64(record.expires_at_ms()), ], )?; + // A card converted to a tombstone in place counts against the + // tombstone budget of each key it stays under, like a new one. + if record.is_tombstone() { + for key in &indexed { + let key = PodcastDhtKey::from_bytes(*key); + make_room_under_key(conn, limits, &key, true, (&podcast_id, &publisher), now, 0)?; + } + } } // New placements. A tombstone only takes a new place under the podcast-id @@ -549,7 +578,15 @@ fn store_one( if record.is_tombstone() && *key != id_key { continue; } - if !admits_placement(conn, limits, key, record.is_tombstone(), now)? { + if !make_room_under_key( + conn, + limits, + key, + record.is_tombstone(), + (&podcast_id, &publisher), + now, + 1, + )? { continue; } conn.execute( @@ -596,58 +633,197 @@ fn store_one( Ok(written) } -/// Whether the cache may hold the new version of a record: per-publisher and -/// global record counts for a new owner, and the global byte budget. -fn admits_record( +/// Makes room in the cache for the new version of a record of `owner`. +/// +/// * A publisher at [`StoreLimits::per_publisher`] gets no new record. +/// * When the record count (for a new owner) or the byte budget would be +/// exceeded, random records of other owners from the younger half of the +/// cache (by first indexing time) are evicted, at most +/// [`MAX_EVICTIONS_PER_RECORD`]. If that is not enough, nothing is evicted +/// and the record is refused. +fn make_room_for_record( conn: &Connection, limits: &StoreLimits, - publisher: &str, + owner: (&str, &str), new_owner: bool, previous_bytes: usize, new_bytes: usize, now: i64, ) -> rusqlite::Result { + if new_bytes > limits.bytes { + return Ok(false); + } + if new_owner { + let own: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", + params![owner.1, now], + |row| row.get(0), + )?; + if own.max(0) as usize >= limits.per_publisher { + return Ok(false); + } + } let (records, bytes): (i64, i64) = conn.query_row( "SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records WHERE expires_at_ms > ?1", [now], |row| Ok((row.get(0)?, row.get(1)?)), )?; - let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); - if bytes.saturating_add(new_bytes) > limits.bytes { - return Ok(false); - } - if !new_owner { + let mut records = records.max(0) as usize; + let mut bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes); + let fits = |records: usize, bytes: usize| { + bytes.saturating_add(new_bytes) <= limits.bytes && (!new_owner || records < limits.records) + }; + if fits(records, bytes) { return Ok(true); } - if records.max(0) as usize >= limits.records { + + // Candidates: the younger half of the other owners' records, shuffled. + let others = records.saturating_sub(usize::from(!new_owner)); + let mut statement = conn.prepare( + "SELECT podcast_id, publisher, payload_bytes FROM ( + SELECT r.podcast_id, r.publisher, r.payload_bytes, + (SELECT MIN(k.first_stored_at_ms) FROM podcast_dht_keys k + WHERE k.podcast_id = r.podcast_id AND k.publisher = r.publisher) AS since + FROM podcast_dht_records r + WHERE r.expires_at_ms > ?1 AND NOT (r.podcast_id = ?2 AND r.publisher = ?3) + ORDER BY since DESC, random() + LIMIT ?4) + ORDER BY random() LIMIT ?5", + )?; + let candidates = statement + .query_map( + params![ + now, + owner.0, + owner.1, + others.div_ceil(2) as i64, + MAX_EVICTIONS_PER_RECORD as i64 + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + }, + )? + .collect::>>()?; + let mut evict = Vec::new(); + for (podcast_id, publisher, payload_bytes) in candidates { + if fits(records, bytes) { + break; + } + records = records.saturating_sub(1); + bytes = bytes.saturating_sub(payload_bytes.max(0) as usize); + evict.push((podcast_id, publisher)); + } + if !fits(records, bytes) { return Ok(false); } - let own: i64 = conn.query_row( - "SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2", - params![publisher, now], - |row| row.get(0), - )?; - Ok((own.max(0) as usize) < limits.per_publisher) + for (podcast_id, publisher) in evict { + delete_owner(conn, &podcast_id, &publisher)?; + } + Ok(true) } -/// Whether `key` has room for one more placement. Existing placements are -/// never evicted. -fn admits_placement( +/// Makes room under `key` for `incoming` more placements (0 when an existing +/// placement of `owner` was converted to a tombstone in place) and returns +/// whether the key has room. +/// +/// Over [`StoreLimits::per_key`], or over [`StoreLimits::tombstones_per_key`] +/// for a tombstone, a random placement of another owner from the younger +/// half of the key (by first indexing time; only tombstones for the +/// tombstone budget) is evicted. The older half is never evicted for a +/// newcomer. +fn make_room_under_key( conn: &Connection, limits: &StoreLimits, key: &PodcastDhtKey, tombstone: bool, + owner: (&str, &str), now: i64, + incoming: usize, ) -> rusqlite::Result { - let (total, tombstones): (i64, i64) = conn.query_row( - "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys - WHERE dht_key = ?1 AND expires_at_ms > ?2", - params![key.as_bytes().as_slice(), now], - |row| Ok((row.get(0)?, row.get(1)?)), + for _ in 0..=MAX_EVICTIONS_PER_RECORD { + let (total, tombstones): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys + WHERE dht_key = ?1 AND expires_at_ms > ?2", + params![key.as_bytes().as_slice(), now], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let over_tombstones = + tombstone && tombstones.max(0) as usize + incoming > limits.tombstones_per_key; + let over_total = total.max(0) as usize + incoming > limits.per_key; + if !over_tombstones && !over_total { + return Ok(true); + } + let Some((podcast_id, publisher)) = + young_placement(conn, key, over_tombstones, owner, now)? + else { + return Ok(false); + }; + conn.execute( + "DELETE FROM podcast_dht_keys + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_bytes().as_slice(), podcast_id, publisher], + )?; + let placed: bool = conn.query_row( + "SELECT EXISTS (SELECT 1 FROM podcast_dht_keys + WHERE podcast_id = ?1 AND publisher = ?2)", + params![podcast_id, publisher], + |row| row.get(0), + )?; + if !placed { + delete_owner(conn, &podcast_id, &publisher)?; + } + } + Ok(false) +} + +/// A random unexpired placement of another owner than `owner` from the +/// younger half of `key` (only tombstones with `tombstones_only`). +fn young_placement( + conn: &Connection, + key: &PodcastDhtKey, + tombstones_only: bool, + owner: (&str, &str), + now: i64, +) -> rusqlite::Result> { + let filter = "dht_key = ?1 AND expires_at_ms > ?2 AND (?3 = 0 OR tombstone = 1) + AND NOT (podcast_id = ?4 AND publisher = ?5)"; + let count: i64 = conn.query_row( + &format!("SELECT COUNT(*) FROM podcast_dht_keys WHERE {filter}"), + params![ + key.as_bytes().as_slice(), + now, + tombstones_only, + owner.0, + owner.1 + ], + |row| row.get(0), )?; - Ok((total.max(0) as usize) < limits.per_key - && (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key)) + if count <= 0 { + return Ok(None); + } + conn.query_row( + &format!( + "SELECT podcast_id, publisher FROM ( + SELECT podcast_id, publisher FROM podcast_dht_keys WHERE {filter} + ORDER BY first_stored_at_ms DESC, random() LIMIT ?6) + ORDER BY random() LIMIT 1" + ), + params![ + key.as_bytes().as_slice(), + now, + tombstones_only, + owner.0, + owner.1, + (count as usize).div_ceil(2) as i64 + ], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() } fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> { diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs index 034f6c3..316bbb5 100644 --- a/crates/music-dht/src/podcast_dht/tests.rs +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -1,6 +1,8 @@ //! Unit tests of the podcast-discovery overlay: records, ordering, cache and //! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`. +use std::collections::HashSet; + use federation_net::{NetworkId, SecretKey}; use crate::routing::NodeId; @@ -584,50 +586,239 @@ async fn open_db_with( .unwrap() } +fn rust_card(index: usize, issued: u64) -> SignedPodcastRecord { + card_with( + spec(&format!("show-{index}.xml"), "Rust Radio"), + &numbered_secret(index), + 1, + issued, + ) +} + +async fn store_all( + db: &store::PodcastDhtDatabase, + entries: Vec<(Vec, SignedPodcastRecord)>, + now: u64, +) -> usize { + let mut written = 0; + for chunk in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + written += db.store(chunk.to_vec(), now).await.unwrap(); + } + written +} + +async fn count_records(temp: &tempfile::TempDir) -> i64 { + rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")) + .unwrap() + .query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| { + row.get(0) + }) + .unwrap() +} + #[tokio::test] -async fn cache_full_key_keeps_its_records_and_refuses_newcomers() { +async fn cache_full_key_admits_newcomers_from_its_younger_half() { let temp = tempfile::tempdir().unwrap(); let db = open_db(&temp).await; - let token_key = PodcastDhtKey::token(&network(), "rust"); - let mut entries = Vec::new(); - for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { - // Later publishers expire later; that buys them nothing. - let record = card_with( - spec("show.xml", "Rust Radio"), - &numbered_secret(index), + let rust = PodcastDhtKey::token(&network(), "rust"); + let half = MAX_PODCAST_RECORDS_PER_KEY / 2; + let older: Vec<_> = (0..half).map(|index| rust_card(index, NOW)).collect(); + let younger: Vec<_> = (half..MAX_PODCAST_RECORDS_PER_KEY) + .map(|index| rust_card(index, NOW)) + .collect(); + let keyed = |records: &[SignedPodcastRecord]| -> Vec<_> { + records + .iter() + .map(|record| (vec![rust], record.clone())) + .collect() + }; + assert_eq!(store_all(&db, keyed(&older), NOW).await, half); + assert_eq!(store_all(&db, keyed(&younger), NOW + 1).await, half); + + // Newcomers keep getting in; each takes a random place of the younger + // half, the older half stays. + let mut newcomers = Vec::new(); + for index in 0..half { + let newcomer = rust_card(1_000 + index, NOW); + assert_eq!( + db.store(vec![(vec![rust], newcomer.clone())], NOW + 2 + index as u64) + .await + .unwrap(), 1, - NOW + index as u64, + "a full key admits a newcomer" ); - entries.push((vec![token_key], record)); + newcomers.push(newcomer); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(older.iter().all(|record| cached.contains(record))); + assert!(cached.contains(newcomers.last().unwrap())); } - let first = entries[0].1.clone(); - let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone(); - // The first publisher arrived first; the rest a moment later. - db.store(entries[..1].to_vec(), NOW).await.unwrap(); - for chunk in entries[1..].chunks(64) { - db.store(chunk.to_vec(), NOW + 1).await.unwrap(); - } - let cached = db - .records_by_key(token_key, Vec::new(), NOW, usize::MAX) - .await - .unwrap(); - assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); - assert!(cached.contains(&first)); - assert!(!cached.contains(&last)); - assert_eq!(cached[0], first, "records indexed longest come first"); - // The first publisher keeps refreshing its place. + // Owners refresh in place without evicting anybody. let refresh = card_with( - spec("show.xml", "Rust Radio"), + spec("show-0.xml", "Rust Radio"), &numbered_secret(0), 1, NOW + 1_000, ); assert_eq!( - db.store(vec![(vec![token_key], refresh)], NOW) + db.store(vec![(vec![rust], refresh.clone())], NOW + 1_000) .await .unwrap(), 1 ); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 1_000, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(cached.contains(&refresh)); +} + +#[tokio::test] +async fn sybils_planted_first_cannot_lock_a_late_card_out_of_a_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let rust = PodcastDhtKey::token(&network(), "rust"); + // 128 throwaway identities, each with its own podcast titled like the + // victim's, fill the key first. + let sybils: Vec<_> = (0..MAX_PODCAST_DHT_RECORDS) + .map(|index| (vec![rust], rust_card(index, NOW))) + .collect(); + store_all(&db, sybils, NOW).await; + let victim = card(&secret(1), 1); + assert_eq!( + db.store(vec![entry(&victim)], NOW + 1).await.unwrap(), + victim.dht_keys().len() + ); + let tokens = vec!["rust".to_string(), "radio".to_string()]; + let served = db + .records_by_key(rust, tokens.clone(), NOW + 1, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(served.len() <= MAX_PODCAST_DHT_RECORDS); + assert!(served.contains(&victim), "everything admitted is served"); + let hits = rank_hits(aggregate_hits(served), "Rust Radio", &tokens); + assert!( + hits.iter() + .any(|hit| hit.podcast_id == *victim.podcast_id()) + ); +} + +#[tokio::test] +async fn a_card_back_after_expiring_gets_into_a_cache_full_of_sybils() { + let limits = store::StoreLimits { + records: 16, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + let victim = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id()); + db.store(vec![entry(&victim)], NOW).await.unwrap(); + let sybils = |issued: u64| -> Vec<_> { + (0..16) + .map(|index| { + let record = rust_card(index, issued); + (record.dht_keys(), record) + }) + .collect() + }; + store_all(&db, sybils(NOW), NOW + 1).await; + assert_eq!(count_records(&temp).await, 16); + + // The victim's device is offline longer than a card lives; the sybils + // keep refreshing theirs in place. + let later = NOW + PODCAST_CARD_TTL.as_millis() as u64 + 3_600_000; + store_all(&db, sybils(later), later).await; + assert!( + db.records_by_key(id_key, Vec::new(), later, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap() + .is_empty() + ); + let back = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later + 1); + assert!(db.store(vec![entry(&back)], later + 1).await.unwrap() > 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), later + 1, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(), + vec![back] + ); + assert_eq!(count_records(&temp).await, 16); +} + +#[tokio::test] +async fn sybil_tombstones_planted_first_cannot_hide_a_revocation() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let victim_card = card(&secret(1), 1); + let id_key = PodcastDhtKey::podcast(&network(), victim_card.podcast_id()); + let rust = PodcastDhtKey::token(&network(), "rust"); + // 64 sybils publish a card of the victim's podcast, then convert it to + // a tombstone in place, under the id key and the text keys. + for index in 0..64 { + let key = numbered_secret(index); + let sybil_card = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW); + db.store(vec![entry(&sybil_card)], NOW).await.unwrap(); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + sybil_card.podcast_id().clone(), + 2, + NOW, + sybil_card.dht_keys(), + ); + let tombstone = + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap(); + assert!(db.store(vec![entry(&tombstone)], NOW).await.unwrap() > 0); + } + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let tombstones = |key: &PodcastDhtKey| -> usize { + conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_keys WHERE dht_key = ?1 AND tombstone = 1", + [key.as_bytes().as_slice()], + |row| row.get::<_, i64>(0), + ) + .unwrap() as usize + }; + for key in [id_key, rust] { + assert_eq!( + tombstones(&key), + MAX_PODCAST_TOMBSTONES_PER_KEY, + "in-place conversions count against the tombstone budget" + ); + } + + // The honest publisher publishes later, then unpublishes. + assert!(db.store(vec![entry(&victim_card)], NOW + 1).await.unwrap() > 0); + let victim_tombstone = tombstone(&secret(1), 2, NOW + 2); + assert!( + db.store(vec![entry(&victim_tombstone)], NOW + 2) + .await + .unwrap() + > 0 + ); + for key in [id_key, rust] { + assert!(tombstones(&key) <= MAX_PODCAST_TOMBSTONES_PER_KEY); + let served = db + .records_by_key(key, vec![], NOW + 2, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!( + served.contains(&victim_tombstone), + "the revocation is served" + ); + // It overrules a stale replica still holding the card. + let mut all = served; + all.push(victim_card.clone()); + assert!(aggregate_hits(all).iter().all(|hit| { + hit.podcast_id != *victim_card.podcast_id() + || !hit.publishers.contains(&secret(1).public()) + })); + } } #[tokio::test] @@ -663,8 +854,8 @@ async fn sybil_tombstones_cannot_suppress_a_card() { written += db.store(chunk.to_vec(), NOW).await.unwrap(); } assert_eq!( - written, MAX_PODCAST_TOMBSTONES_PER_KEY, - "only the id key's tombstone budget is used" + written, 256, + "each sybil tombstone takes a young sybil tombstone's place under the id key" ); let by_id = db @@ -726,36 +917,67 @@ async fn cache_enforces_publisher_record_and_byte_quotas() { let temp = tempfile::tempdir().unwrap(); let db = open_db_with(&temp, limits).await; let show = |index: usize, key: &SecretKey| { - entry(&card_with( + card_with( spec(&format!("show-{index}.xml"), "Rust Radio"), key, 1, NOW, - )) + ) }; let heavy = secret(1); - let stored: Vec = { - let mut out = Vec::new(); - for index in 0..4 { - out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap()); - } - out - }; - assert!(stored[..3].iter().all(|written| *written > 0)); + for index in 0..3 { + let record = show(index, &heavy); + assert!( + db.store(vec![entry(&record)], NOW + index as u64) + .await + .unwrap() + > 0 + ); + } assert_eq!( - stored[3], 0, + db.store(vec![entry(&show(3, &heavy))], NOW + 3) + .await + .unwrap(), + 0, "the fourth podcast of one publisher is refused" ); // A refresh of an admitted record is always accepted. let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1); - assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0); - assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0); - assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0); - assert_eq!( - db.store(vec![show(12, &secret(4))], NOW).await.unwrap(), - 0, - "the cache holds at most five records" + assert!(db.store(vec![entry(&refresh)], NOW + 4).await.unwrap() > 0); + assert!( + db.store(vec![entry(&show(10, &secret(2)))], NOW + 10) + .await + .unwrap() + > 0 ); + assert!( + db.store(vec![entry(&show(11, &secret(3)))], NOW + 11) + .await + .unwrap() + > 0 + ); + assert_eq!(count_records(&temp).await, 5); + + // The cache is full: every newcomer still gets in by evicting a random + // record of the younger half; the two oldest records always stay. + let rust = PodcastDhtKey::token(&network(), "rust"); + for index in 0..6 { + let newcomer = show(20 + index, &secret(10 + index as u8)); + assert!( + db.store(vec![entry(&newcomer)], NOW + 20 + index as u64) + .await + .unwrap() + > 0 + ); + assert_eq!(count_records(&temp).await, 5); + let cached = db + .records_by_key(rust, Vec::new(), NOW + 30, MAX_PODCAST_DHT_RECORDS) + .await + .unwrap(); + assert!(cached.contains(&refresh)); + assert!(cached.contains(&show(1, &heavy))); + assert!(cached.contains(&newcomer)); + } let one = card(&secret(1), 1); let limits = store::StoreLimits { @@ -764,25 +986,34 @@ async fn cache_enforces_publisher_record_and_byte_quotas() { }; let temp = tempfile::tempdir().unwrap(); let db = open_db_with(&temp, limits).await; - assert!( - db.store(vec![entry(&card(&secret(1), 1))], NOW) - .await - .unwrap() - > 0 - ); - assert!( - db.store(vec![entry(&card(&secret(2), 1))], NOW) - .await - .unwrap() - > 0 - ); + let id_key = PodcastDhtKey::podcast(&network(), one.podcast_id()); + for (index, now) in [(1, NOW), (2, NOW + 1), (3, NOW + 2)] { + assert!( + db.store(vec![entry(&card(&secret(index), 1))], now) + .await + .unwrap() + > 0 + ); + } + // The byte budget holds two cards: the younger half of the two others + // (the second card) made room for the third. assert_eq!( - db.store(vec![entry(&card(&secret(3), 1))], NOW) + db.records_by_key(id_key, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS) .await - .unwrap(), - 0, - "the byte budget is full" + .unwrap() + .into_iter() + .map(|record| record.publisher()) + .collect::>(), + HashSet::from([secret(1).public(), secret(3).public()]) ); + // A record larger than the whole budget evicts nothing. + let limits = store::StoreLimits { + bytes: one.encoded_len() - 1, + ..store::StoreLimits::default() + }; + let temp = tempfile::tempdir().unwrap(); + let db = open_db_with(&temp, limits).await; + assert_eq!(db.store(vec![entry(&one)], NOW).await.unwrap(), 0); } #[tokio::test] @@ -931,6 +1162,71 @@ fn replicas_only_accept_keys_they_are_close_to() { assert!(is_responsible(&own, &near[1..], &key)); } +fn contact_of(key: &SecretKey) -> NodeContact { + let peer = key.public(); + NodeContact { + node_id: NodeId::from_endpoint(&peer), + peer_id: peer, + ticket: String::new(), + last_seen_ms: NOW, + } +} + +#[test] +fn refused_placements_fall_back_to_the_next_closest_nodes() { + let key = PodcastDhtKey::token(&network(), "rust"); + let contacts: Vec = (0..3 * K) + .map(|index| contact_of(&numbered_secret(index))) + .collect(); + let mut by_distance = contacts.clone(); + by_distance.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + // This node is the farthest one: it is no replica and no fallback. + let own = by_distance.pop().unwrap(); + let others: Vec = by_distance.clone(); + let targets = replica_targets(own.clone(), others.clone(), &key); + assert!(!targets.local); + let ids = |contacts: &[NodeContact]| -> Vec { + contacts.iter().map(|contact| contact.peer_id).collect() + }; + assert_eq!(ids(&targets.primary), ids(&by_distance[..K])); + let mut fallback = targets.fallback.clone(); + fallback.reverse(); + assert_eq!(ids(&fallback), ids(&by_distance[K..2 * K])); + + // Two refusals of one placement go to the two next closest nodes; + // another record's refusal has its own fallbacks. + let mut fallbacks = HashMap::from([ + ((0, key), targets.fallback.clone()), + ((1, key), targets.fallback), + ]); + let retry = retry_plan(vec![(0, key), (0, key), (1, key)], &mut fallbacks); + let first = by_distance[K].peer_id; + let second = by_distance[K + 1].peer_id; + assert_eq!(retry.targets.len(), 2); + assert_eq!( + retry.targets[&first].1.keys().copied().collect::>(), + vec![0, 1] + ); + assert_eq!( + retry.targets[&second].1.keys().copied().collect::>(), + vec![0] + ); + assert_eq!(fallbacks[&(0, key)].len(), K - 2); + + // With this node among the closest it keeps a local replica and is never + // its own fallback. + let targets = replica_targets(by_distance[0].clone(), others, &key); + assert!(targets.local); + assert_eq!(targets.primary.len(), K - 1); + assert!( + targets + .fallback + .iter() + .chain(&targets.primary) + .all(|contact| contact.peer_id != by_distance[0].peer_id) + ); +} + #[test] fn inbound_requests_are_limited_per_peer() { let inflight = Arc::new(Mutex::new(HashMap::new())); @@ -946,21 +1242,50 @@ fn inbound_requests_are_limited_per_peer() { } #[test] -fn lookups_keep_a_bounded_number_of_owners() { +fn lookups_give_each_response_a_bounded_share_of_owners() { let mut winners = HashMap::new(); let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id()); - let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS) - .map(|index| { - card_with( - spec("show.xml", "Rust Radio"), - &numbered_secret(index), - 1, - now_ms(), - ) - }) - .collect(); - let first = records[0].clone(); - merge_records(&mut winners, records, &network(), &id_key, &[]); + let response = |first: usize| -> Vec { + (first..first + MAX_PODCAST_DHT_RECORDS) + .map(|index| { + card_with( + spec("show.xml", "Rust Radio"), + &numbered_secret(index), + 1, + now_ms(), + ) + }) + .collect() + }; + // Four attacker responders answer first, each with 128 sybil owners. + for responder in 0..4 { + merge_records( + &mut winners, + response(10_000 + responder * MAX_PODCAST_DHT_RECORDS), + &network(), + &id_key, + &[], + ); + } + assert_eq!(winners.len(), 4 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE); + // An honest replica answering later still contributes its owners. + let honest = response(0); + let victim = honest[0].clone(); + merge_records(&mut winners, honest, &network(), &id_key, &[]); + let owner = (victim.podcast_id().clone(), victim.publisher()); + assert_eq!(winners[&owner], victim); + assert_eq!(winners.len(), 5 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE); + + // The lookup as a whole stays bounded. + for responder in 0..8 { + merge_records( + &mut winners, + response(20_000 + responder * MAX_PODCAST_DHT_RECORDS), + &network(), + &id_key, + &[], + ); + } assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS); // A newer record of a known owner still replaces its winner. let newer = card_with( @@ -970,7 +1295,6 @@ fn lookups_keep_a_bounded_number_of_owners() { now_ms(), ); merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]); - let owner = (first.podcast_id().clone(), first.publisher()); assert_eq!(winners[&owner], newer); } diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs index b3ba983..395f3d5 100644 --- a/crates/music-dht/tests/podcast_dht.rs +++ b/crates/music-dht/tests/podcast_dht.rs @@ -18,6 +18,8 @@ //! and is never asked for records or to store them. //! * `sync_shares` stays fail-safe for revocation and holds back URLs that //! look private (single node). +//! * Contacts a peer returns are never trusted: fabricated, unreachable +//! contacts do not enter the routing table (two nodes). use std::path::Path; use std::sync::Arc; @@ -566,3 +568,91 @@ async fn single_node_scenario() { a.stop().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fabricated_contacts_never_join_the_overlay() { + tokio::time::timeout(TEST_TIMEOUT, fabricated_contacts_scenario()) + .await + .unwrap(); +} + +/// K contacts whose node ids are genuine hashes of their endpoint ids but +/// whose tickets carry no address, as a malicious peer could return them. +fn fabricated_contacts(network: NetworkId) -> Vec { + (0..music_dht::K) + .map(|index| { + let peer = federation_net::SecretKey::from_bytes(&[index as u8 + 101; 32]).public(); + let ticket = federation_net::PeerTicket { + ticket_version: federation_net::TICKET_VERSION, + protocol_version: 1, + network_id: network, + schema_id: federation_net::SchemaId::from_bytes([0; 32]), + endpoint_addr: federation_net::EndpointAddr::new(peer), + }; + NodeContact { + node_id: NodeId::from_endpoint(&peer), + peer_id: peer, + ticket: ticket.to_string(), + last_seen_ms: u64::MAX, + } + }) + .collect() +} + +async fn fabricated_contacts_scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-fabricated-contacts"); + let a = Node::start(&temp.path().join("a"), network).await; + // M speaks the overlay ALPN but answers every request with fabricated + // contacts. + let (m, m_events) = start_service(&temp.path().join("m"), network, true).await; + let mut acceptor = m.stream_acceptor(PODCAST_DHT_ALPN).unwrap(); + let fabricated = fabricated_contacts(network); + let response = PodcastDhtResponse { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: true, + error: None, + nodes: fabricated.clone(), + records: Vec::new(), + stored: 0, + }; + let payload = postcard::to_stdvec(&response).unwrap(); + let server = tokio::spawn(async move { + while let Some(mut stream) = acceptor.accept().await { + let mut request = Vec::new(); + let _ = AsyncReadExt::read_to_end( + &mut (&mut stream.recv).take(MAX_PODCAST_DHT_REQUEST_BYTES as u64), + &mut request, + ) + .await; + let _ = stream.send.write_all(&payload).await; + let _ = stream.send.finish(); + let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await; + } + }); + m.connect(a.service.ticket().await.unwrap()).await.unwrap(); + wait_for("music DHT contact", async || { + (!a.service.known_peers().is_empty()).then_some(()) + }) + .await; + + // Bootstrap learns M, which answered, and none of the contacts it made + // up: they are tried once and never answer. + a.dht.bootstrap().await.unwrap(); + assert_eq!(a.dht.known_peers(), 1); + // A lookup treats them as candidates only. + let podcast_id = PodcastId::from_feed_url(&feed("nothing.xml")); + assert!(a.dht.lookup_podcast(&podcast_id).await.unwrap().is_none()); + assert_eq!(a.dht.known_peers(), 1); + a.dht.bootstrap().await.unwrap(); + assert_eq!( + a.dht.known_peers(), + 1, + "fabricated contacts never count for replica responsibility" + ); + + server.abort(); + a.stop().await; + m_events.abort(); + m.shutdown().await.unwrap(); +} -- 2.54.0 From 912527a671bb02fe47e0717ac2792fb22b54f421 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 18:50:22 +0200 Subject: [PATCH 8/9] Document podcast replica eviction and contact verification Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 9 ++++--- CHANGELOG.md | 9 ++++--- PODCASTS.md | 65 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6641fdf..658bda2 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -251,9 +251,12 @@ per normalized title/author token; unpublishing writes a signed tombstone of the next revision that supersedes lower and equal revisions on replicas that hold it, for the keys it lists, and outlives every older card. Only a record's publisher may store it; replicas keep one payload per publisher and podcast, -admit placements only for keys they are close to, within per-key, -per-publisher and total budgets, and never evict existing records for -newcomers (tombstones cannot displace cards). Records use the music DHT's +admit placements only for keys they are close to (counting only contacts that +answered or authenticated, never ones another peer suggested), within per-key, +per-publisher and total budgets. A key holds no more records than one response +carries; a full key or cache admits a newcomer by evicting a random record from +its younger half, so the older half survives floods and capacity is never +first-come-forever (over their budget, tombstones only displace other tombstones). Records use the music DHT's active/tombstone TTLs and republish cadence, are verified on every receive and cache read, and are kept in a separate, replaceable SQLite cache. Search filters on every replica by all query tokens, merges cards of one podcast diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b14174..186c877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,12 @@ `feed_url_privacy_warning`. - Replicas accept records only from their publisher, store one payload per `(podcast_id, publisher)`, admit placements only for keys they are close to - and within per-key, per-publisher and total budgets, and never evict - existing records for newcomers. Search offers only the URL the podcast id - hashes from; moved-feed URLs are reported as unverified claims. + (counting verified contacts only) and within per-key, per-publisher and + total budgets. A key never holds more than one response carries; full keys + and caches admit newcomers by evicting random records from their younger + half. Lookups give each response a bounded share of owners. Search offers + only the URL the podcast id hashes from; moved-feed URLs are reported as + unverified claims. - `sync_shares` is fail-safe for revocation (per-podcast `failed`) and holds back cards with privacy warnings until confirmed (`held_back`, `PodcastShareState::privacy_confirmed`). diff --git a/PODCASTS.md b/PODCASTS.md index 2443e0a..2804dcf 100644 --- a/PODCASTS.md +++ b/PODCASTS.md @@ -650,7 +650,10 @@ Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. authenticated transport peer, `node_id` must be `NodeId::from_endpoint` of it and its ticket must name the same endpoint and network; otherwise the request is refused. The responder learns the requester as an overlay - contact. + contact (it is authenticated). +* `nodes` in a response are unverified suggestions. A requester treats them + as lookup or bootstrap candidates only and adds one to its routing table + (and persisted contacts) only after it answered a request itself. * `FindNode` returns the `K` closest overlay contacts (the responder included). * `FindValue` returns the records under `key` (see Replica storage) and the @@ -663,7 +666,9 @@ Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. records), pass `verify_at` (bounds, network, lifetime, clock skew, signature, size) and belong under each listed key; otherwise the whole batch is refused. Keys the responder is not responsible for are skipped. - `stored` counts the `(key, record)` placements written. + `stored` counts the `(key, record)` placements written. A publisher + offers the placements of a batch answered with `stored = 0` (or that + failed) once to the next closest node of each key. * A refused request gets `ok = false` with an `error` of at most 1 KiB and empty `nodes`, `records` and `stored = 0`; an accepted one has `ok = true` and no `error`. Requesters reject responses that violate this or exceed @@ -676,17 +681,34 @@ Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. it in place; placements the new version no longer belongs under (an old title's tokens, keys a tombstone does not list) are dropped, so on one replica a tombstone supersedes the card under every key it held. -* A node accepts a placement only when fewer than `2 * K` of its known - overlay contacts are closer to the key than itself. -* Admission, never eviction: a key holds at most 256 records. A full key - admits no new `(podcast_id, publisher)` until one expires; existing records - keep their place and their publishers refresh them in place. A tombstone - is indexed under a text key only in place of its owner's record there; new - tombstones under a key (in practice the id key, for cards this replica - never saw) are limited to 32. -* A replica holds at most 256 records per publisher, 4 096 records and - 32 MiB of payload; records beyond that are refused, not evicted. At most - 1 024 overlay contacts are persisted. +* A node accepts a placement only when fewer than `2 * K` of its verified + overlay contacts that are not backed off are closer to the key than + itself. Contacts only enter the routing table by answering a request or + by sending one, so a peer cannot make a replica refuse keys by returning + made-up contacts. A contact that fails three times in a row (each after + its 30 s backoff) is removed. +* A key holds at most 128 records, at most 32 of them tombstones: exactly + what one `FindValue` response carries, so every admitted record is served. + A tombstone is indexed under a text key only in place of its owner's + record there, and such an in-place conversion counts against the + tombstone budget like a new tombstone. +* Full key: a new placement evicts a random placement of another owner from + the younger half of the key (by first indexing time; only tombstones when + the tombstone budget is full). The older half is never evicted for a + newcomer, so records placed before a flood keep their place and their + publishers refresh them in place. Capacity is never first-come-forever: + a newcomer always gets a place, though placements in the younger half can + be evicted again by later newcomers. +* A replica holds at most 256 records per publisher (a publisher over it is + refused), 4 096 records and 32 MiB of payload. A full cache admits a new + record by evicting random records of other owners from its younger half, + at most 16; when that is not enough room, nothing is evicted and the record + is refused. At most 1 024 overlay contacts are persisted. +* Device identities cost nothing, so these rules bound flooding but cannot + prevent it: an attacker with many identities can keep part of the younger + half of a key or cache busy, and a newcomer's place there can be evicted + again until its publisher's next republish (every 10 min) puts it back. + Protocol 1 has no proof of work or other identity cost. * `FindValue` answers with at most 128 records: at most 32 tombstones and the cards indexed longest first. Every record is verified again on every read. @@ -764,9 +786,12 @@ Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. `merge_search_results` merge podcast hits with `SearchOutcome` music results; `SearchOutcome` itself is unchanged. * Lookups query the 8 closest overlay nodes (α = 3, at most 32 requests, - 15 s deadline), keep at most 512 `(podcast_id, publisher)` records each and - return partial results when peers fail. A peer that fails a request is - skipped for 30 s. + 15 s deadline) and return partial results when peers fail. They keep at + most 512 `(podcast_id, publisher)` owners; each response (and the local + cache) may add at most 64 new owners (`512 / K`), so the responders that + answer first cannot fill a lookup and crowd out the rest. A newer record + of a known owner always replaces its winner. A peer that fails a request + is skipped for 30 s. Subscribing from a hit creates the user's own private subscription with the hit's `feed_url` (the origin URL): resolve the id with `resolve_podcast_id`, @@ -785,10 +810,10 @@ is not needed afterwards, and the episode catalog comes from RSS. | Store batch | 1..16 entries, each 1..33 distinct keys, records of the requester only | | Records per `FindValue` response | 128, at most 32 of them tombstones | | Filter tokens | ≤ 8, each 1..64 bytes | -| Records per key | 256 (new tombstones 32); admission only, no eviction | -| Records per replica | 256 per publisher, 4 096 in all, 32 MiB of payload | -| Store responsibility | fewer than `2 * K` = 16 known contacts closer to the key | -| Records kept per lookup | 512 `(podcast_id, publisher)` | +| Records per key | 128, at most 32 of them tombstones; random eviction from the younger half | +| Records per replica | 256 per publisher (refused beyond), 4 096 in all, 32 MiB of payload (random eviction from the younger half, at most 16 per record) | +| Store responsibility | fewer than `2 * K` = 16 verified, not backed-off contacts closer to the key | +| Records kept per lookup | 512 `(podcast_id, publisher)`, at most 64 new ones per response | | Persisted overlay contacts | 1 024 | | Inbound requests | 64 concurrent, 8 per peer (more are dropped); 10 s each | | Error message | ≤ 1 KiB | -- 2.54.0 From 9c82f6d91691a66d0a559caba06e13a1293f06be Mon Sep 17 00:00:00 2001 From: Sanapach Date: Fri, 20 Nov 2026 00:48:01 +0200 Subject: [PATCH 9/9] Withhold persisted podcast cards until the first share sync After a restart, PodcastDht::maintenance republished every locally active card as soon as its first bootstrap found peers, before the client could reconcile with sync_shares, so a share revoked on another trusted device while this one was offline was re-sent from the stale cache. Own cards are now neither republished nor served to FindValue until the first sync_shares of the process (or an explicit publish of that podcast); tombstones are still sent. The first sync_shares re-sends unchanged cards. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 5 +- CHANGELOG.md | 5 ++ PODCASTS.md | 20 ++++- crates/music-dht/README.md | 10 ++- crates/music-dht/src/podcast_dht/mod.rs | 83 +++++++++++++++++-- crates/music-dht/tests/podcast_dht.rs | 105 ++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 13 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 658bda2..9746ad4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -241,7 +241,10 @@ capability `podcast_dht`), built like the similarity-routing overlay. A device publishes a card only for podcasts whose merged registers satisfy `effective_share` (SQLite clients read the merged registers from `PodcastStore`); `sync_shares` reconciles publications declaratively and -nothing is sent for a podcast the device never published. A card is the +nothing is sent for a podcast the device never published. Own cards persisted +by an earlier run are neither republished nor served until the first +`sync_shares` of the process, so a stale cache cannot revive a share revoked +elsewhere while the device was offline. A card is the public projection of a podcast (feed URL and channel metadata, no personal state), signed by the device's iroh identity over a domain-separated canonical encoding. Only the origin URL the id hashes from is bound to the diff --git a/CHANGELOG.md b/CHANGELOG.md index 186c877..20c7d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,11 @@ - `sync_shares` is fail-safe for revocation (per-podcast `failed`) and holds back cards with privacy warnings until confirmed (`held_back`, `PodcastShareState::privacy_confirmed`). +- Own cards persisted by an earlier run are neither republished (by + `maintenance` or `republish`) nor served to lookups until the first + `sync_shares` of the process, so a share revoked on another trusted device + while this one was offline is not re-sent on startup. That first call also + re-sends unchanged cards; tombstones are always re-sent. - `music-dht 0.6.0`: capability manifests can advertise `podcast_dht = 1` (`capabilities::PODCAST_DHT_ID`, `CapabilityManifest::with_podcast_dht`, `CapabilityManifest::protocol_version`, diff --git a/PODCASTS.md b/PODCASTS.md index 2804dcf..0ca514b 100644 --- a/PODCASTS.md +++ b/PODCASTS.md @@ -747,8 +747,22 @@ Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes. confirmation. See [SQLite reference store](#sqlite-reference-store). * `PodcastDht::unpublish(podcast_id)` writes the tombstone. Nothing is ever sent for a podcast this device never published: no tombstone, no id. -* `PodcastDht::republish` (run by `maintenance`) re-signs active cards and - re-sends unexpired tombstones every 10 minutes. +* `PodcastDht::republish` (run by `maintenance` every 10 minutes and after + a bootstrap that found new peers) re-signs active cards and re-sends + unexpired tombstones. +* Own cards persisted by an earlier run are not sent again until the client + calls `sync_shares` in this process: before that, `republish` sends only + tombstones and cards published with `publish` in this process, and + `FindValue` answers leave those cards out (they stay in the cache). A share + revoked on another trusted device while this one was offline is therefore + never re-sent on startup from the stale cache. Clients call `sync_shares` + on start, with the registers they hold, and again after device sync + applies newer operations; the first call also re-sends the unchanged cards + as they are (`PodcastDhtSyncStats::unchanged`, counted in `push`). + `sync_shares` itself still trusts the registers it is given: an unchanged + card is only re-signed when it used half its lifetime, so clients that may + hold stale registers on start lose nothing by reconciling again after + device sync. * The privacy heuristics (`feed_url_privacy_warning`) are advisory: query parameter names containing words such as `token`, `key`, `auth`, `secret`, `sig`/`signature`, `uid`, `user`, `pass`, `session`, `access`, `private`; @@ -823,6 +837,8 @@ is not needed afterwards, and the episode catalog comes from RSS. * Only podcasts with `effective_share` are published; never-shared podcasts produce no network traffic at all. +* After a restart nothing is re-sent from the persisted cards until the + client reconciled its registers with `sync_shares` (see Publication). * Cards carry the public projection only (see Card); tombstones carry key hashes only. * Records are signed by the publisher device and verified on every receive diff --git a/crates/music-dht/README.md b/crates/music-dht/README.md index e34c778..ae59b0a 100644 --- a/crates/music-dht/README.md +++ b/crates/music-dht/README.md @@ -264,10 +264,12 @@ explicitly shared. Wiring, as for `SimilarityDht`: simply does not add it); 3. pass every peer manifest received to `note_peer_capabilities`, so peers without the overlay are not dialed; -4. call `sync_shares` with the merged subscription and share registers after - applying sync operations: it publishes cards only where `effective_share` - holds, tombstones every other own publication, and reports cards it could - not build (`failed`) or held back for privacy (`held_back`); +4. call `sync_shares` with the merged subscription and share registers on + start and after applying sync operations (until the first call, own cards + persisted by an earlier run are neither republished nor served): it + publishes cards only where `effective_share` holds, tombstones every other + own publication, and reports cards it could not build (`failed`) or held + back for privacy (`held_back`); 5. ask the user before publishing when `PodcastCardSpec::privacy_warnings` (or `feed_url_privacy_warning`) reports a URL that looks private, then call `publish` or set `PodcastShareState::privacy_confirmed`. diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs index ce8d5ba..0106728 100644 --- a/crates/music-dht/src/podcast_dht/mod.rs +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -22,6 +22,11 @@ //! * Every record is verified (bounds, network, key placement, lifetime and //! signature) when it is received and again whenever it is read from the //! cache. +//! * Own cards persisted by an earlier run are not sent again, neither +//! republished nor served to lookups, until the client reconciles with +//! [`PodcastDht::sync_shares`] in this process: a share revoked on another +//! trusted device while this one was offline is never re-sent from the +//! stale cache. //! //! Applications own the serving and maintenance tasks, exactly as for //! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with @@ -46,6 +51,7 @@ mod tests; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::Path; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; @@ -264,7 +270,10 @@ pub struct PodcastDhtSyncStats { /// Cards published with a new revision, or refreshed because they used /// half their lifetime. pub published: usize, - /// Shared podcasts whose current card was already published. + /// Shared podcasts whose current card was already published. The first + /// call in a process re-sends these cards as they are (counted in + /// [`Self::push`]), since [`PodcastDht::republish`] withholds them until + /// then. pub unchanged: usize, /// Podcasts unpublished with a tombstone. pub unpublished: usize, @@ -364,6 +373,13 @@ pub struct PodcastDht { failed: Mutex>, failures: Mutex>, publish_gate: Semaphore, + /// Set by the first [`PodcastDht::sync_shares`] of this process. Until + /// then own cards from the persisted cache are neither republished nor + /// served: the registers they were derived from may be stale. + synced: AtomicBool, + /// Podcasts published with [`PodcastDht::publish`] in this process; their + /// cards are sent even before the first `sync_shares`. + published_here: Mutex>, serve_limit: Arc, inflight: Arc>>, } @@ -412,6 +428,8 @@ impl PodcastDht { failed: Mutex::new(HashMap::new()), failures: Mutex::new(HashMap::new()), publish_gate: Semaphore::new(1), + synced: AtomicBool::new(false), + published_here: Mutex::new(HashSet::new()), serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), inflight: Arc::new(Mutex::new(HashMap::new())), })) @@ -449,7 +467,9 @@ impl PodcastDht { /// Periodically discovers overlay peers, republishes own cards and /// tombstones every [`PODCAST_REPUBLISH_INTERVAL`] and removes expired - /// replicas. The first bootstrap runs immediately. + /// replicas. The first bootstrap runs immediately. Own cards are only + /// republished once the client called [`Self::sync_shares`] in this + /// process (see [`Self::republish`]). pub async fn maintenance(self: Arc) { let mut bootstrap = tokio::time::interval(PODCAST_BOOTSTRAP_INTERVAL); let mut republish = tokio::time::interval(PODCAST_REPUBLISH_INTERVAL); @@ -621,7 +641,9 @@ impl PodcastDht { let card = card.bounded(); card.validate()?; let _permit = self.publish_permit().await?; + let podcast_id = card.podcast_id.clone(); let record = self.sign_card_locked(card, true).await?; + lock(&self.published_here).insert(podcast_id); self.push(record.into_iter().collect()).await } @@ -647,6 +669,11 @@ impl PodcastDht { /// including podcasts absent from `states`, is unpublished. The call is /// idempotent: repeating it publishes nothing new. /// + /// The first successful call in a process arms [`Self::republish`] and + /// serving of own cards: clients call it on start, after loading their + /// merged registers, and again whenever they change. That call also + /// re-sends the unchanged cards to the overlay as they are. + /// /// Each podcast is handled on its own, so revocation is fail-safe: a /// shared podcast whose card cannot be built is reported in /// [`PodcastDhtSyncStats::failed`] and unpublished, and never prevents @@ -657,6 +684,7 @@ impl PodcastDht { /// publishes the same feed URLs. pub async fn sync_shares(&self, states: Vec) -> Result { let _permit = self.publish_permit().await?; + let first_sync = !self.synced.load(Ordering::SeqCst); let local = self.database.local_publications().await?; let active: HashMap<&PodcastId, &PodcastCardSpec> = local .iter() @@ -727,33 +755,61 @@ impl PodcastDht { records.push(record); stats.published += 1; } - Ok(None) => stats.unchanged += 1, + Ok(None) => { + stats.unchanged += 1; + // Withheld by `republish` until now: send it as it is. + if first_sync + && let Some(publication) = local + .iter() + .find(|publication| publication.podcast_id == podcast_id) + { + records.push(publication.record.clone()); + } + } Err(error) => stats.failed.push(PodcastSyncFailure { podcast_id, error: error.to_string(), }), } } + // Own publications now reflect the client's registers. + self.synced.store(true, Ordering::SeqCst); stats.push = self.push(records).await?; Ok(stats) } /// Re-signs every published card with a fresh lifetime (same revision) /// and re-sends every unexpired tombstone. Called by - /// [`Self::maintenance`] every [`PODCAST_REPUBLISH_INTERVAL`]. + /// [`Self::maintenance`] every [`PODCAST_REPUBLISH_INTERVAL`] and after a + /// bootstrap that found new peers. + /// + /// Until [`Self::sync_shares`] has run in this process, only tombstones + /// and cards published with [`Self::publish`] in this process are sent: + /// the persisted cards were derived from registers that may have been + /// revoked on another trusted device meanwhile. pub async fn republish(&self) -> Result { let _permit = self.publish_permit().await?; let now = now_ms(); let mut records = Vec::new(); + let mut withheld = 0usize; for publication in self.database.local_publications().await? { match publication.spec { - Some(spec) => records.extend(self.sign_card_locked(spec, true).await?), + Some(spec) if self.may_send_card(&publication.podcast_id) => { + records.extend(self.sign_card_locked(spec, true).await?); + } + Some(_) => withheld += 1, None if publication.record.expires_at_ms() > now => { records.push(publication.record); } None => {} } } + if withheld > 0 { + debug!( + withheld, + "own podcast cards wait for the first share reconciliation" + ); + } if records.is_empty() { return Ok(PodcastDhtPublishStats::default()); } @@ -837,6 +893,13 @@ impl PodcastDht { .find(|hit| hit.podcast_id == *podcast_id)) } + /// Whether an own card of `podcast_id` may leave this device: after the + /// first [`Self::sync_shares`] of this process, or when it was published + /// with [`Self::publish`] in this process. + fn may_send_card(&self, podcast_id: &PodcastId) -> bool { + self.synced.load(Ordering::SeqCst) || lock(&self.published_here).contains(podcast_id) + } + async fn publish_permit(&self) -> Result> { self.publish_gate .acquire() @@ -1174,10 +1237,18 @@ impl PodcastDht { )), PodcastDhtOperation::FindValue { key, filter_tokens } => { validate_filter_tokens(&filter_tokens)?; - let records = self + let mut records = self .database .records_by_key(key, filter_tokens, now_ms(), MAX_PODCAST_DHT_RECORDS) .await?; + // Own cards persisted by an earlier run wait for the first + // share reconciliation, like republishing. + let own = self.own_contact.peer_id; + records.retain(|record| { + record.publisher() != own + || record.is_tombstone() + || self.may_send_card(record.podcast_id()) + }); Ok(PodcastDhtResponse::success( self.closest_contacts(key.as_bytes(), K), records, diff --git a/crates/music-dht/tests/podcast_dht.rs b/crates/music-dht/tests/podcast_dht.rs index 395f3d5..3b5ad35 100644 --- a/crates/music-dht/tests/podcast_dht.rs +++ b/crates/music-dht/tests/podcast_dht.rs @@ -20,6 +20,9 @@ //! look private (single node). //! * Contacts a peer returns are never trusted: fabricated, unreachable //! contacts do not enter the routing table (two nodes). +//! * After a restart, own cards persisted by the previous run are neither +//! republished nor served until the first `sync_shares`, so a share +//! revoked on another device meanwhile is never re-sent (two nodes). use std::path::Path; use std::sync::Arc; @@ -656,3 +659,105 @@ async fn fabricated_contacts_scenario() { m_events.abort(); m.shutdown().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restarted_device_waits_for_sync_shares_before_resending_cards() { + tokio::time::timeout(TEST_TIMEOUT, restart_scenario()) + .await + .unwrap(); +} + +async fn restart_scenario() { + let temp = tempfile::tempdir().unwrap(); + let network = NetworkId::from_name("podcast-dht-restart"); + let state = + |subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState { + subscription: subscription.clone(), + share: Some(share(subscription, shared, generation)), + card: None, + privacy_confirmed: false, + }; + let kept = subscription("kept.xml", "Kept Show"); + let revoked = subscription("revoked.xml", "Revoked Show"); + + // A publishes both podcasts while alone, then goes offline. + let a = Node::start(&temp.path().join("a"), network).await; + let stats = a + .dht + .sync_shares(vec![state(&kept, true, 2), state(&revoked, true, 2)]) + .await + .unwrap(); + assert_eq!((stats.published, stats.push.remote_nodes), (2, 0)); + a.stop().await; + + // Meanwhile another trusted device revokes "Revoked Show". A comes back + // with its persisted cards and runs maintenance before its client + // reconciled: the first bootstrap finds B. + let b = Node::start(&temp.path().join("b"), network).await; + let mut a = Node::start(&temp.path().join("a"), network).await; + b.service + .connect(a.service.ticket().await.unwrap()) + .await + .unwrap(); + wait_for("music DHT contacts", async || { + (!a.service.known_peers().is_empty()).then_some(()) + }) + .await; + a.tasks.push(tokio::spawn(Arc::clone(&a.dht).maintenance())); + wait_for("overlay contacts", async || { + (a.dht.known_peers() >= 1 && b.dht.known_peers() >= 1).then_some(()) + }) + .await; + // Neither maintenance nor an explicit republish sends a stale card, and + // A does not serve them to lookups either. + let stats = a.dht.republish().await.unwrap(); + assert_eq!(stats.records, 0, "stale cards are withheld"); + for podcast in [&kept, &revoked] { + assert!( + b.dht + .lookup_podcast(&podcast.podcast_id) + .await + .unwrap() + .is_none() + ); + } + assert!(search(&b, "show").await.is_empty()); + + // The client's first reconciliation revokes the stale share and + // re-sends the unchanged card. + let stats = a + .dht + .sync_shares(vec![state(&kept, true, 2), state(&revoked, false, 3)]) + .await + .unwrap(); + assert_eq!( + (stats.published, stats.unchanged, stats.unpublished), + (0, 1, 1) + ); + assert_eq!(stats.push.records, 2, "the kept card and the tombstone"); + assert!(stats.push.remote_nodes >= 1); + let a_only = vec![a.service.endpoint_id()]; + let hit = b + .dht + .lookup_podcast(&kept.podcast_id) + .await + .unwrap() + .expect("the kept card is found"); + assert_eq!(hit.publishers, a_only); + assert!( + b.dht + .lookup_podcast(&revoked.podcast_id) + .await + .unwrap() + .is_none() + ); + // From now on republishing covers the kept card and the tombstone. + let stats = a.dht.republish().await.unwrap(); + assert_eq!(stats.records, 2); + let hits = search(&b, "show").await; + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].podcast_id, kept.podcast_id); + + a.stop().await; + b.stop().await; +} -- 2.54.0