From d27155b25711d3984d34ecf2ee292b1120744567 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:45:52 +0200 Subject: [PATCH] Add signed public podcast discovery overlay New music_dht::podcast_dht: a schema-independent Kademlia overlay on furumi-fd/podcast-dht/1 (version 1) with its own SQLite cache. Devices publish signed public podcast cards only where effective_share holds, retract them with signed tombstones of the next revision, and search by normalized title/author tokens or podcast id. Records are verified on every receive and cache read. Adds typed federated search hits, the advisory feed_url_privacy_warning helper, the podcast_dht capability and MusicDhtError::PodcastPublication. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/capabilities.rs | 33 + crates/music-dht/src/error.rs | 5 + crates/music-dht/src/lib.rs | 4 + crates/music-dht/src/podcast_dht/card.rs | 898 +++++++++++++ crates/music-dht/src/podcast_dht/mod.rs | 1258 +++++++++++++++++++ crates/music-dht/src/podcast_dht/privacy.rs | 339 +++++ crates/music-dht/src/podcast_dht/search.rs | 238 ++++ crates/music-dht/src/podcast_dht/store.rs | 522 ++++++++ crates/music-dht/src/podcast_dht/tests.rs | 845 +++++++++++++ 9 files changed, 4142 insertions(+) create mode 100644 crates/music-dht/src/podcast_dht/card.rs create mode 100644 crates/music-dht/src/podcast_dht/mod.rs create mode 100644 crates/music-dht/src/podcast_dht/privacy.rs create mode 100644 crates/music-dht/src/podcast_dht/search.rs create mode 100644 crates/music-dht/src/podcast_dht/store.rs create mode 100644 crates/music-dht/src/podcast_dht/tests.rs diff --git a/crates/music-dht/src/capabilities.rs b/crates/music-dht/src/capabilities.rs index 94d27a6..f4525f8 100644 --- a/crates/music-dht/src/capabilities.rs +++ b/crates/music-dht/src/capabilities.rs @@ -35,6 +35,8 @@ pub const CATALOG_ID: &str = "catalog"; pub const SIMILARITY_ID: &str = "similarity"; /// Stable protocol identifier for DHT-routed similarity discovery. pub const SIMILARITY_DHT_ID: &str = "similarity_dht"; +/// Stable protocol identifier for the public podcast-discovery overlay. +pub const PODCAST_DHT_ID: &str = "podcast_dht"; /// Stable protocol identifier for personal-device synchronization. pub const DEVICE_SYNC_ID: &str = "device_sync"; /// Stable protocol identifier for Jam playback control. @@ -79,6 +81,10 @@ impl CapabilityManifest { SIMILARITY_DHT_ID.to_string(), crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION, ); + protocols.insert( + PODCAST_DHT_ID.to_string(), + crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION, + ); protocols.insert( DEVICE_SYNC_ID.to_string(), crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION, @@ -105,6 +111,18 @@ impl CapabilityManifest { self } + /// Version of protocol `id` advertised by the peer, if any. + pub fn protocol_version(&self, id: &str) -> Option { + self.protocols.get(id).copied() + } + + /// Whether the peer advertises the public podcast-discovery overlay at + /// the version this crate speaks. Peers without it are never queried. + pub fn supports_podcast_dht(&self) -> bool { + self.protocol_version(PODCAST_DHT_ID) + == Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION) + } + /// Validates bounds and identifiers received from a peer. pub fn validate(&self) -> Result<()> { if self.manifest_version != CAPABILITIES_PROTOCOL_VERSION { @@ -219,6 +237,7 @@ mod tests { CATALOG_ID, SIMILARITY_ID, SIMILARITY_DHT_ID, + PODCAST_DHT_ID, DEVICE_SYNC_ID, JAM_ID, ] { @@ -227,6 +246,20 @@ mod tests { manifest.validate().unwrap(); } + #[test] + fn podcast_dht_capability_is_advertised_and_removable() { + let manifest = CapabilityManifest::frid("furumi", "1.2.3"); + assert_eq!( + manifest.protocol_version(PODCAST_DHT_ID), + Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION) + ); + assert!(manifest.supports_podcast_dht()); + let without = manifest.clone().without_protocol(PODCAST_DHT_ID); + assert!(!without.supports_podcast_dht()); + let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2); + assert!(!other_version.supports_podcast_dht()); + } + #[test] fn application_protocol_is_additive() { let manifest = CapabilityManifest::frid("furumi", "1.2.3").with_protocol("audio", 1); diff --git a/crates/music-dht/src/error.rs b/crates/music-dht/src/error.rs index 8dee393..e1d80d6 100644 --- a/crates/music-dht/src/error.rs +++ b/crates/music-dht/src/error.rs @@ -50,6 +50,11 @@ pub enum MusicDhtError { /// The service is shutting down and no longer accepts operations. #[error("service is shutting down")] ShuttingDown, + + /// A podcast card cannot be published, for example because the user did + /// not share the podcast (see `music_dht::podcast::effective_share`). + #[error("podcast publication refused: {0}")] + PodcastPublication(String), } impl From for MusicDhtError { diff --git a/crates/music-dht/src/lib.rs b/crates/music-dht/src/lib.rs index 30447d8..7e18b1c 100644 --- a/crates/music-dht/src/lib.rs +++ b/crates/music-dht/src/lib.rs @@ -27,6 +27,9 @@ //! podcast/episode ids, metadata DTOs and convergent merge rules), with an //! optional RSS/Atom parser behind the `podcast-feed` feature; //! [`device_sync`] protocol v4 carries it between trusted devices. +//! * [`podcast_dht`] is a separate signed overlay for public podcast +//! discovery: cards a user explicitly shared, found by title/author tokens +//! or podcast id, retracted by signed tombstones. //! * Records are replicated to the `K` nodes whose ids are XOR-closest to //! each key. Publishers pick the targets from their routing table and send //! batched store requests (one pipeline per peer), so even a large library @@ -97,6 +100,7 @@ mod message; mod node; mod normalization; pub mod podcast; +pub mod podcast_dht; mod record; mod request; mod routing; diff --git a/crates/music-dht/src/podcast_dht/card.rs b/crates/music-dht/src/podcast_dht/card.rs new file mode 100644 index 0000000..2bcfd75 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/card.rs @@ -0,0 +1,898 @@ +//! Signed public podcast cards, tombstones, DHT keys and their canonical +//! signing encoding. +//! +//! Everything here is pure and deterministic: no clock, no I/O. Callers pass +//! the current time to the validation functions. + +use std::cmp::Ordering; +use std::collections::HashSet; +use std::time::Duration; + +use federation_net::{EndpointId, NetworkId, Signature}; +use serde::{Deserialize, Serialize}; + +use crate::error::{MusicDhtError, Result}; +use crate::normalization::{normalize_name, tokenize}; +use crate::podcast::{ + FeedUrl, MAX_URL_BYTES, PodcastId, SubscriptionRecord, bounded_text, is_http_url, +}; + +use super::privacy::{FeedUrlPrivacyWarning, feed_url_privacy_warning}; + +/// Current card, tombstone and overlay wire version. +pub const PODCAST_DHT_PROTOCOL_VERSION: u16 = 1; +/// Signing domain of a [`PodcastCard`]. Changing it invalidates every card. +pub const PODCAST_CARD_SIGNING_DOMAIN: &str = "furumi-podcast-card:v1"; +/// Signing domain of a [`PodcastTombstone`]. +pub const PODCAST_TOMBSTONE_SIGNING_DOMAIN: &str = "furumi-podcast-card-tombstone:v1"; +/// BLAKE3 derivation domain of [`PodcastDhtKey`]. +pub const PODCAST_DHT_KEY_DOMAIN: &str = "furumi-podcast-dht:key:v1"; + +/// Maximum lifetime of a card: the main DHT's active-record TTL. +pub const PODCAST_CARD_TTL: Duration = crate::record::ACTIVE_RECORD_TTL; +/// Maximum lifetime of a tombstone: the main DHT's tombstone TTL. It exceeds +/// [`PODCAST_CARD_TTL`], so a card issued before a tombstone always expires +/// before the tombstone does. +pub const PODCAST_TOMBSTONE_TTL: Duration = crate::record::TOMBSTONE_TTL; +/// Tolerated clock skew for `issued_at_ms` in the future. +pub const PODCAST_DHT_MAX_CLOCK_SKEW: Duration = Duration::from_secs(5 * 60); + +/// Maximum characters of a card title. +pub const MAX_CARD_TITLE_CHARS: usize = 300; +/// Maximum characters of a card author. +pub const MAX_CARD_AUTHOR_CHARS: usize = 300; +/// Maximum characters of a card description. +pub const MAX_CARD_DESCRIPTION_CHARS: usize = 1_000; +/// Maximum bytes of a card language tag (BCP 47 shape: ASCII letters, +/// digits and `-`). +pub const MAX_CARD_LANGUAGE_BYTES: usize = 35; +/// Maximum categories on a card. +pub const MAX_CARD_CATEGORIES: usize = 8; +/// Maximum characters of one category. +pub const MAX_CARD_CATEGORY_CHARS: usize = 64; +/// Maximum indexed search tokens of a card (title tokens, then author tokens). +pub const MAX_CARD_INDEX_TOKENS: usize = 32; +/// Maximum bytes of one indexed or query token. Longer tokens are not indexed. +pub const MAX_INDEX_TOKEN_BYTES: usize = 64; +/// Maximum DHT keys a record is stored under: the podcast-id key plus one +/// key per indexed token. +pub const MAX_RECORD_DHT_KEYS: usize = MAX_CARD_INDEX_TOKENS + 1; +/// Maximum postcard-encoded size of one signed record. +pub const MAX_PODCAST_RECORD_BYTES: usize = 24 * 1024; + +const KEY_KIND_PODCAST: u8 = 1; +const KEY_KIND_TOKEN: u8 = 2; + +/// A 256-bit point in the podcast-discovery key space. +/// +/// `BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind || material)` +/// with kind `0x01` and the podcast id string for the id key, or kind `0x02` +/// and a normalized token for a text key. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct PodcastDhtKey([u8; 32]); + +impl PodcastDhtKey { + /// Key under which every card and tombstone of `podcast_id` is stored. + pub fn podcast(network_id: &NetworkId, podcast_id: &PodcastId) -> Self { + Self::derive(network_id, KEY_KIND_PODCAST, podcast_id.as_str()) + } + + /// Key of one normalized search token (see [`card_index_tokens`]). + pub fn token(network_id: &NetworkId, token: &str) -> Self { + Self::derive(network_id, KEY_KIND_TOKEN, token) + } + + fn derive(network_id: &NetworkId, kind: u8, material: &str) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(PODCAST_DHT_KEY_DOMAIN.as_bytes()); + hasher.update(&[0]); + hasher.update(network_id.as_bytes()); + hasher.update(&[kind]); + hasher.update(material.as_bytes()); + Self(*hasher.finalize().as_bytes()) + } + + /// Creates a key from raw bytes received on the wire. + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Raw key bytes. + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +/// Indexed search tokens of a card: distinct [`normalize_name`] tokens of the +/// title, then of the author, in order, skipping tokens longer than +/// [`MAX_INDEX_TOKEN_BYTES`], at most [`MAX_CARD_INDEX_TOKENS`]. +pub fn card_index_tokens(title: &str, author: Option<&str>) -> Vec { + let mut seen = HashSet::new(); + let mut tokens = Vec::new(); + let sources = std::iter::once(title).chain(author); + for source in sources { + for token in tokenize(&normalize_name(source)) { + if tokens.len() == MAX_CARD_INDEX_TOKENS { + return tokens; + } + if token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()) { + tokens.push(token); + } + } + } + tokens +} + +/// Unsigned public projection of a podcast prepared by its publisher. +/// +/// It is built from the user's subscription ([`PodcastCardSpec::from_subscription`]) +/// and contains only publisher metadata: never progress, listening history, +/// subscription times, operation ids, device names or episodes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastCardSpec { + /// Podcast. + pub podcast_id: PodcastId, + /// Canonical URL the feed is currently fetched from. + pub feed_url: FeedUrl, + /// URL `podcast_id` was derived from, when it differs from `feed_url`. + pub id_feed_url: Option, + /// Podcast title. + pub title: String, + /// Author. + pub author: Option, + /// Plain-text description. + pub description: Option, + /// Artwork URL. + pub image_url: Option, + /// Publisher web page. + pub link: Option, + /// Feed language tag, for example `en` or `pt-BR`. + pub language: Option, + /// Feed categories. + pub categories: Vec, + /// Number of episodes in the feed, when known. + pub episode_count: Option, + /// Publication time of the newest episode in unix milliseconds. + pub latest_episode_at_ms: Option, +} + +impl PodcastCardSpec { + /// Builds the bounded public projection of a subscription. + /// + /// Only `podcast_id`, the feed URLs and the channel metadata are read. + /// `id_feed_url` is the alias that hashes to `podcast_id` + /// ([`crate::podcast::merge_feed_aliases`] always keeps it). Fails when no + /// such URL is known. Callers may then set `language`, `categories`, + /// `episode_count` and `latest_episode_at_ms` from their feed cache. + pub fn from_subscription(subscription: &SubscriptionRecord) -> Result { + let origin = subscription + .feed_urls() + .into_iter() + .find(|url| PodcastId::from_feed_url(url) == subscription.podcast_id) + .ok_or_else(|| { + publication_error("the feed URL the podcast id was derived from is unknown") + })?; + let feed_url = subscription.podcast.feed_url.clone(); + let meta = &subscription.podcast; + Ok(Self { + podcast_id: subscription.podcast_id.clone(), + id_feed_url: (origin != feed_url).then_some(origin), + feed_url, + title: meta.title.clone(), + author: meta.author.clone(), + description: meta.description.clone(), + image_url: meta.image_url.clone(), + link: meta.link.clone(), + language: None, + categories: Vec::new(), + episode_count: None, + latest_episode_at_ms: None, + } + .bounded()) + } + + /// Returns a copy that fits the card bounds: text is truncated with + /// [`bounded_text`], control characters are removed, invalid URLs and + /// language tags are dropped, and categories are trimmed, deduplicated and + /// bounded. An empty title becomes `"Untitled podcast"`. + pub fn bounded(&self) -> Self { + let title = clean_line(&self.title); + let mut seen = HashSet::new(); + let categories = self + .categories + .iter() + .map(|category| bounded_text(&clean_line(category), MAX_CARD_CATEGORY_CHARS)) + .filter(|category| !category.is_empty() && seen.insert(category.to_lowercase())) + .take(MAX_CARD_CATEGORIES) + .collect(); + Self { + podcast_id: self.podcast_id.clone(), + feed_url: self.feed_url.clone(), + id_feed_url: self + .id_feed_url + .clone() + .filter(|origin| *origin != self.feed_url), + title: if title.is_empty() { + "Untitled podcast".to_string() + } else { + bounded_text(&title, MAX_CARD_TITLE_CHARS) + }, + author: bounded_line(self.author.as_deref(), MAX_CARD_AUTHOR_CHARS), + description: self + .description + .as_deref() + .map(|text| bounded_text(clean_block(text).trim(), MAX_CARD_DESCRIPTION_CHARS)) + .filter(|text| !text.is_empty()), + image_url: self.image_url.clone().filter(|url| is_http_url(url)), + link: self.link.clone().filter(|url| is_http_url(url)), + language: self + .language + .as_deref() + .map(str::trim) + .filter(|language| valid_language(language)) + .map(str::to_string), + categories, + episode_count: self.episode_count, + latest_episode_at_ms: self.latest_episode_at_ms.filter(|at| *at >= 0), + } + } + + /// Validates the card bounds without bounding anything. + pub fn validate(&self) -> Result<()> { + let id_url = self.id_feed_url.as_ref().unwrap_or(&self.feed_url); + if PodcastId::from_feed_url(id_url) != self.podcast_id { + return Err(invalid("podcast id does not match its feed URL")); + } + if self.id_feed_url.as_ref() == Some(&self.feed_url) { + return Err(invalid("id_feed_url must differ from feed_url")); + } + check_line("title", &self.title, MAX_CARD_TITLE_CHARS, true)?; + if let Some(author) = &self.author { + check_line("author", author, MAX_CARD_AUTHOR_CHARS, false)?; + } + if let Some(description) = &self.description { + check_chars("description", description, MAX_CARD_DESCRIPTION_CHARS)?; + if description + .chars() + .any(|c| c.is_control() && c != '\n' && c != '\t') + { + return Err(invalid("description contains control characters")); + } + } + for url in [&self.image_url, &self.link].into_iter().flatten() { + if !is_http_url(url) { + return Err(invalid("card URLs must be absolute http(s) URLs")); + } + } + if self + .language + .as_deref() + .is_some_and(|language| !valid_language(language)) + { + return Err(invalid("invalid language tag")); + } + if self.categories.len() > MAX_CARD_CATEGORIES { + return Err(invalid("too many categories")); + } + for category in &self.categories { + check_line("category", category, MAX_CARD_CATEGORY_CHARS, true)?; + } + if self.latest_episode_at_ms.is_some_and(|at| at < 0) { + return Err(invalid("negative latest_episode_at_ms")); + } + Ok(()) + } + + /// Advisory privacy warnings for both feed URLs of the card (see + /// [`feed_url_privacy_warning`]). The UI must ask before publishing a + /// card with warnings. + pub fn privacy_warnings(&self) -> Vec { + std::iter::once(&self.feed_url) + .chain(self.id_feed_url.as_ref()) + .filter_map(feed_url_privacy_warning) + .collect() + } + + /// Indexed search tokens of this card. + pub fn index_tokens(&self) -> Vec { + card_index_tokens(&self.title, self.author.as_deref()) + } +} + +/// A signed public podcast card: one publisher device's projection of one +/// podcast. See `PODCASTS.md`, "Public discovery (podcast_dht)". +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastCard { + /// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]). + pub version: u16, + /// Network scope; prevents replay into another federation. + pub network_id: NetworkId, + /// Publisher device (its persistent transport identity); signs the card. + pub publisher: EndpointId, + /// Podcast. + pub podcast_id: PodcastId, + /// Canonical URL the feed is currently fetched from. + pub feed_url: FeedUrl, + /// URL `podcast_id` was derived from, when it differs from `feed_url`. + pub id_feed_url: Option, + /// Podcast title. + pub title: String, + /// Author. + pub author: Option, + /// Plain-text description. + pub description: Option, + /// Artwork URL. + pub image_url: Option, + /// Publisher web page. + pub link: Option, + /// Feed language tag. + pub language: Option, + /// Feed categories. + pub categories: Vec, + /// Number of episodes in the feed, when known. + pub episode_count: Option, + /// Publication time of the newest episode in unix milliseconds. + pub latest_episode_at_ms: Option, + /// Monotonic per `(podcast_id, publisher)` revision. + pub revision: u64, + /// Unix time at which this version was signed. + pub issued_at_ms: u64, + /// Unix time after which the card must be discarded. + pub expires_at_ms: u64, +} + +impl PodcastCard { + /// Builds a card from a spec. `expires_at_ms` is + /// `issued_at_ms + PODCAST_CARD_TTL`. + pub fn from_spec( + spec: PodcastCardSpec, + network_id: NetworkId, + publisher: EndpointId, + revision: u64, + issued_at_ms: u64, + ) -> Self { + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + network_id, + publisher, + podcast_id: spec.podcast_id, + feed_url: spec.feed_url, + id_feed_url: spec.id_feed_url, + title: spec.title, + author: spec.author, + description: spec.description, + image_url: spec.image_url, + link: spec.link, + language: spec.language, + categories: spec.categories, + episode_count: spec.episode_count, + latest_episode_at_ms: spec.latest_episode_at_ms, + revision, + issued_at_ms, + expires_at_ms: issued_at_ms.saturating_add(PODCAST_CARD_TTL.as_millis() as u64), + } + } + + /// The publisher-provided part of the card. + pub fn spec(&self) -> PodcastCardSpec { + PodcastCardSpec { + podcast_id: self.podcast_id.clone(), + feed_url: self.feed_url.clone(), + id_feed_url: self.id_feed_url.clone(), + title: self.title.clone(), + author: self.author.clone(), + description: self.description.clone(), + image_url: self.image_url.clone(), + link: self.link.clone(), + language: self.language.clone(), + categories: self.categories.clone(), + episode_count: self.episode_count, + latest_episode_at_ms: self.latest_episode_at_ms, + } + } + + /// Indexed search tokens. + pub fn index_tokens(&self) -> Vec { + card_index_tokens(&self.title, self.author.as_deref()) + } + + /// Canonical signing input (see `PODCASTS.md`). + pub fn signing_bytes(&self) -> Vec { + let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN); + out.u16(self.version); + out.raw(self.network_id.as_bytes()); + out.raw(self.publisher.as_bytes()); + out.str(self.podcast_id.as_str()); + out.str(self.feed_url.as_str()); + out.opt_str(self.id_feed_url.as_ref().map(FeedUrl::as_str)); + out.str(&self.title); + out.opt_str(self.author.as_deref()); + out.opt_str(self.description.as_deref()); + out.opt_str(self.image_url.as_deref()); + out.opt_str(self.link.as_deref()); + out.opt_str(self.language.as_deref()); + out.u32(self.categories.len() as u32); + for category in &self.categories { + out.str(category); + } + match self.episode_count { + Some(count) => { + out.u8(1); + out.u32(count); + } + None => out.u8(0), + } + match self.latest_episode_at_ms { + Some(at) => { + out.u8(1); + out.i64(at); + } + None => out.u8(0), + } + out.u64(self.revision); + out.u64(self.issued_at_ms); + out.u64(self.expires_at_ms); + out.finish() + } + + fn validate(&self, expected_network: &NetworkId) -> Result<()> { + check_header(self.version, &self.network_id, expected_network)?; + self.spec().validate()?; + check_lifetime( + self.revision, + self.issued_at_ms, + self.expires_at_ms, + PODCAST_CARD_TTL, + ) + } +} + +/// A signed retraction of one publisher's card of one podcast. +/// +/// It supersedes every card of the same `(podcast_id, publisher)` with a +/// revision lower than or equal to its own and is retained until it expires. +/// `index_keys` lists the text keys of the retracted cards so the tombstone +/// can be stored next to them; it carries hashes, not the tokens. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastTombstone { + /// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]). + pub version: u16, + /// Network scope. + pub network_id: NetworkId, + /// Publisher device; signs the tombstone. + pub publisher: EndpointId, + /// Podcast. + pub podcast_id: PodcastId, + /// Revision; greater than that of every retracted card. + pub revision: u64, + /// Unix time at which the tombstone was signed. + pub issued_at_ms: u64, + /// Unix time after which the tombstone may be discarded. + pub expires_at_ms: u64, + /// Sorted, distinct text keys besides the podcast-id key, at most + /// [`MAX_CARD_INDEX_TOKENS`]. + pub index_keys: Vec, +} + +impl PodcastTombstone { + /// Builds a tombstone. `expires_at_ms` is `issued_at_ms + PODCAST_TOMBSTONE_TTL`; + /// `index_keys` is sorted, deduplicated and stripped of the podcast-id key. + pub fn new( + network_id: NetworkId, + publisher: EndpointId, + podcast_id: PodcastId, + revision: u64, + issued_at_ms: u64, + index_keys: impl IntoIterator, + ) -> Self { + let id_key = PodcastDhtKey::podcast(&network_id, &podcast_id); + let mut index_keys: Vec<_> = index_keys + .into_iter() + .filter(|key| *key != id_key) + .collect(); + index_keys.sort(); + index_keys.dedup(); + index_keys.truncate(MAX_CARD_INDEX_TOKENS); + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + network_id, + publisher, + podcast_id, + revision, + issued_at_ms, + expires_at_ms: issued_at_ms.saturating_add(PODCAST_TOMBSTONE_TTL.as_millis() as u64), + index_keys, + } + } + + /// Canonical signing input (see `PODCASTS.md`). + pub fn signing_bytes(&self) -> Vec { + let mut out = Canonical::new(PODCAST_TOMBSTONE_SIGNING_DOMAIN); + out.u16(self.version); + out.raw(self.network_id.as_bytes()); + out.raw(self.publisher.as_bytes()); + out.str(self.podcast_id.as_str()); + out.u64(self.revision); + out.u64(self.issued_at_ms); + out.u64(self.expires_at_ms); + out.u32(self.index_keys.len() as u32); + for key in &self.index_keys { + out.raw(key.as_bytes()); + } + out.finish() + } + + fn validate(&self, expected_network: &NetworkId) -> Result<()> { + check_header(self.version, &self.network_id, expected_network)?; + if self.index_keys.len() > MAX_CARD_INDEX_TOKENS { + return Err(invalid("tombstone has too many keys")); + } + let id_key = PodcastDhtKey::podcast(&self.network_id, &self.podcast_id); + if self.index_keys.contains(&id_key) + || self.index_keys.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(invalid("tombstone keys are not canonical")); + } + check_lifetime( + self.revision, + self.issued_at_ms, + self.expires_at_ms, + PODCAST_TOMBSTONE_TTL, + ) + } +} + +/// Body of a signed podcast-discovery record. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PodcastRecordBody { + /// An active public card. + Card(Box), + /// A retraction. + Tombstone(PodcastTombstone), +} + +/// End-to-end signed podcast-discovery record. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SignedPodcastRecord { + /// Signed body. + pub body: PodcastRecordBody, + /// Ed25519 signature by `publisher` over [`Self::signing_bytes`]. + pub signature: Signature, +} + +impl SignedPodcastRecord { + /// Validates and signs a card. + pub fn sign_card(card: PodcastCard, signer: impl FnOnce(&[u8]) -> Signature) -> Result { + card.validate(&card.network_id)?; + let signature = signer(&card.signing_bytes()); + let record = Self { + body: PodcastRecordBody::Card(Box::new(card)), + signature, + }; + record.check_size()?; + Ok(record) + } + + /// Validates and signs a tombstone. + pub fn sign_tombstone( + tombstone: PodcastTombstone, + signer: impl FnOnce(&[u8]) -> Signature, + ) -> Result { + tombstone.validate(&tombstone.network_id)?; + let signature = signer(&tombstone.signing_bytes()); + let record = Self { + body: PodcastRecordBody::Tombstone(tombstone), + signature, + }; + record.check_size()?; + Ok(record) + } + + /// Canonical signing input of the body. + pub fn signing_bytes(&self) -> Vec { + match &self.body { + PodcastRecordBody::Card(card) => card.signing_bytes(), + PodcastRecordBody::Tombstone(tombstone) => tombstone.signing_bytes(), + } + } + + /// Validates bounds, network, lifetime shape, encoded size and the + /// publisher's signature. Does not look at the clock; see + /// [`Self::verify_at`]. + pub fn verify(&self, expected_network: &NetworkId) -> Result<()> { + match &self.body { + PodcastRecordBody::Card(card) => card.validate(expected_network)?, + PodcastRecordBody::Tombstone(tombstone) => tombstone.validate(expected_network)?, + } + self.check_size()?; + self.publisher() + .verify(&self.signing_bytes(), &self.signature) + .map_err(|_| invalid("invalid podcast record signature")) + } + + /// [`Self::verify`] plus time checks: not issued more than + /// [`PODCAST_DHT_MAX_CLOCK_SKEW`] in the future and not expired at `now_ms`. + pub fn verify_at(&self, expected_network: &NetworkId, now_ms: u64) -> Result<()> { + self.verify(expected_network)?; + let future_limit = now_ms.saturating_add(PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64); + if self.issued_at_ms() > future_limit || self.expires_at_ms() <= now_ms { + return Err(invalid("podcast record is expired or from the future")); + } + Ok(()) + } + + /// The card, when this is not a tombstone. + pub fn card(&self) -> Option<&PodcastCard> { + match &self.body { + PodcastRecordBody::Card(card) => Some(card.as_ref()), + PodcastRecordBody::Tombstone(_) => None, + } + } + + /// Whether this is a tombstone. + pub fn is_tombstone(&self) -> bool { + matches!(self.body, PodcastRecordBody::Tombstone(_)) + } + + /// Podcast. + pub fn podcast_id(&self) -> &PodcastId { + match &self.body { + PodcastRecordBody::Card(card) => &card.podcast_id, + PodcastRecordBody::Tombstone(tombstone) => &tombstone.podcast_id, + } + } + + /// Publisher device. + pub fn publisher(&self) -> EndpointId { + match &self.body { + PodcastRecordBody::Card(card) => card.publisher, + PodcastRecordBody::Tombstone(tombstone) => tombstone.publisher, + } + } + + /// Network scope. + pub fn network_id(&self) -> NetworkId { + match &self.body { + PodcastRecordBody::Card(card) => card.network_id, + PodcastRecordBody::Tombstone(tombstone) => tombstone.network_id, + } + } + + /// Revision. + pub fn revision(&self) -> u64 { + match &self.body { + PodcastRecordBody::Card(card) => card.revision, + PodcastRecordBody::Tombstone(tombstone) => tombstone.revision, + } + } + + /// Signing time. + pub fn issued_at_ms(&self) -> u64 { + match &self.body { + PodcastRecordBody::Card(card) => card.issued_at_ms, + PodcastRecordBody::Tombstone(tombstone) => tombstone.issued_at_ms, + } + } + + /// Expiry time. + pub fn expires_at_ms(&self) -> u64 { + match &self.body { + PodcastRecordBody::Card(card) => card.expires_at_ms, + PodcastRecordBody::Tombstone(tombstone) => tombstone.expires_at_ms, + } + } + + /// Every DHT key this record belongs under: the podcast-id key, then the + /// text keys (card tokens or the tombstone's `index_keys`). + pub fn dht_keys(&self) -> Vec { + let network_id = self.network_id(); + let mut keys = vec![PodcastDhtKey::podcast(&network_id, self.podcast_id())]; + match &self.body { + PodcastRecordBody::Card(card) => keys.extend( + card.index_tokens() + .iter() + .map(|token| PodcastDhtKey::token(&network_id, token)), + ), + PodcastRecordBody::Tombstone(tombstone) => { + keys.extend(tombstone.index_keys.iter().copied()); + } + } + keys + } + + /// Whether the record may be stored under `key`. + pub fn belongs_under(&self, key: &PodcastDhtKey) -> bool { + self.dht_keys().contains(key) + } + + /// postcard-encoded size. + pub fn encoded_len(&self) -> usize { + postcard::to_stdvec(self).map_or(usize::MAX, |bytes| bytes.len()) + } + + fn check_size(&self) -> Result<()> { + if self.encoded_len() > MAX_PODCAST_RECORD_BYTES { + Err(invalid("podcast record is too large")) + } else { + Ok(()) + } + } +} + +/// Total order of two records of the same `(podcast_id, publisher)`. +/// +/// Greater wins: higher `revision`; at an equal revision a tombstone beats a +/// card; then later `issued_at_ms` (a republish); then later +/// `expires_at_ms`; then the lexicographically greater canonical signing +/// input. Every node therefore keeps the same winner whatever the delivery +/// order. +pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) -> Ordering { + left.revision() + .cmp(&right.revision()) + .then_with(|| left.is_tombstone().cmp(&right.is_tombstone())) + .then_with(|| left.issued_at_ms().cmp(&right.issued_at_ms())) + .then_with(|| left.expires_at_ms().cmp(&right.expires_at_ms())) + .then_with(|| left.signing_bytes().cmp(&right.signing_bytes())) +} + +/// Whether `winner` makes `other` obsolete everywhere, not just under one key: +/// `other` is a card of a lower revision, or `winner` is a tombstone of an +/// equal or higher revision. +pub(crate) fn supersedes_everywhere( + winner: &SignedPodcastRecord, + other: &SignedPodcastRecord, +) -> bool { + !other.is_tombstone() + && (other.revision() < winner.revision() + || (winner.is_tombstone() && other.revision() <= winner.revision())) +} + +struct Canonical(Vec); + +impl Canonical { + fn new(domain: &str) -> Self { + let mut bytes = Vec::with_capacity(512); + bytes.extend_from_slice(domain.as_bytes()); + bytes.push(0); + Self(bytes) + } + + fn raw(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } + + fn u8(&mut self, value: u8) { + self.0.push(value); + } + + fn u16(&mut self, value: u16) { + self.raw(&value.to_le_bytes()); + } + + fn u32(&mut self, value: u32) { + self.raw(&value.to_le_bytes()); + } + + fn u64(&mut self, value: u64) { + self.raw(&value.to_le_bytes()); + } + + fn i64(&mut self, value: i64) { + self.raw(&value.to_le_bytes()); + } + + fn str(&mut self, value: &str) { + self.u32(value.len() as u32); + self.raw(value.as_bytes()); + } + + fn opt_str(&mut self, value: Option<&str>) { + match value { + Some(value) => { + self.u8(1); + self.str(value); + } + None => self.u8(0), + } + } + + fn finish(self) -> Vec { + self.0 + } +} + +fn check_header(version: u16, network_id: &NetworkId, expected: &NetworkId) -> Result<()> { + if version != PODCAST_DHT_PROTOCOL_VERSION { + return Err(invalid("unsupported podcast record version")); + } + if network_id != expected { + return Err(invalid("podcast record belongs to another network")); + } + Ok(()) +} + +fn check_lifetime( + revision: u64, + issued_at_ms: u64, + expires_at_ms: u64, + ttl: Duration, +) -> Result<()> { + if revision == 0 + || issued_at_ms == 0 + || expires_at_ms <= issued_at_ms + || expires_at_ms - issued_at_ms > ttl.as_millis() as u64 + { + return Err(invalid("invalid podcast record revision or lifetime")); + } + Ok(()) +} + +fn check_chars(field: &'static str, value: &str, max_chars: usize) -> Result<()> { + if value.len() > max_chars * 4 || value.chars().count() > max_chars { + return Err(MusicDhtError::Protocol(format!("card {field} is too long"))); + } + Ok(()) +} + +fn check_line(field: &'static str, value: &str, max_chars: usize, required: bool) -> Result<()> { + if required && value.trim().is_empty() { + return Err(MusicDhtError::Protocol(format!("card {field} is empty"))); + } + check_chars(field, value, max_chars)?; + if value.chars().any(char::is_control) { + return Err(MusicDhtError::Protocol(format!( + "card {field} contains control characters" + ))); + } + Ok(()) +} + +fn valid_language(language: &str) -> bool { + !language.is_empty() + && language.len() <= MAX_CARD_LANGUAGE_BYTES + && language + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + +fn clean_line(text: &str) -> String { + text.split(char::is_control) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" ") + .trim() + .to_string() +} + +fn clean_block(text: &str) -> String { + text.chars() + .filter(|c| !c.is_control() || *c == '\n' || *c == '\t') + .collect() +} + +fn bounded_line(text: Option<&str>, max_chars: usize) -> Option { + text.map(clean_line) + .filter(|text| !text.is_empty()) + .map(|text| bounded_text(&text, max_chars)) +} + +fn invalid(reason: &str) -> MusicDhtError { + MusicDhtError::Protocol(reason.to_string()) +} + +pub(crate) fn publication_error(reason: &str) -> MusicDhtError { + MusicDhtError::PodcastPublication(reason.to_string()) +} + +// Statically bound the worst-case card so `MAX_PODCAST_RECORD_BYTES` is a +// defensive limit, not the one users hit first. +const _: () = assert!( + MAX_CARD_TITLE_CHARS * 4 + + MAX_CARD_AUTHOR_CHARS * 4 + + MAX_CARD_DESCRIPTION_CHARS * 4 + + MAX_CARD_CATEGORIES * MAX_CARD_CATEGORY_CHARS * 4 + + MAX_CARD_LANGUAGE_BYTES + + 4 * MAX_URL_BYTES + + 1024 + <= MAX_PODCAST_RECORD_BYTES +); diff --git a/crates/music-dht/src/podcast_dht/mod.rs b/crates/music-dht/src/podcast_dht/mod.rs new file mode 100644 index 0000000..276b897 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/mod.rs @@ -0,0 +1,1258 @@ +//! Public podcast discovery: signed podcast cards in a separate DHT overlay. +//! +//! This overlay implements decision R7 of the podcast plan and the +//! "Public discovery (podcast_dht)" section of `PODCASTS.md`. It follows the +//! pattern of [`crate::similarity_dht`]: a schema-independent Kademlia +//! overlay on its own ALPN ([`PODCAST_DHT_ALPN`]) carried by `music-dht` byte +//! streams, with its own wire version, its own SQLite cache and its own +//! routing contacts. The music DHT schema and `federation-net` are unchanged. +//! +//! * A [`PodcastCard`] is the public projection of one podcast published by +//! one device: feed URL and channel metadata only, signed by the device's +//! persistent identity over a domain-separated canonical encoding. +//! Progress, history, subscription registers and device names never enter +//! a card. +//! * Cards are stored under the podcast-id key and one key per normalized +//! title/author token, so peers outside the publisher's trusted-device group +//! find them by text or by id. +//! * A device publishes only podcasts for which [`effective_share`] holds +//! ([`PodcastDht::publish`], [`PodcastDht::sync_shares`]). Unpublishing +//! writes a signed [`PodcastTombstone`] with the next revision. Nothing is +//! ever sent for a podcast this device never published. +//! * Every record is verified (bounds, network, key placement, lifetime and +//! signature) when it is received and again whenever it is read from the +//! cache. +//! +//! Applications own the serving and maintenance tasks, exactly as for +//! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with +//! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take +//! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`]. + +mod card; +mod privacy; +mod search; +mod store; + +#[cfg(test)] +mod tests; + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use federation_net::{ByteStream, EndpointId, NetworkId, PeerTicket, StreamAcceptor}; +use futures::stream::{self, FuturesUnordered, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Semaphore; +use tracing::{debug, info, warn}; + +use crate::MusicDhtService; +use crate::capabilities::CapabilityManifest; +use crate::error::{MusicDhtError, Result}; +use crate::podcast::{PodcastId, ShareRecord, SubscriptionRecord, effective_share}; +use crate::record::now_ms; +use crate::routing::{ALPHA, K, MAX_LOOKUP_REQUESTS, NodeContact, NodeId, RoutingTable, distance}; + +pub use card::{ + MAX_CARD_AUTHOR_CHARS, MAX_CARD_CATEGORIES, MAX_CARD_CATEGORY_CHARS, + MAX_CARD_DESCRIPTION_CHARS, MAX_CARD_INDEX_TOKENS, MAX_CARD_LANGUAGE_BYTES, + MAX_CARD_TITLE_CHARS, MAX_INDEX_TOKEN_BYTES, MAX_PODCAST_RECORD_BYTES, MAX_RECORD_DHT_KEYS, + PODCAST_CARD_SIGNING_DOMAIN, PODCAST_CARD_TTL, PODCAST_DHT_KEY_DOMAIN, + PODCAST_DHT_MAX_CLOCK_SKEW, PODCAST_DHT_PROTOCOL_VERSION, PODCAST_TOMBSTONE_SIGNING_DOMAIN, + PODCAST_TOMBSTONE_TTL, PodcastCard, PodcastCardSpec, PodcastDhtKey, PodcastRecordBody, + PodcastTombstone, SignedPodcastRecord, card_index_tokens, compare_records, +}; +pub use privacy::{ + FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning, +}; +pub use search::{ + FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, + SearchHitKind, merge_search_results, podcast_query_tokens, +}; +pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION}; + +use card::publication_error; +use search::{aggregate_hits, rank_hits}; +use store::{LocalPublication, PodcastDhtDatabase}; + +/// ALPN of the podcast-discovery overlay. +pub const PODCAST_DHT_ALPN: &[u8] = b"furumi-fd/podcast-dht/1"; +/// Stable identifier advertised in capability manifests. +pub const PODCAST_DHT_ID: &str = "podcast_dht"; +/// Maximum records returned for one key. +pub const MAX_PODCAST_DHT_RECORDS: usize = 128; +/// Maximum entries in one store request. +pub const MAX_PODCAST_STORE_RECORDS: usize = 16; +/// Maximum encoded request. A full store batch of worst-case records fits. +pub const MAX_PODCAST_DHT_REQUEST_BYTES: usize = 512 * 1024; +/// Maximum encoded response. A full key of worst-case records fits. +pub const MAX_PODCAST_DHT_RESPONSE_BYTES: usize = 4 * 1024 * 1024; +/// Maximum hits returned by one search. +pub const MAX_PODCAST_SEARCH_RESULTS: usize = 64; +/// Text keys looked up per search: the longest query tokens. Every replica +/// filters by all query tokens, so one key is enough; three add redundancy. +pub const MAX_PODCAST_QUERY_LOOKUPS: usize = 3; +/// Republish cadence of own cards and tombstones: the main DHT's +/// [`crate::DEFAULT_REPUBLISH_INTERVAL`]. +pub const PODCAST_REPUBLISH_INTERVAL: Duration = crate::config::DEFAULT_REPUBLISH_INTERVAL; +/// Replica expiry sweep: the main DHT's [`crate::DEFAULT_EXPIRE_INTERVAL`]. +pub const PODCAST_EXPIRE_INTERVAL: Duration = crate::config::DEFAULT_EXPIRE_INTERVAL; +/// Periodic bootstrap for discovering upgraded peers. +pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60); +/// Maximum inbound requests processed concurrently; further streams are +/// dropped. +pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64; +/// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is +/// not contacted. +pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60); + +const MAX_BOOTSTRAP_PEERS: usize = 64; +const BOOTSTRAP_CONCURRENCY: usize = 8; +const PUBLISH_CONCURRENCY: usize = 8; +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const SERVE_TIMEOUT: Duration = Duration::from_secs(10); +/// A peer that failed to answer (network error or timeout) is skipped by +/// lookups, publication and bootstrap for this long. +const FAILED_PEER_BACKOFF: Duration = Duration::from_secs(30); +const MAX_TICKET_BYTES: usize = 16 * 1024; +const MAX_ERROR_BYTES: usize = 1024; + +/// One request on [`PODCAST_DHT_ALPN`] (postcard, one per stream). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastDhtRequest { + /// [`PODCAST_DHT_PROTOCOL_VERSION`]. + pub version: u16, + /// Requesting overlay node; must match the authenticated peer. + pub requester: NodeContact, + /// Operation. + pub operation: PodcastDhtOperation, +} + +/// Overlay operations. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PodcastDhtOperation { + /// Returns the `K` closest overlay contacts to `target`. + FindNode { + /// Target id. + target: [u8; 32], + }, + /// Returns records stored under `key` and the closest contacts. + FindValue { + /// Key. + key: PodcastDhtKey, + /// Normalized tokens every returned card must be indexed under + /// (tombstones are always returned); at most + /// [`MAX_PODCAST_QUERY_TOKENS`]. + filter_tokens: Vec, + }, + /// Stores records; at most [`MAX_PODCAST_STORE_RECORDS`] entries. + StoreBatch { + /// Entries. + records: Vec, + }, +} + +/// A record and the keys it is to be stored under on the receiving node. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastStoreEntry { + /// Distinct keys the record belongs under, at most [`MAX_RECORD_DHT_KEYS`]. + pub keys: Vec, + /// Signed record. + pub record: SignedPodcastRecord, +} + +/// Response to a [`PodcastDhtRequest`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastDhtResponse { + /// [`PODCAST_DHT_PROTOCOL_VERSION`]. + pub version: u16, + /// Whether the request was accepted. + pub ok: bool, + /// Diagnostic when `ok` is false, at most 1 KiB. + pub error: Option, + /// Closest overlay contacts, at most `K`. + pub nodes: Vec, + /// Records, at most [`MAX_PODCAST_DHT_RECORDS`]. + pub records: Vec, + /// `(key, record)` rows stored by a `StoreBatch`. + pub stored: u32, +} + +impl PodcastDhtResponse { + fn success(nodes: Vec, records: Vec, stored: u32) -> Self { + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: true, + error: None, + nodes, + records, + stored, + } + } + + fn error(message: impl Into) -> Self { + let mut message = message.into(); + if message.len() > MAX_ERROR_BYTES { + let mut end = MAX_ERROR_BYTES; + while !message.is_char_boundary(end) { + end -= 1; + } + message.truncate(end); + } + Self { + version: PODCAST_DHT_PROTOCOL_VERSION, + ok: false, + error: Some(message), + nodes: Vec::new(), + records: Vec::new(), + stored: 0, + } + } +} + +/// Statistics of one publication round. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PodcastDhtPublishStats { + /// Signed records pushed. + pub records: usize, + /// `(key, record)` placements pushed. + pub keys: usize, + /// Remote overlay nodes that stored at least one placement. + pub remote_nodes: usize, + /// Whether this node kept at least one replica itself. + pub local_replica: bool, +} + +/// Result of [`PodcastDht::sync_shares`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PodcastDhtSyncStats { + /// Cards published with a new revision, or refreshed because they used + /// half their lifetime. + pub published: usize, + /// Shared podcasts whose current card was already published. + pub unchanged: usize, + /// Podcasts unpublished with a tombstone. + pub unpublished: usize, + /// Network statistics of the pushed records. + pub push: PodcastDhtPublishStats, +} + +/// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastShareState { + /// Merged subscription register. + pub subscription: SubscriptionRecord, + /// Merged share register, if any. + pub share: Option, + /// Card to publish when shared; `None` derives it with + /// [`PodcastCardSpec::from_subscription`]. + pub card: Option, +} + +impl PodcastShareState { + /// Whether the podcast must be published ([`effective_share`]). + pub fn is_shared(&self) -> bool { + effective_share(Some(&self.subscription), self.share.as_ref()) + } +} + +/// This device's publication state of one podcast. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublishedPodcast { + /// Podcast. + pub podcast_id: PodcastId, + /// Latest revision used by this device. + pub revision: u64, + /// Whether a card is published (false after unpublishing). + pub active: bool, + /// Latest signed record (card or tombstone). + pub record: SignedPodcastRecord, +} + +/// Podcast-discovery overlay attached to a [`MusicDhtService`]. +pub struct PodcastDht { + service: Arc, + database: PodcastDhtDatabase, + network_id: NetworkId, + own_contact: NodeContact, + routing: Mutex, + unsupported: Mutex>, + failed: Mutex>, + publish_gate: Semaphore, + serve_limit: Arc, +} + +impl std::fmt::Debug for PodcastDht { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PodcastDht") + .field("endpoint_id", &self.own_contact.peer_id) + .field("known_peers", &lock(&self.routing).len()) + .finish_non_exhaustive() + } +} + +impl PodcastDht { + /// Opens the overlay cache at `database_path` and restores previously + /// learned overlay contacts. The caller owns the serving and maintenance + /// tasks so they share the application's shutdown lifecycle. + pub async fn open( + service: Arc, + database_path: impl AsRef, + ) -> Result> { + let network_id = service.network_id(); + let database = PodcastDhtDatabase::open(database_path.as_ref(), network_id).await?; + let ticket = service.ticket().await?.to_string(); + let owner = service.endpoint_id(); + let own_contact = NodeContact { + node_id: NodeId::from_endpoint(&owner), + peer_id: owner, + ticket, + last_seen_ms: now_ms(), + }; + let mut routing = RoutingTable::new(own_contact.node_id); + for contact in database.load_peers().await? { + if validate_contact(&contact, &network_id, Some(owner)).is_ok() { + routing.upsert(contact); + } + } + Ok(Arc::new(Self { + service, + database, + network_id, + own_contact, + routing: Mutex::new(routing), + unsupported: Mutex::new(HashMap::new()), + failed: Mutex::new(HashMap::new()), + publish_gate: Semaphore::new(1), + serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)), + })) + } + + /// Number of known peers that implement the overlay. + pub fn known_peers(&self) -> usize { + lock(&self.routing).len() + } + + /// Serves authenticated overlay requests until `acceptor` closes. + pub async fn serve(self: Arc, mut acceptor: StreamAcceptor) { + while let Some(stream) = acceptor.accept().await { + let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else { + debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded"); + continue; + }; + let this = Arc::clone(&self); + tokio::spawn(async move { + let _permit = permit; + let peer = stream.peer_id; + match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await { + Ok(Ok(())) => {} + Ok(Err(error)) => debug!(%peer, %error, "podcast DHT request failed"), + Err(_) => debug!(%peer, "podcast DHT request timed out"), + } + }); + } + } + + /// Periodically discovers overlay peers, republishes own cards and + /// tombstones every [`PODCAST_REPUBLISH_INTERVAL`] and removes expired + /// replicas. The first bootstrap runs immediately. + pub async fn maintenance(self: Arc) { + let mut bootstrap = tokio::time::interval(PODCAST_BOOTSTRAP_INTERVAL); + let mut republish = tokio::time::interval(PODCAST_REPUBLISH_INTERVAL); + let mut expire = tokio::time::interval(PODCAST_EXPIRE_INTERVAL); + bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + republish.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + expire.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + republish.tick().await; + loop { + tokio::select! { + _ = bootstrap.tick() => { + let before = self.known_peers(); + if let Err(error) = self.bootstrap().await { + debug!(%error, "podcast DHT bootstrap failed"); + } else if self.known_peers() > before + && let Err(error) = self.republish().await + { + warn!(%error, "podcast DHT publication after bootstrap failed"); + } + } + _ = republish.tick() => { + if let Err(error) = self.republish().await { + warn!(%error, "podcast DHT republish failed"); + } + } + _ = expire.tick() => { + match self.database.delete_expired(now_ms()).await { + Ok(0) => {} + Ok(count) => debug!(count, "expired podcast DHT replicas removed"), + Err(error) => warn!(%error, "podcast DHT expiry sweep failed"), + } + } + } + } + } + + /// Records what a peer's capability manifest says about this overlay. + /// + /// A peer whose manifest lacks [`PODCAST_DHT_ID`] at + /// [`PODCAST_DHT_PROTOCOL_VERSION`] is removed from the overlay routing + /// table and not contacted for [`UNSUPPORTED_PEER_BACKOFF`]. Peers only + /// enter the routing table after answering on [`PODCAST_DHT_ALPN`] or + /// being returned by an overlay peer, so a peer without the overlay is + /// never queried for records or asked to store them. + pub async fn note_peer_capabilities( + &self, + peer: EndpointId, + manifest: &CapabilityManifest, + ) -> Result<()> { + if manifest.supports_podcast_dht() { + lock(&self.unsupported).remove(&peer); + return Ok(()); + } + let until = now_ms().saturating_add(UNSUPPORTED_PEER_BACKOFF.as_millis() as u64); + lock(&self.unsupported).insert(peer, until); + lock(&self.routing).remove(&peer); + self.database.remove_peer(peer).await + } + + /// Discovers overlay peers among the routing table and the music DHT + /// contacts. + pub async fn bootstrap(&self) -> Result<()> { + let known_feature: HashSet = lock(&self.routing) + .contacts() + .into_iter() + .map(|contact| contact.peer_id) + .collect(); + let mut candidates = lock(&self.routing).contacts(); + candidates.extend( + self.service + .known_peers() + .into_iter() + .filter(|contact| !known_feature.contains(&contact.peer_id)), + ); + let mut seen = HashSet::new(); + candidates.retain(|contact| { + contact.peer_id != self.own_contact.peer_id + && !self.is_unsupported(&contact.peer_id) + && !self.is_backed_off(&contact.peer_id) + && seen.insert(contact.peer_id) + }); + candidates.truncate(MAX_BOOTSTRAP_PEERS); + let target = *self.own_contact.node_id.as_bytes(); + let responses = stream::iter(candidates.into_iter().map(|contact| async move { + let response = self + .exchange(&contact, PodcastDhtOperation::FindNode { target }) + .await; + (contact, response) + })) + .buffer_unordered(BOOTSTRAP_CONCURRENCY) + .collect::>() + .await; + for (contact, response) in responses { + let response = match response { + Ok(response) => response, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT bootstrap peer unavailable"); + continue; + } + }; + self.learn_contact(contact).await?; + self.learn_response_nodes(response.nodes).await; + } + Ok(()) + } + + /// Publishes (or refreshes) this device's card of a podcast. + /// + /// Refused with [`MusicDhtError::PodcastPublication`] unless + /// [`effective_share`] holds for the given registers and they and `card` + /// describe the same podcast. `card` is bounded with + /// [`PodcastCardSpec::bounded`]. A changed card gets revision + /// `max(previous + 1, now_ms)`; an unchanged one keeps its revision and + /// is re-signed with a fresh lifetime. + /// + /// Call [`feed_url_privacy_warning`] (or + /// [`PodcastCardSpec::privacy_warnings`]) first and ask the user before + /// publishing a URL that looks private. + pub async fn publish( + &self, + subscription: &SubscriptionRecord, + share: &ShareRecord, + card: PodcastCardSpec, + ) -> Result { + if card.podcast_id != subscription.podcast_id { + return Err(publication_error( + "card and subscription describe different podcasts", + )); + } + if !effective_share(Some(subscription), Some(share)) { + return Err(publication_error("the podcast is not shared")); + } + let card = card.bounded(); + card.validate()?; + let _permit = self.publish_permit().await?; + let record = self.sign_card_locked(card, true).await?; + self.push(record.into_iter().collect()).await + } + + /// Unpublishes this device's card of `podcast_id` with a tombstone of + /// revision `previous + 1`, retained by replicas for + /// [`PODCAST_TOMBSTONE_TTL`]. + /// + /// Does nothing, and sends nothing, when this device never published the + /// podcast; re-sends the existing tombstone when already unpublished. + pub async fn unpublish(&self, podcast_id: &PodcastId) -> Result { + let _permit = self.publish_permit().await?; + let record = self.unpublish_locked(podcast_id).await?; + self.push(record.into_iter().collect()).await + } + + /// Declaratively reconciles this device's publications with the merged + /// personal registers. + /// + /// Podcasts for which [`effective_share`] holds are published when their + /// card changed, is not published yet or has used half its lifetime; + /// otherwise they count as unchanged (periodic refresh is + /// [`Self::republish`]). Every other podcast this device published, + /// including podcasts absent from `states`, is unpublished. The call is + /// idempotent: repeating it publishes nothing new. + pub async fn sync_shares(&self, states: Vec) -> Result { + let mut desired: HashMap = HashMap::new(); + for state in states { + if !state.is_shared() { + continue; + } + let card = match state.card { + Some(card) => card.bounded(), + None => PodcastCardSpec::from_subscription(&state.subscription)?, + }; + if card.podcast_id != state.subscription.podcast_id { + return Err(publication_error( + "card and subscription describe different podcasts", + )); + } + card.validate()?; + desired.insert(card.podcast_id.clone(), card); + } + + let _permit = self.publish_permit().await?; + let local = self.database.local_publications().await?; + let mut stats = PodcastDhtSyncStats::default(); + let mut records = Vec::new(); + for publication in &local { + if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) { + records.extend(self.unpublish_locked(&publication.podcast_id).await?); + stats.unpublished += 1; + } + } + let mut desired: Vec<_> = desired.into_values().collect(); + desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id)); + for card in desired { + match self.sign_card_locked(card, false).await? { + Some(record) => { + records.push(record); + stats.published += 1; + } + None => stats.unchanged += 1, + } + } + stats.push = self.push(records).await?; + Ok(stats) + } + + /// Re-signs every published card with a fresh lifetime (same revision) + /// and re-sends every unexpired tombstone. Called by + /// [`Self::maintenance`] every [`PODCAST_REPUBLISH_INTERVAL`]. + pub async fn republish(&self) -> Result { + let _permit = self.publish_permit().await?; + let now = now_ms(); + let mut records = Vec::new(); + for publication in self.database.local_publications().await? { + match publication.spec { + Some(spec) => records.extend(self.sign_card_locked(spec, true).await?), + None if publication.record.expires_at_ms() > now => { + records.push(publication.record); + } + None => {} + } + } + if records.is_empty() { + return Ok(PodcastDhtPublishStats::default()); + } + let stats = self.push(records).await?; + info!( + records = stats.records, + nodes = stats.remote_nodes, + "podcast DHT records republished" + ); + Ok(stats) + } + + /// This device's publication state, sorted by podcast id. + pub async fn published(&self) -> Result> { + Ok(self + .database + .local_publications() + .await? + .into_iter() + .map(|publication| PublishedPodcast { + podcast_id: publication.podcast_id, + revision: publication.revision, + active: publication.spec.is_some(), + record: publication.record, + }) + .collect()) + } + + /// Finds public podcasts whose title or author tokens contain every + /// normalized token of `query` ([`podcast_query_tokens`]). + /// + /// Hits are merged by `podcast_id` over publishers and ranked: exact + /// normalized title first, then more publishers. Partial results are + /// returned when peers fail. + pub async fn search(&self, query: &str, limit: usize) -> Result> { + let tokens = podcast_query_tokens(query); + if tokens.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + if self.known_peers() == 0 { + self.bootstrap().await?; + } + let mut lookup_tokens = tokens.clone(); + lookup_tokens.sort_by(|left, right| right.len().cmp(&left.len()).then(left.cmp(right))); + lookup_tokens.truncate(MAX_PODCAST_QUERY_LOOKUPS); + let outcomes = stream::iter(lookup_tokens.into_iter().map(|token| { + let filter = tokens.clone(); + async move { + self.lookup(PodcastDhtKey::token(&self.network_id, &token), filter) + .await + } + })) + .buffer_unordered(MAX_PODCAST_QUERY_LOOKUPS) + .collect::>() + .await; + let mut records = Vec::new(); + for outcome in outcomes { + match outcome { + Ok(found) => records.extend(found), + Err(error) => debug!(%error, "podcast DHT text lookup failed"), + } + } + let mut hits = rank_hits(aggregate_hits(records), query, &tokens); + hits.truncate(limit.min(MAX_PODCAST_SEARCH_RESULTS)); + Ok(hits) + } + + /// Finds the public cards of one podcast. + pub async fn lookup_podcast(&self, podcast_id: &PodcastId) -> Result> { + if self.known_peers() == 0 { + self.bootstrap().await?; + } + let records = self + .lookup( + PodcastDhtKey::podcast(&self.network_id, podcast_id), + Vec::new(), + ) + .await?; + Ok(aggregate_hits(records) + .into_iter() + .find(|hit| hit.podcast_id == *podcast_id)) + } + + async fn publish_permit(&self) -> Result> { + self.publish_gate + .acquire() + .await + .map_err(|_| publication_error("podcast DHT publisher closed")) + } + + /// Signs and records a card. With `force` false, an unchanged active card + /// that is valid for more than half its TTL is left alone and `None` is + /// returned. The publish gate must be held. + async fn sign_card_locked( + &self, + spec: PodcastCardSpec, + force: bool, + ) -> Result> { + let now = now_ms(); + let previous = self + .database + .local_publication(spec.podcast_id.clone()) + .await?; + let revision = match &previous { + Some(previous) if previous.spec.as_ref() == Some(&spec) => { + // Unchanged and still fresh: leave it to the republish loop. + let fresh_until = now.saturating_add(PODCAST_CARD_TTL.as_millis() as u64 / 2); + if !force && previous.record.expires_at_ms() > fresh_until { + return Ok(None); + } + previous.revision + } + Some(previous) => previous.revision.saturating_add(1).max(now), + None => now.max(1), + }; + let card = PodcastCard::from_spec( + spec.clone(), + self.network_id, + self.own_contact.peer_id, + revision, + now, + ); + let record = + SignedPodcastRecord::sign_card(card, |bytes| self.service.sign_identity(bytes))?; + self.database + .put_local_publication( + LocalPublication { + podcast_id: spec.podcast_id.clone(), + revision, + spec: Some(spec), + record: record.clone(), + }, + now, + ) + .await?; + Ok(Some(record)) + } + + /// Builds the tombstone of an own publication. The publish gate must be + /// held. + async fn unpublish_locked( + &self, + podcast_id: &PodcastId, + ) -> Result> { + let now = now_ms(); + let Some(previous) = self.database.local_publication(podcast_id.clone()).await? else { + return Ok(None); + }; + if previous.spec.is_none() { + return Ok( + (previous.record.is_tombstone() && previous.record.expires_at_ms() > now) + .then_some(previous.record), + ); + } + let revision = previous.revision.saturating_add(1); + let tombstone = PodcastTombstone::new( + self.network_id, + self.own_contact.peer_id, + podcast_id.clone(), + revision, + now, + previous.record.dht_keys(), + ); + let record = SignedPodcastRecord::sign_tombstone(tombstone, |bytes| { + self.service.sign_identity(bytes) + })?; + self.database + .put_local_publication( + LocalPublication { + podcast_id: podcast_id.clone(), + revision, + spec: None, + record: record.clone(), + }, + now, + ) + .await?; + Ok(Some(record)) + } + + /// Stores records on the `K` closest overlay nodes of each of their keys + /// (including this node when it is among them). + async fn push(&self, records: Vec) -> Result { + let mut stats = PodcastDhtPublishStats { + records: records.len(), + ..PodcastDhtPublishStats::default() + }; + if records.is_empty() { + return Ok(stats); + } + let mut local = Vec::new(); + let mut remote: HashMap)> = HashMap::new(); + for record in records { + let mut local_keys = Vec::new(); + let mut remote_keys: HashMap)> = + HashMap::new(); + for key in record.dht_keys() { + stats.keys += 1; + let (store_local, contacts) = self.replica_targets(&key); + if store_local { + local_keys.push(key); + } + for contact in contacts { + remote_keys + .entry(contact.peer_id) + .or_insert_with(|| (contact, Vec::new())) + .1 + .push(key); + } + } + if !local_keys.is_empty() { + local.push((local_keys, record.clone())); + } + for (peer, (contact, keys)) in remote_keys { + remote + .entry(peer) + .or_insert_with(|| (contact, Vec::new())) + .1 + .push(PodcastStoreEntry { + keys, + record: record.clone(), + }); + } + } + if !local.is_empty() { + self.database.store(local, now_ms()).await?; + stats.local_replica = true; + } + let sends = stream::iter(remote.into_values().map(|(contact, entries)| async move { + let mut accepted = false; + for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) { + let response = self + .exchange( + &contact, + PodcastDhtOperation::StoreBatch { + records: batch.to_vec(), + }, + ) + .await; + match response { + Ok(response) => accepted |= response.stored > 0, + Err(error) => { + debug!(peer = %contact.peer_id, %error, "podcast DHT store failed"); + break; + } + } + } + accepted + })) + .buffer_unordered(PUBLISH_CONCURRENCY) + .collect::>() + .await; + stats.remote_nodes = sends.into_iter().filter(|accepted| *accepted).count(); + Ok(stats) + } + + /// Iterative lookup of `key`: queries the `K` closest known overlay + /// nodes until all have answered, the request budget is spent or the + /// deadline passes. Returns the winning verified record per + /// `(podcast_id, publisher)`. + async fn lookup( + &self, + key: PodcastDhtKey, + filter_tokens: Vec, + ) -> Result> { + let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new(); + let local = self + .database + .records_by_key( + key, + filter_tokens.clone(), + now_ms(), + MAX_PODCAST_DHT_RECORDS, + ) + .await?; + merge_records(&mut winners, local, &self.network_id, &key, &filter_tokens); + + let mut candidates = lock(&self.routing).closest(key.as_bytes(), K); + let mut known: HashSet<_> = candidates.iter().map(|contact| contact.peer_id).collect(); + let mut queried = HashSet::new(); + let mut sent = 0usize; + let deadline = tokio::time::Instant::now() + LOOKUP_TIMEOUT; + while tokio::time::Instant::now() < deadline && sent < MAX_LOOKUP_REQUESTS { + candidates.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + let batch: Vec<_> = candidates + .iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) + .take(K) + .filter(|contact| !queried.contains(&contact.peer_id)) + .take(ALPHA.min(MAX_LOOKUP_REQUESTS - sent)) + .cloned() + .collect(); + if batch.is_empty() { + break; + } + sent += batch.len(); + queried.extend(batch.iter().map(|contact| contact.peer_id)); + let operation = PodcastDhtOperation::FindValue { + key, + filter_tokens: filter_tokens.clone(), + }; + let mut futures: FuturesUnordered<_> = batch + .iter() + .map(|contact| { + let operation = operation.clone(); + async move { self.exchange(contact, operation).await } + }) + .collect(); + while let Ok(Some(response)) = tokio::time::timeout_at(deadline, futures.next()).await { + let Ok(response) = response else { + continue; + }; + merge_records( + &mut winners, + response.records, + &self.network_id, + &key, + &filter_tokens, + ); + for node in response.nodes { + if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)) + .is_ok() + && !self.is_unsupported(&node.peer_id) + && known.insert(node.peer_id) + { + self.learn_contact(node.clone()).await?; + candidates.push(node); + } + } + } + } + Ok(winners.into_values().collect()) + } + + async fn serve_one(&self, mut stream: ByteStream) -> Result<()> { + let authenticated_peer = stream.peer_id; + let response = match read_postcard::<_, PodcastDhtRequest>( + &mut stream.recv, + MAX_PODCAST_DHT_REQUEST_BYTES, + ) + .await + { + Ok(request) => match self.handle_request(authenticated_peer, request).await { + Ok(response) => response, + Err(error) => PodcastDhtResponse::error(error.to_string()), + }, + Err(error) => PodcastDhtResponse::error(error.to_string()), + }; + write_postcard(&mut stream.send, &response, MAX_PODCAST_DHT_RESPONSE_BYTES).await?; + stream.send.finish().map_err(network_error)?; + let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await; + Ok(()) + } + + async fn handle_request( + &self, + authenticated_peer: EndpointId, + request: PodcastDhtRequest, + ) -> Result { + if request.version != PODCAST_DHT_PROTOCOL_VERSION { + return Err(protocol_error("unsupported podcast DHT request version")); + } + if request.requester.peer_id != authenticated_peer { + return Err(protocol_error("podcast DHT requester identity mismatch")); + } + self.learn_contact(request.requester).await?; + match request.operation { + PodcastDhtOperation::FindNode { target } => Ok(PodcastDhtResponse::success( + self.closest_contacts(&target, K), + Vec::new(), + 0, + )), + PodcastDhtOperation::FindValue { key, filter_tokens } => { + validate_filter_tokens(&filter_tokens)?; + let records = self + .database + .records_by_key(key, filter_tokens, now_ms(), MAX_PODCAST_DHT_RECORDS) + .await?; + Ok(PodcastDhtResponse::success( + self.closest_contacts(key.as_bytes(), K), + records, + 0, + )) + } + PodcastDhtOperation::StoreBatch { records } => { + let entries = validate_store_batch(records, &self.network_id, now_ms())?; + let stored = self.database.store(entries, now_ms()).await?; + Ok(PodcastDhtResponse::success( + Vec::new(), + Vec::new(), + u32::try_from(stored).unwrap_or(u32::MAX), + )) + } + } + } + + /// One request with [`REQUEST_TIMEOUT`]. Network failures and timeouts + /// put the peer in a short backoff; a success clears it. + async fn exchange( + &self, + contact: &NodeContact, + operation: PodcastDhtOperation, + ) -> Result { + let result = tokio::time::timeout(REQUEST_TIMEOUT, self.exchange_once(contact, operation)) + .await + .unwrap_or(Err(MusicDhtError::Timeout)); + match &result { + Ok(_) => { + lock(&self.failed).remove(&contact.peer_id); + } + Err(MusicDhtError::Network(_) | MusicDhtError::Timeout) => { + let until = now_ms().saturating_add(FAILED_PEER_BACKOFF.as_millis() as u64); + lock(&self.failed).insert(contact.peer_id, until); + } + Err(_) => {} + } + result + } + + async fn exchange_once( + &self, + contact: &NodeContact, + operation: PodcastDhtOperation, + ) -> Result { + validate_contact(contact, &self.network_id, Some(self.own_contact.peer_id))?; + let ticket = PeerTicket::from_str(&contact.ticket) + .map_err(|error| MusicDhtError::InvalidTicket(error.to_string()))?; + let mut stream = self + .service + .open_stream_to(&ticket, PODCAST_DHT_ALPN) + .await?; + let request = PodcastDhtRequest { + version: PODCAST_DHT_PROTOCOL_VERSION, + requester: self.fresh_own_contact(), + operation, + }; + write_postcard(&mut stream.send, &request, MAX_PODCAST_DHT_REQUEST_BYTES).await?; + stream.send.finish().map_err(network_error)?; + let response: PodcastDhtResponse = + read_postcard(&mut stream.recv, MAX_PODCAST_DHT_RESPONSE_BYTES).await?; + validate_response(&response)?; + if !response.ok { + return Err(protocol_error( + response + .error + .as_deref() + .unwrap_or("podcast DHT request refused"), + )); + } + Ok(response) + } + + fn is_unsupported(&self, peer: &EndpointId) -> bool { + still_listed(&self.unsupported, peer) + } + + fn is_backed_off(&self, peer: &EndpointId) -> bool { + still_listed(&self.failed, peer) + } + + fn replica_targets(&self, key: &PodcastDhtKey) -> (bool, Vec) { + let mut contacts: Vec<_> = lock(&self.routing) + .contacts() + .into_iter() + .filter(|contact| !self.is_backed_off(&contact.peer_id)) + .collect(); + contacts.push(self.fresh_own_contact()); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes())); + contacts.dedup_by_key(|contact| contact.peer_id); + contacts.truncate(K); + let local = contacts + .iter() + .any(|contact| contact.peer_id == self.own_contact.peer_id); + contacts.retain(|contact| contact.peer_id != self.own_contact.peer_id); + (local, contacts) + } + + fn closest_contacts(&self, target: &[u8; 32], count: usize) -> Vec { + let mut contacts = lock(&self.routing).contacts(); + contacts.push(self.fresh_own_contact()); + contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), target)); + contacts.dedup_by_key(|contact| contact.peer_id); + contacts.truncate(count); + contacts + } + + fn fresh_own_contact(&self) -> NodeContact { + let mut contact = self.own_contact.clone(); + contact.last_seen_ms = now_ms(); + contact + } + + async fn learn_response_nodes(&self, nodes: Vec) { + for contact in nodes { + if self.is_unsupported(&contact.peer_id) { + continue; + } + if let Err(error) = self.learn_contact(contact).await { + debug!(%error, "invalid podcast DHT contact ignored"); + } + } + } + + async fn learn_contact(&self, mut contact: NodeContact) -> Result<()> { + validate_contact(&contact, &self.network_id, Some(self.own_contact.peer_id))?; + contact.node_id = NodeId::from_endpoint(&contact.peer_id); + // A remote peer does not get to extend its own lifetime with a future + // timestamp; the authenticated contact is the local observation. + contact.last_seen_ms = now_ms(); + let is_new = lock(&self.routing).upsert(contact.clone()); + self.database.upsert_peer(contact.clone()).await?; + if is_new { + debug!(peer = %contact.peer_id, "learned podcast DHT peer"); + } + Ok(()) + } +} + +/// Validates a store batch received from a peer: bounds, every record's +/// signature and lifetime, and that every key belongs to its record. +fn validate_store_batch( + entries: Vec, + network_id: &NetworkId, + now_ms: u64, +) -> Result, SignedPodcastRecord)>> { + if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS { + return Err(protocol_error("invalid podcast DHT store batch")); + } + entries + .into_iter() + .map(|entry| { + let mut keys = entry.keys; + let unique: HashSet<_> = keys.iter().collect(); + if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() { + return Err(protocol_error("invalid podcast DHT store keys")); + } + entry.record.verify_at(network_id, now_ms)?; + let valid_keys = entry.record.dht_keys(); + if keys.iter().any(|key| !valid_keys.contains(key)) { + return Err(protocol_error("podcast record does not belong under key")); + } + keys.sort(); + Ok((keys, entry.record)) + }) + .collect() +} + +fn validate_filter_tokens(tokens: &[String]) -> Result<()> { + if tokens.len() > MAX_PODCAST_QUERY_TOKENS + || tokens + .iter() + .any(|token| token.is_empty() || token.len() > MAX_INDEX_TOKEN_BYTES) + { + return Err(protocol_error("invalid podcast DHT filter tokens")); + } + Ok(()) +} + +/// Adds verified records found under `key` to the per-owner winners. +fn merge_records( + winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>, + records: Vec, + network_id: &NetworkId, + key: &PodcastDhtKey, + filter_tokens: &[String], +) { + let now = now_ms(); + for record in records { + if record.verify_at(network_id, now).is_err() || !record.belongs_under(key) { + continue; + } + if let Some(card) = record.card() { + let tokens = card.index_tokens(); + if !filter_tokens.iter().all(|token| tokens.contains(token)) { + continue; + } + } + let owner = (record.podcast_id().clone(), record.publisher()); + match winners.get(&owner) { + Some(current) if compare_records(current, &record).is_ge() => {} + _ => { + winners.insert(owner, record); + } + } + } +} + +fn validate_contact( + contact: &NodeContact, + network_id: &NetworkId, + excluded: Option, +) -> Result<()> { + if excluded == Some(contact.peer_id) + || contact.node_id != NodeId::from_endpoint(&contact.peer_id) + || contact.ticket.len() > MAX_TICKET_BYTES + { + return Err(protocol_error("invalid podcast DHT contact")); + } + let ticket = PeerTicket::from_str(&contact.ticket) + .map_err(|error| MusicDhtError::InvalidTicket(error.to_string()))?; + if ticket.endpoint_id() != contact.peer_id || &ticket.network_id != network_id { + return Err(protocol_error("podcast DHT contact ticket mismatch")); + } + Ok(()) +} + +fn validate_response(response: &PodcastDhtResponse) -> Result<()> { + if response.version != PODCAST_DHT_PROTOCOL_VERSION + || response.nodes.len() > K + || response.records.len() > MAX_PODCAST_DHT_RECORDS + || response + .error + .as_ref() + .is_some_and(|error| error.len() > MAX_ERROR_BYTES) + || (response.ok && response.error.is_some()) + || (!response.ok && response.error.is_none()) + { + return Err(protocol_error("invalid podcast DHT response")); + } + Ok(()) +} + +async fn write_postcard( + writer: &mut W, + value: &T, + maximum: usize, +) -> Result<()> { + let payload = postcard::to_stdvec(value).map_err(protocol_error)?; + if payload.len() > maximum { + return Err(protocol_error("podcast DHT message is too large")); + } + writer.write_all(&payload).await.map_err(network_error) +} + +async fn read_postcard Deserialize<'de>>( + reader: &mut R, + maximum: usize, +) -> Result { + let mut payload = Vec::new(); + reader + .take(maximum as u64 + 1) + .read_to_end(&mut payload) + .await + .map_err(network_error)?; + if payload.is_empty() || payload.len() > maximum { + return Err(protocol_error("invalid podcast DHT message size")); + } + postcard::from_bytes(&payload).map_err(protocol_error) +} + +/// Whether `peer` is in a timed list; expired entries are removed. +fn still_listed(list: &Mutex>, peer: &EndpointId) -> bool { + let now = now_ms(); + let mut list = lock(list); + match list.get(peer) { + Some(until) if *until > now => true, + Some(_) => { + list.remove(peer); + false + } + None => false, + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn protocol_error(error: impl std::fmt::Display) -> MusicDhtError { + MusicDhtError::Protocol(error.to_string()) +} + +fn network_error(error: impl std::fmt::Display) -> MusicDhtError { + MusicDhtError::Network(error.to_string()) +} diff --git a/crates/music-dht/src/podcast_dht/privacy.rs b/crates/music-dht/src/podcast_dht/privacy.rs new file mode 100644 index 0000000..c298682 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/privacy.rs @@ -0,0 +1,339 @@ +//! Advisory detection of feed URLs that look private. +//! +//! Premium and member feeds usually carry a personal secret in the URL. A +//! card publishes its feed URL verbatim, so the UI must ask the user before +//! publishing a URL for which [`feed_url_privacy_warning`] returns a warning. +//! The heuristics are deliberately broad and can produce false positives +//! (and miss secrets); they never block publication by themselves. + +use std::collections::HashMap; +use std::fmt; + +use serde::{Deserialize, Serialize}; + +use crate::podcast::FeedUrl; + +/// Query-parameter name words that usually carry a credential. A name is +/// split into lowercase words at non-alphanumeric characters and camelCase +/// boundaries; any listed word triggers a warning. +pub const SECRET_QUERY_WORDS: &[&str] = &[ + "token", + "key", + "auth", + "secret", + "sig", + "signature", + "uid", + "user", + "pass", + "password", + "passwd", + "pwd", + "session", + "sess", + "sid", + "access", + "private", + "credential", + "credentials", + "apikey", + "jwt", + "hmac", +]; + +/// Substrings that trigger a warning anywhere in a query-parameter name, +/// for names such as `apitoken` or `privatefeed` that do not split. +const SECRET_QUERY_SUBSTRINGS: &[&str] = &[ + "token", + "secret", + "signature", + "password", + "passwd", + "session", + "private", + "credential", + "apikey", +]; + +/// Hosts (and their subdomains) known to serve personal member feeds. +pub const PRIVATE_FEED_HOSTS: &[&str] = &[ + "patreon.com", + "supercast.com", + "supportingcast.fm", + "glow.fm", +]; + +/// Minimum length of a query value considered a possible secret. +const MIN_SECRET_VALUE_CHARS: usize = 16; +/// Minimum length of a path segment considered a possible secret. +const MIN_SECRET_SEGMENT_CHARS: usize = 32; +/// Minimum Shannon entropy, in bits per character, of a possible secret. +const MIN_SECRET_ENTROPY_BITS: f64 = 3.0; + +/// Why a feed URL may contain a personal secret. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "reason", rename_all = "snake_case")] +#[non_exhaustive] +pub enum FeedUrlPrivacyWarning { + /// A query parameter whose name suggests a credential. + SecretQueryParameter { + /// Parameter name as it appears in the URL. + name: String, + }, + /// The host is known to serve personal member feeds. + PrivateFeedHost { + /// Lowercase host. + host: String, + }, + /// A long, random-looking query value. + HighEntropyQueryValue { + /// Name of the parameter carrying the value. + name: String, + }, + /// A long, random-looking path segment. + HighEntropyPathSegment { + /// Zero-based index of the segment after the host. + index: usize, + }, +} + +impl fmt::Display for FeedUrlPrivacyWarning { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SecretQueryParameter { name } => write!( + f, + "the feed URL has a query parameter \"{name}\" that looks like a credential" + ), + Self::PrivateFeedHost { host } => { + write!(f, "{host} usually serves personal member feeds") + } + Self::HighEntropyQueryValue { name } => write!( + f, + "the value of the query parameter \"{name}\" looks like a secret token" + ), + Self::HighEntropyPathSegment { index } => write!( + f, + "path segment {} of the feed URL looks like a secret token", + index + 1 + ), + } + } +} + +/// Returns why `url` may contain a personal secret, or `None`. +/// +/// Advisory only (see the module documentation). Checks, in order: query +/// parameter names ([`SECRET_QUERY_WORDS`]), known member-feed hosts +/// ([`PRIVATE_FEED_HOSTS`]), long high-entropy query values, and long +/// high-entropy path segments. +pub fn feed_url_privacy_warning(url: &FeedUrl) -> Option { + let value = url.as_str(); + let rest = value.split_once("://").map_or(value, |(_, rest)| rest); + let (authority, path_and_query) = match rest.find('/') { + Some(index) => rest.split_at(index), + None => (rest, "/"), + }; + let host = if authority.starts_with('[') { + authority + .find(']') + .map_or(authority, |end| &authority[..=end]) + } else { + authority.split(':').next().unwrap_or(authority) + }; + let (path, query) = path_and_query + .split_once('?') + .unwrap_or((path_and_query, "")); + let parameters: Vec<(&str, &str)> = query + .split('&') + .filter(|pair| !pair.is_empty()) + .map(|pair| pair.split_once('=').unwrap_or((pair, ""))) + .collect(); + + if let Some((name, _)) = parameters + .iter() + .find(|(name, _)| is_secret_parameter_name(name)) + { + return Some(FeedUrlPrivacyWarning::SecretQueryParameter { + name: (*name).to_string(), + }); + } + if PRIVATE_FEED_HOSTS + .iter() + .any(|known| host == *known || host.ends_with(&format!(".{known}"))) + { + return Some(FeedUrlPrivacyWarning::PrivateFeedHost { + host: host.to_string(), + }); + } + if let Some((name, _)) = parameters + .iter() + .find(|(_, value)| looks_random(value, MIN_SECRET_VALUE_CHARS)) + { + return Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { + name: (*name).to_string(), + }); + } + path.split('/') + .filter(|segment| !segment.is_empty()) + .position(|segment| { + let stem = segment.rsplit_once('.').map_or(segment, |(stem, _)| stem); + looks_random(stem, MIN_SECRET_SEGMENT_CHARS) + }) + .map(|index| FeedUrlPrivacyWarning::HighEntropyPathSegment { index }) +} + +fn is_secret_parameter_name(name: &str) -> bool { + let words = split_words(name); + if words + .iter() + .any(|word| SECRET_QUERY_WORDS.contains(&word.as_str())) + { + return true; + } + let compact: String = name + .chars() + .filter(char::is_ascii_alphanumeric) + .collect::() + .to_ascii_lowercase(); + SECRET_QUERY_SUBSTRINGS + .iter() + .any(|needle| compact.contains(needle)) +} + +/// Splits at non-alphanumeric characters and lower-to-upper case changes. +fn split_words(name: &str) -> Vec { + let mut words = Vec::new(); + let mut current = String::new(); + let mut previous_lower = false; + for c in name.chars() { + if !c.is_ascii_alphanumeric() { + if !current.is_empty() { + words.push(std::mem::take(&mut current)); + } + previous_lower = false; + continue; + } + if c.is_ascii_uppercase() && previous_lower && !current.is_empty() { + words.push(std::mem::take(&mut current)); + } + previous_lower = c.is_ascii_lowercase() || c.is_ascii_digit(); + current.push(c.to_ascii_lowercase()); + } + if !current.is_empty() { + words.push(current); + } + words +} + +/// A long token-alphabet string mixing letters and digits (or both letter +/// cases) with high per-character entropy. +fn looks_random(value: &str, min_chars: usize) -> bool { + if value.chars().count() < min_chars + || !value + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.~%=+".contains(c)) + { + return false; + } + let has_digit = value.chars().any(|c| c.is_ascii_digit()); + let has_lower = value.chars().any(|c| c.is_ascii_lowercase()); + let has_upper = value.chars().any(|c| c.is_ascii_uppercase()); + let classes = [has_digit, has_lower, has_upper] + .into_iter() + .filter(|present| *present) + .count(); + classes >= 2 && shannon_entropy(value) >= MIN_SECRET_ENTROPY_BITS +} + +fn shannon_entropy(value: &str) -> f64 { + let mut counts: HashMap = HashMap::new(); + for c in value.chars() { + *counts.entry(c).or_default() += 1; + } + let total = value.chars().count() as f64; + counts + .values() + .map(|count| { + let p = *count as f64 / total; + -p * p.log2() + }) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn warn(url: &str) -> Option { + feed_url_privacy_warning(&FeedUrl::parse(url).unwrap()) + } + + #[test] + fn public_feeds_have_no_warning() { + for url in [ + "https://feeds.example.com/podcast.xml", + "https://example.com/feed?format=rss&page=2", + "https://anchor.example/s/abc123de/podcast/rss", + "https://feeds.example.com/12345/rss", + "https://example.com/feed?author=jane", + "https://example.com:8443/shows/my-great-show/feed.xml", + ] { + assert_eq!(warn(url), None, "{url}"); + } + } + + #[test] + fn secret_query_names_warn() { + for (url, name) in [ + ("https://example.com/feed?token=abc", "token"), + ("https://example.com/feed?auth_token=abc", "auth_token"), + ( + "https://example.com/feed?X-Amz-Signature=abc", + "X-Amz-Signature", + ), + ("https://example.com/feed?accessKey=abc", "accessKey"), + ("https://example.com/feed?a=1&uid=42", "uid"), + ("https://example.com/feed?apitoken=x", "apitoken"), + ("https://example.com/feed?PRIVATE=1", "PRIVATE"), + ("https://example.com/feed?session_id=1", "session_id"), + ] { + assert_eq!( + warn(url), + Some(FeedUrlPrivacyWarning::SecretQueryParameter { name: name.into() }), + "{url}" + ); + } + } + + #[test] + fn known_member_hosts_warn() { + assert_eq!( + warn("https://www.patreon.com/rss/show"), + Some(FeedUrlPrivacyWarning::PrivateFeedHost { + host: "www.patreon.com".into() + }) + ); + assert_eq!(warn("https://notpatreon.com/rss"), None); + } + + #[test] + fn high_entropy_values_and_segments_warn() { + assert_eq!( + warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb"), + Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { name: "c".into() }) + ); + assert_eq!(warn("https://example.com/feed?c=aaaaaaaaaaaaaaaa1"), None); + assert_eq!( + warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB4sYc1HdWe9GuJa3F.rss"), + Some(FeedUrlPrivacyWarning::HighEntropyPathSegment { index: 1 }) + ); + // Below the segment length: an ordinary short id. + assert_eq!(warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB.rss"), None); + } + + #[test] + fn warnings_render_without_the_secret() { + let warning = warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb").unwrap(); + assert!(!warning.to_string().contains("Zx81")); + } +} diff --git a/crates/music-dht/src/podcast_dht/search.rs b/crates/music-dht/src/podcast_dht/search.rs new file mode 100644 index 0000000..8b9e3b9 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/search.rs @@ -0,0 +1,238 @@ +//! Podcast search hits and their merge with music search results. + +use std::cmp::Reverse; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; + +use federation_net::EndpointId; +use serde::{Deserialize, Serialize}; + +use crate::normalization::{normalize_name, tokenize}; +use crate::podcast::PodcastId; +use crate::record::{ItemKind, LibraryItem}; + +use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records}; + +/// Maximum distinct query tokens used by a podcast text search. +pub const MAX_PODCAST_QUERY_TOKENS: usize = 8; +/// Maximum publishers reported per hit. +pub const MAX_HIT_PUBLISHERS: usize = 64; + +/// Kind of a federated search hit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SearchHitKind { + /// A musical artist. + Artist, + /// A release. + Release, + /// A track. + Track, + /// A public podcast card. + Podcast, +} + +impl SearchHitKind { + /// Stable lowercase name. + pub fn as_str(&self) -> &'static str { + match self { + Self::Artist => "artist", + Self::Release => "release", + Self::Track => "track", + Self::Podcast => "podcast", + } + } +} + +impl fmt::Display for SearchHitKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From for SearchHitKind { + fn from(kind: ItemKind) -> Self { + match kind { + ItemKind::Artist => Self::Artist, + ItemKind::Release => Self::Release, + ItemKind::Track => Self::Track, + } + } +} + +/// One public podcast found in the discovery overlay. +/// +/// Cards of the same podcast published by several devices are merged by +/// `podcast_id`. The hit carries no personal state of any publisher. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastSearchHit { + /// Podcast. + pub podcast_id: PodcastId, + /// Representative card: the most common `feed_url` among the + /// publishers' current cards, then the most recently issued card, then + /// the greatest publisher key. + pub card: PodcastCard, + /// Sorted distinct publisher devices with a current valid card, at most + /// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers: + /// one user may publish from several devices. + pub publishers: Vec, +} + +impl PodcastSearchHit { + /// Always [`SearchHitKind::Podcast`]. + pub fn kind(&self) -> SearchHitKind { + SearchHitKind::Podcast + } +} + +/// A music or podcast result of a federated search. +/// +/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use +/// [`FederatedSearchHit::kind`] for the fine-grained kind. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", content = "hit", rename_all = "snake_case")] +pub enum FederatedSearchHit { + /// A music-DHT record (artist, release or track). + Music(LibraryItem), + /// A public podcast. + Podcast(PodcastSearchHit), +} + +impl FederatedSearchHit { + /// Kind of the hit: `artist`, `release`, `track` or `podcast`. + pub fn kind(&self) -> SearchHitKind { + match self { + Self::Music(item) => item.kind.into(), + Self::Podcast(_) => SearchHitKind::Podcast, + } + } + + /// Display name: the item name or the podcast title. + pub fn title(&self) -> &str { + match self { + Self::Music(item) => &item.name, + Self::Podcast(hit) => &hit.card.title, + } + } +} + +/// Merges music results (as returned by `MusicDhtService::search_network`) +/// with podcast hits for the same query. +/// +/// Results whose normalized name equals the normalized query come first; +/// within each tier music keeps its order and precedes podcasts, which keep +/// theirs. Existing music consumers that only read `SearchOutcome` are +/// unaffected. +pub fn merge_search_results( + query: &str, + music: Vec, + podcasts: Vec, +) -> Vec { + let normalized = normalize_name(query); + let mut exact = Vec::new(); + let mut rest = Vec::new(); + let hits = music + .into_iter() + .map(FederatedSearchHit::Music) + .chain(podcasts.into_iter().map(FederatedSearchHit::Podcast)); + for hit in hits { + let is_exact = !normalized.is_empty() && normalize_name(hit.title()) == normalized; + if is_exact { + exact.push(hit); + } else { + rest.push(hit); + } + } + exact.extend(rest); + exact +} + +/// Distinct normalized query tokens, at most [`MAX_PODCAST_QUERY_TOKENS`]. +pub fn podcast_query_tokens(query: &str) -> Vec { + let mut seen = HashSet::new(); + tokenize(&normalize_name(query)) + .into_iter() + .filter(|token| token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone())) + .take(MAX_PODCAST_QUERY_TOKENS) + .collect() +} + +/// Keeps the winning record per `(podcast_id, publisher)` and groups the +/// cards that are still active by podcast. Records must be verified. +pub(crate) fn aggregate_hits( + records: impl IntoIterator, +) -> Vec { + let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new(); + for record in records { + let owner = (record.podcast_id().clone(), record.publisher()); + match winners.get(&owner) { + Some(current) if compare_records(current, &record).is_ge() => {} + _ => { + winners.insert(owner, record); + } + } + } + let mut by_podcast: BTreeMap> = BTreeMap::new(); + for record in winners.into_values() { + if let Some(card) = record.card() { + by_podcast + .entry(card.podcast_id.clone()) + .or_default() + .push(card.clone()); + } + } + by_podcast + .into_iter() + .map(|(podcast_id, cards)| { + let mut feed_votes: HashMap<&str, usize> = HashMap::new(); + for card in &cards { + *feed_votes.entry(card.feed_url.as_str()).or_default() += 1; + } + let card = cards + .iter() + .max_by_key(|card| { + ( + feed_votes[card.feed_url.as_str()], + card.issued_at_ms, + *card.publisher.as_bytes(), + ) + }) + .cloned() + .expect("a group has at least one card"); + let mut publishers: Vec = cards.iter().map(|card| card.publisher).collect(); + publishers.sort_by_key(|publisher| *publisher.as_bytes()); + publishers.dedup(); + publishers.truncate(MAX_HIT_PUBLISHERS); + PodcastSearchHit { + podcast_id, + card, + publishers, + } + }) + .collect() +} + +/// Keeps hits whose card is indexed under every query token and ranks +/// them: exact normalized title first, then more publishers, then title and +/// id. +pub(crate) fn rank_hits( + mut hits: Vec, + query: &str, + tokens: &[String], +) -> Vec { + let normalized_query = normalize_name(query); + hits.retain(|hit| { + let indexed = hit.card.index_tokens(); + tokens.iter().all(|token| indexed.contains(token)) + }); + hits.sort_by_cached_key(|hit| { + let title = normalize_name(&hit.card.title); + ( + title != normalized_query, + Reverse(hit.publishers.len()), + title, + hit.podcast_id.clone(), + ) + }); + hits +} diff --git a/crates/music-dht/src/podcast_dht/store.rs b/crates/music-dht/src/podcast_dht/store.rs new file mode 100644 index 0000000..9f27104 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/store.rs @@ -0,0 +1,522 @@ +//! SQLite cache of the podcast-discovery overlay: replicas, routing contacts +//! and this device's own publications. +//! +//! The cache is replaceable federation state, kept in its own database file +//! (like the similarity-routing cache). Every record read from it is decoded +//! and verified again before it is used or served. + +use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use federation_net::{EndpointId, NetworkId}; +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::error::{MusicDhtError, Result}; +use crate::podcast::PodcastId; +use crate::routing::{NodeContact, NodeId}; + +use super::card::{ + PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere, +}; + +/// `PRAGMA user_version` of the podcast-discovery cache schema. +pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1; +/// Maximum distinct `(podcast_id, publisher)` records cached under one key. +/// When full, the record expiring first is evicted if the new one outlives it. +pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256; + +const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS podcast_dht_records ( + dht_key BLOB NOT NULL, + podcast_id TEXT NOT NULL, + publisher TEXT NOT NULL, + tombstone INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + payload BLOB NOT NULL, + PRIMARY KEY (dht_key, podcast_id, publisher) +); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry + ON podcast_dht_records(expires_at_ms); + +CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner + ON podcast_dht_records(podcast_id, publisher); + +CREATE TABLE IF NOT EXISTS podcast_dht_peers ( + peer_id TEXT PRIMARY KEY, + node_id BLOB NOT NULL, + ticket TEXT NOT NULL, + last_seen_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS podcast_dht_local ( + podcast_id TEXT PRIMARY KEY, + revision INTEGER NOT NULL, + spec BLOB, + record BLOB NOT NULL, + updated_at_ms INTEGER NOT NULL +); +"; + +/// This device's latest publication of one podcast. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LocalPublication { + pub podcast_id: PodcastId, + /// Greatest revision ever used for this podcast by this device. + pub revision: u64, + /// Published card content; `None` once unpublished. + pub spec: Option, + /// Latest signed record (card or tombstone). + pub record: SignedPodcastRecord, +} + +#[derive(Clone)] +pub(crate) struct PodcastDhtDatabase { + conn: Arc>, + network_id: NetworkId, +} + +impl std::fmt::Debug for PodcastDhtDatabase { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PodcastDhtDatabase") + .finish_non_exhaustive() + } +} + +impl PodcastDhtDatabase { + pub async fn open(path: &Path, network_id: NetworkId) -> Result { + let path = path.to_path_buf(); + let conn = tokio::task::spawn_blocking(move || -> Result { + let conn = Connection::open(&path).map_err(|error| { + MusicDhtError::Database(format!("failed to open {}: {error}", path.display())) + })?; + let version: i32 = conn + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(database_error)?; + if version > PODCAST_DHT_CACHE_SCHEMA_VERSION { + return Err(MusicDhtError::Database(format!( + "podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}" + ))); + } + conn.execute_batch(SCHEMA).map_err(|error| { + MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}")) + })?; + conn.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION) + .map_err(database_error)?; + Ok(conn) + }) + .await + .map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))??; + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + network_id, + }) + } + + async fn call(&self, operation: F) -> Result + where + F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result + Send + 'static, + T: Send + 'static, + { + let conn = Arc::clone(&self.conn); + let network_id = self.network_id; + tokio::task::spawn_blocking(move || { + let guard = lock(&conn); + operation(&guard, &network_id).map_err(database_error) + }) + .await + .map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))? + } + + /// Stores records under the given keys. The caller has verified every + /// record and checked that each key belongs to it. Returns the number of + /// `(key, record)` rows written. + pub async fn store( + &self, + entries: Vec<(Vec, SignedPodcastRecord)>, + now_ms: u64, + ) -> Result { + self.call(move |conn, network_id| { + let transaction = conn.unchecked_transaction()?; + let mut written = 0; + for (keys, record) in entries { + written += store_one(&transaction, network_id, &keys, &record, now_ms)?; + } + transaction.commit()?; + Ok(written) + }) + .await + } + + /// Valid, unexpired records under `key`, newest expiry first. Cards must + /// contain every token in `filter_tokens`; tombstones are always + /// returned so stale replicas elsewhere can be overruled. + pub async fn records_by_key( + &self, + key: PodcastDhtKey, + filter_tokens: Vec, + now_ms: u64, + limit: usize, + ) -> Result> { + self.call(move |conn, network_id| { + let mut statement = conn.prepare( + "SELECT payload FROM podcast_dht_records + WHERE dht_key = ?1 AND expires_at_ms > ?2 + ORDER BY expires_at_ms DESC, podcast_id, publisher", + )?; + let rows = statement.query_map( + params![key.as_bytes().as_slice(), clamp_i64(now_ms)], + |row| row.get::<_, Vec>(0), + )?; + let mut records = Vec::new(); + for row in rows { + let Some(record) = decode_valid(&row?, network_id, now_ms) else { + continue; + }; + if !record.belongs_under(&key) { + continue; + } + let matches = record.card().is_none_or(|card| { + let tokens = card.index_tokens(); + filter_tokens.iter().all(|token| tokens.contains(token)) + }); + if matches { + records.push(record); + if records.len() == limit { + break; + } + } + } + Ok(records) + }) + .await + } + + pub async fn delete_expired(&self, now_ms: u64) -> Result { + self.call(move |conn, _| { + conn.execute( + "DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1", + [clamp_i64(now_ms)], + ) + }) + .await + } + + pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> { + self.call(move |conn, _| { + conn.execute( + "INSERT OR REPLACE INTO podcast_dht_peers + (peer_id, node_id, ticket, last_seen_ms) + VALUES (?1, ?2, ?3, ?4)", + params![ + contact.peer_id.to_string(), + contact.node_id.as_bytes().as_slice(), + contact.ticket, + clamp_i64(contact.last_seen_ms), + ], + )?; + Ok(()) + }) + .await + } + + pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> { + self.call(move |conn, _| { + conn.execute( + "DELETE FROM podcast_dht_peers WHERE peer_id = ?1", + [peer.to_string()], + )?; + Ok(()) + }) + .await + } + + pub async fn load_peers(&self) -> Result> { + self.call(move |conn, _| { + let mut statement = + conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + let mut peers = Vec::new(); + for row in rows { + let (peer, ticket, last_seen_ms) = row?; + let Ok(peer_id) = EndpointId::from_str(&peer) else { + continue; + }; + peers.push(NodeContact { + node_id: NodeId::from_endpoint(&peer_id), + peer_id, + ticket, + last_seen_ms: last_seen_ms.max(0) as u64, + }); + } + Ok(peers) + }) + .await + } + + pub async fn local_publication( + &self, + podcast_id: PodcastId, + ) -> Result> { + self.call(move |conn, _| { + conn.query_row( + "SELECT podcast_id, revision, spec, record FROM podcast_dht_local + WHERE podcast_id = ?1", + [podcast_id.as_str()], + local_from_row, + ) + .optional() + .map(Option::flatten) + }) + .await + } + + pub async fn local_publications(&self) -> Result> { + self.call(move |conn, _| { + let mut statement = conn.prepare( + "SELECT podcast_id, revision, spec, record FROM podcast_dht_local + ORDER BY podcast_id", + )?; + let rows = statement.query_map([], local_from_row)?; + let mut out = Vec::new(); + for row in rows { + if let Some(publication) = row? { + out.push(publication); + } + } + Ok(out) + }) + .await + } + + pub async fn put_local_publication( + &self, + publication: LocalPublication, + now_ms: u64, + ) -> Result<()> { + self.call(move |conn, _| { + let spec = publication + .spec + .as_ref() + .map(postcard::to_stdvec) + .transpose() + .map_err(to_sql_error)?; + let record = postcard::to_stdvec(&publication.record).map_err(to_sql_error)?; + conn.execute( + "INSERT OR REPLACE INTO podcast_dht_local + (podcast_id, revision, spec, record, updated_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + publication.podcast_id.as_str(), + publication.revision as i64, + spec, + record, + clamp_i64(now_ms), + ], + )?; + Ok(()) + }) + .await + } +} + +fn store_one( + conn: &Connection, + network_id: &NetworkId, + keys: &[PodcastDhtKey], + record: &SignedPodcastRecord, + now_ms: u64, +) -> rusqlite::Result { + let podcast_id = record.podcast_id().as_str().to_string(); + let publisher = record.publisher().to_string(); + + // Every cached version of this (podcast, publisher), re-verified. + let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new(); + let mut invalid_keys = Vec::new(); + { + let mut statement = conn.prepare( + "SELECT dht_key, payload FROM podcast_dht_records + WHERE podcast_id = ?1 AND publisher = ?2", + )?; + let rows = statement.query_map(params![podcast_id, publisher], |row| { + Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)) + })?; + for row in rows { + let (key, payload) = row?; + let decoded = decode_valid(&payload, network_id, now_ms); + match (<[u8; 32]>::try_from(key.as_slice()), decoded) { + (Ok(key_bytes), Some(cached)) => { + existing.insert(key_bytes, cached); + } + _ => invalid_keys.push(key), + } + } + } + for key in invalid_keys { + conn.execute( + "DELETE FROM podcast_dht_records + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key, podcast_id, publisher], + )?; + } + + // A tombstone or a newer revision cached under any key wins everywhere. + if existing + .values() + .any(|cached| supersedes_everywhere(cached, record)) + { + return Ok(0); + } + + let payload = postcard::to_stdvec(record).map_err(to_sql_error)?; + let mut written = 0; + for key in keys { + if let Some(cached) = existing.get(key.as_bytes()) + && compare_records(cached, record).is_ge() + { + continue; + } + if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? { + continue; + } + conn.execute( + "INSERT INTO podcast_dht_records + (dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET + tombstone = excluded.tombstone, + expires_at_ms = excluded.expires_at_ms, + payload = excluded.payload", + params![ + key.as_bytes().as_slice(), + podcast_id, + publisher, + record.is_tombstone(), + clamp_i64(record.expires_at_ms()), + payload, + ], + )?; + written += 1; + } + + // Drop versions the new record makes obsolete under other keys too. + if written > 0 { + for (key, cached) in &existing { + if !keys.iter().any(|stored| stored.as_bytes() == key) + && supersedes_everywhere(record, cached) + { + conn.execute( + "DELETE FROM podcast_dht_records + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_slice(), podcast_id, publisher], + )?; + } + } + } + Ok(written) +} + +/// Ensures there is room for one more record under `key`. +fn make_room( + conn: &Connection, + key: &PodcastDhtKey, + record: &SignedPodcastRecord, + now_ms: u64, +) -> rusqlite::Result { + conn.execute( + "DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2", + params![key.as_bytes().as_slice(), clamp_i64(now_ms)], + )?; + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1", + [key.as_bytes().as_slice()], + |row| row.get(0), + )?; + if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY { + return Ok(true); + } + let victim: Option<(String, String, i64)> = conn + .query_row( + "SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records + WHERE dht_key = ?1 + ORDER BY expires_at_ms ASC, podcast_id, publisher + LIMIT 1", + [key.as_bytes().as_slice()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + match victim { + Some((podcast_id, publisher, expires_at_ms)) + if expires_at_ms < clamp_i64(record.expires_at_ms()) => + { + conn.execute( + "DELETE FROM podcast_dht_records + WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3", + params![key.as_bytes().as_slice(), podcast_id, publisher], + )?; + Ok(true) + } + _ => Ok(false), + } +} + +fn decode_valid( + payload: &[u8], + network_id: &NetworkId, + now_ms: u64, +) -> Option { + let record: SignedPodcastRecord = postcard::from_bytes(payload).ok()?; + record.verify_at(network_id, now_ms).ok()?; + Some(record) +} + +fn local_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result> { + let podcast_id: String = row.get(0)?; + let revision: i64 = row.get(1)?; + let spec: Option> = row.get(2)?; + let record: Vec = row.get(3)?; + let Ok(podcast_id) = PodcastId::parse(&podcast_id) else { + return Ok(None); + }; + let spec = match spec { + Some(bytes) => match postcard::from_bytes::(&bytes) { + Ok(spec) => Some(spec), + Err(_) => return Ok(None), + }, + None => None, + }; + let Ok(record) = postcard::from_bytes::(&record) else { + return Ok(None); + }; + Ok(Some(LocalPublication { + podcast_id, + revision: revision as u64, + spec, + record, + })) +} + +fn clamp_i64(value: u64) -> i64 { + value.min(i64::MAX as u64) as i64 +} + +fn to_sql_error(error: impl std::error::Error + Send + Sync + 'static) -> rusqlite::Error { + rusqlite::Error::ToSqlConversionFailure(Box::new(error)) +} + +fn database_error(error: rusqlite::Error) -> MusicDhtError { + MusicDhtError::Database(error.to_string()) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} diff --git a/crates/music-dht/src/podcast_dht/tests.rs b/crates/music-dht/src/podcast_dht/tests.rs new file mode 100644 index 0000000..faf6159 --- /dev/null +++ b/crates/music-dht/src/podcast_dht/tests.rs @@ -0,0 +1,845 @@ +//! Unit tests of the podcast-discovery overlay: records, ordering, cache and +//! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`. + +use federation_net::{NetworkId, SecretKey}; + +use super::card::supersedes_everywhere; +use super::search::{aggregate_hits, rank_hits}; +use super::*; +use crate::podcast::{FeedUrl, PodcastMeta}; +use crate::record::{ItemId, ItemKind, LibraryItem}; + +const NOW: u64 = 1_750_000_000_000; + +fn network() -> NetworkId { + NetworkId::from_name("podcast-dht-unit") +} + +fn secret(seed: u8) -> SecretKey { + SecretKey::from_bytes(&[seed; 32]) +} + +fn feed(path: &str) -> FeedUrl { + FeedUrl::parse(&format!("https://feeds.example.test/{path}")).unwrap() +} + +fn spec(path: &str, title: &str) -> PodcastCardSpec { + let feed_url = feed(path); + PodcastCardSpec { + podcast_id: PodcastId::from_feed_url(&feed_url), + feed_url, + id_feed_url: None, + title: title.to_string(), + author: Some("Jane Doe".to_string()), + description: Some("A show about\nthings.".to_string()), + image_url: Some("https://cdn.example.test/art.jpg".to_string()), + link: Some("https://example.test/show".to_string()), + language: Some("en".to_string()), + categories: vec!["Technology".to_string()], + episode_count: Some(12), + latest_episode_at_ms: Some(1_700_000_000_000), + } +} + +fn card_with( + spec: PodcastCardSpec, + key: &SecretKey, + revision: u64, + issued: u64, +) -> SignedPodcastRecord { + let card = PodcastCard::from_spec(spec, network(), key.public(), revision, issued); + SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)).unwrap() +} + +fn card(key: &SecretKey, revision: u64) -> SignedPodcastRecord { + card_with(spec("show.xml", "Rust Radio"), key, revision, NOW) +} + +fn tombstone(key: &SecretKey, revision: u64, issued: u64) -> SignedPodcastRecord { + let retracted = card(key, 1); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + retracted.podcast_id().clone(), + revision, + issued, + retracted.dht_keys(), + ); + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap() +} + +fn resign(record: &mut SignedPodcastRecord, key: &SecretKey) { + record.signature = key.sign(&record.signing_bytes()); +} + +fn subscription(spec: &PodcastCardSpec) -> SubscriptionRecord { + SubscriptionRecord { + podcast_id: spec.podcast_id.clone(), + subscribed: true, + podcast: PodcastMeta { + feed_url: spec.feed_url.clone(), + title: spec.title.clone(), + author: spec.author.clone(), + description: spec.description.clone(), + link: spec.link.clone(), + image_url: spec.image_url.clone(), + }, + feed_aliases: Vec::new(), + generation: 1, + hlc_ms: 100, + op_id: "private-op-id-123".to_string(), + } +} + +// ---------------------------------------------------------------- signatures + +#[test] +fn signed_card_verifies_and_rejects_tampering() { + let key = secret(1); + let record = card(&key, 1); + record.verify_at(&network(), NOW).unwrap(); + + let mut tampered = record.clone(); + if let PodcastRecordBody::Card(card) = &mut tampered.body { + card.title = "Evil Radio".to_string(); + } + assert!(tampered.verify(&network()).is_err()); + + let mut moved = record.clone(); + if let PodcastRecordBody::Card(card) = &mut moved.body { + card.revision += 1; + } + assert!(moved.verify(&network()).is_err()); + + assert!(record.verify(&NetworkId::from_name("other")).is_err()); +} + +#[test] +fn record_signed_by_another_key_is_rejected() { + let publisher = secret(1); + let forger = secret(2); + let card = PodcastCard::from_spec( + spec("show.xml", "Rust Radio"), + network(), + publisher.public(), + 1, + NOW, + ); + let forged = SignedPodcastRecord::sign_card(card, |bytes| forger.sign(bytes)).unwrap(); + assert!(forged.verify(&network()).is_err()); + + let mut forged_tombstone = tombstone(&forger, 2, NOW); + if let PodcastRecordBody::Tombstone(tombstone) = &mut forged_tombstone.body { + tombstone.publisher = publisher.public(); + } + resign(&mut forged_tombstone, &forger); + assert!(forged_tombstone.verify(&network()).is_err()); +} + +#[test] +fn canonical_encoding_is_stable() { + let key = secret(9); + let record = card_with(spec("vector.xml", "Vector Show"), &key, 7, NOW); + let bytes = record.signing_bytes(); + assert!(bytes.starts_with(b"furumi-podcast-card:v1\0\x01\x00")); + assert_eq!( + blake3::hash(&bytes).to_hex().as_str(), + "cbbc512b077ad05baf01728fb5d179a1d89301f19ea15c66058df284a716e9b5", + "card signing input changed: this breaks every published card" + ); + let tombstone = PodcastTombstone::new( + network(), + key.public(), + record.podcast_id().clone(), + 8, + NOW, + record.dht_keys(), + ); + let bytes = tombstone.signing_bytes(); + assert!(bytes.starts_with(b"furumi-podcast-card-tombstone:v1\0\x01\x00")); + assert_eq!( + blake3::hash(&bytes).to_hex().as_str(), + "032ccef911bff5f2f37367137d428d60c111efb65529c8d75754434434dbd083", + "tombstone signing input changed" + ); + assert_eq!( + hex(PodcastDhtKey::token(&network(), "vector").as_bytes()), + "4367d03f8ea2a541ebec43b73d767b42ad733e3f26f90372cff9f92cf19ccbf1" + ); +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[test] +fn optional_fields_are_unambiguous_in_the_signing_input() { + let key = secret(1); + let mut a = spec("show.xml", "Rust Radio"); + a.author = Some(String::new()); + let mut b = spec("show.xml", "Rust Radio"); + b.author = None; + let a = PodcastCard::from_spec(a, network(), key.public(), 1, NOW); + let b = PodcastCard::from_spec(b, network(), key.public(), 1, NOW); + assert_ne!(a.signing_bytes(), b.signing_bytes()); +} + +// -------------------------------------------------------------------- bounds + +#[test] +fn card_text_bounds_accept_the_limit_and_reject_one_more() { + let key = secret(1); + let sign = |spec: PodcastCardSpec| { + let card = PodcastCard::from_spec(spec, network(), key.public(), 1, NOW); + SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)) + }; + let mut at = spec("show.xml", &"я".repeat(MAX_CARD_TITLE_CHARS)); + at.author = Some("a".repeat(MAX_CARD_AUTHOR_CHARS)); + at.description = Some("d".repeat(MAX_CARD_DESCRIPTION_CHARS)); + at.categories = (0..MAX_CARD_CATEGORIES) + .map(|index| format!("{index}{}", "c".repeat(MAX_CARD_CATEGORY_CHARS - 1))) + .collect(); + at.language = Some("x".repeat(MAX_CARD_LANGUAGE_BYTES)); + let record = sign(at.clone()).unwrap(); + assert!(record.encoded_len() <= MAX_PODCAST_RECORD_BYTES); + + let mut over = at.clone(); + over.title.push('я'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.author.as_mut().unwrap().push('a'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.description.as_mut().unwrap().push('d'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.categories.push("extra".into()); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.categories[0].push('c'); + assert!(sign(over).is_err()); + let mut over = at.clone(); + over.language.as_mut().unwrap().push('x'); + assert!(sign(over).is_err()); + let mut over = at; + over.title = " ".into(); + assert!(sign(over).is_err()); +} + +#[test] +fn card_rejects_bad_urls_ids_and_control_characters() { + let key = secret(1); + let sign = |spec: PodcastCardSpec| { + let card = PodcastCard::from_spec(spec, network(), key.public(), 1, NOW); + SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)) + }; + let mut bad = spec("show.xml", "Rust Radio"); + bad.image_url = Some("file:///etc/passwd".into()); + assert!(sign(bad).is_err()); + let mut bad = spec("show.xml", "Rust Radio"); + bad.podcast_id = PodcastId::from_feed_url(&feed("other.xml")); + assert!(sign(bad).is_err(), "id must hash from a URL in the card"); + let mut bad = spec("show.xml", "Rust Radio"); + bad.title = "Rust\u{7}Radio".into(); + assert!(sign(bad).is_err()); + let mut bad = spec("show.xml", "Rust Radio"); + bad.id_feed_url = Some(bad.feed_url.clone()); + assert!(sign(bad).is_err()); + + let mut moved = spec("new.xml", "Rust Radio"); + moved.id_feed_url = Some(feed("show.xml")); + moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml")); + sign(moved).unwrap(); +} + +#[test] +fn bounded_spec_fits_the_card() { + let mut raw = spec("show.xml", &format!(" {}\n", "t".repeat(2_000))); + raw.author = Some("\u{1}".into()); + raw.description = Some(format!("{}\u{0}", "d".repeat(5_000))); + raw.image_url = Some("not a url".into()); + raw.language = Some("en US".into()); + raw.categories = (0..20).map(|index| format!("Cat {}", index % 10)).collect(); + raw.categories.push("cat 1".into()); + raw.latest_episode_at_ms = Some(-5); + let bounded = raw.bounded(); + bounded.validate().unwrap(); + assert_eq!(bounded.title.chars().count(), MAX_CARD_TITLE_CHARS); + assert_eq!(bounded.author, None); + assert_eq!(bounded.image_url, None); + assert_eq!(bounded.language, None); + assert_eq!(bounded.categories.len(), MAX_CARD_CATEGORIES); + assert_eq!(bounded.latest_episode_at_ms, None); + assert_eq!(bounded.bounded(), bounded); +} + +#[test] +fn index_tokens_are_distinct_bounded_and_skip_long_tokens() { + let long = "x".repeat(MAX_INDEX_TOKEN_BYTES + 1); + let title = format!("The Rust the {long} radio"); + assert_eq!( + card_index_tokens(&title, Some("Rust Jane")), + vec!["the", "rust", "radio", "jane"] + ); + let many: Vec = (0..50).map(|index| format!("w{index}")).collect(); + let tokens = card_index_tokens(&many.join(" "), Some("author")); + assert_eq!(tokens.len(), MAX_CARD_INDEX_TOKENS); + assert!(!tokens.contains(&"author".to_string())); + let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW); + assert_eq!(record.dht_keys().len(), MAX_RECORD_DHT_KEYS); +} + +#[test] +fn oversized_tombstone_key_lists_are_rejected() { + let key = secret(1); + let id = PodcastId::from_feed_url(&feed("show.xml")); + let keys: Vec<_> = (0..=MAX_CARD_INDEX_TOKENS) + .map(|index| PodcastDhtKey::token(&network(), &format!("t{index}"))) + .collect(); + let mut tombstone = PodcastTombstone::new(network(), key.public(), id.clone(), 2, NOW, keys); + assert_eq!(tombstone.index_keys.len(), MAX_CARD_INDEX_TOKENS); + SignedPodcastRecord::sign_tombstone(tombstone.clone(), |bytes| key.sign(bytes)).unwrap(); + tombstone + .index_keys + .push(PodcastDhtKey::from_bytes([0xff; 32])); + assert!( + SignedPodcastRecord::sign_tombstone(tombstone.clone(), |bytes| key.sign(bytes)).is_err() + ); + tombstone.index_keys.pop(); + tombstone.index_keys.reverse(); + assert!( + SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).is_err(), + "unsorted keys are not canonical" + ); +} + +// ------------------------------------------------------------------ lifetime + +#[test] +fn expiry_future_issue_and_overlong_lifetimes_are_rejected() { + let key = secret(1); + let record = card(&key, 1); + let ttl = PODCAST_CARD_TTL.as_millis() as u64; + record.verify_at(&network(), NOW + ttl - 1).unwrap(); + assert!(record.verify_at(&network(), NOW + ttl).is_err()); + + let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64; + assert!(record.verify_at(&network(), NOW - skew).is_ok()); + assert!(record.verify_at(&network(), NOW - skew - 1).is_err()); + + let mut overlong = record.clone(); + if let PodcastRecordBody::Card(card) = &mut overlong.body { + card.expires_at_ms += 1; + } + resign(&mut overlong, &key); + assert!(overlong.verify(&network()).is_err()); + + let grave = tombstone(&key, 2, NOW); + let tomb_ttl = PODCAST_TOMBSTONE_TTL.as_millis() as u64; + assert!(tomb_ttl > ttl); + grave.verify_at(&network(), NOW + tomb_ttl - 1).unwrap(); + assert!(grave.verify_at(&network(), NOW + tomb_ttl).is_err()); +} + +// ------------------------------------------------------------------ ordering + +#[test] +fn record_order_is_total_and_tombstones_win_ties() { + let key = secret(1); + let older = card(&key, 1); + let newer = card(&key, 2); + let refresh = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + 1); + let edited = card_with(spec("show.xml", "Rust Radio!"), &key, 1, NOW); + let grave = tombstone(&key, 1, NOW); + + assert!(compare_records(&newer, &older).is_gt()); + assert!(compare_records(&refresh, &older).is_gt()); + assert!(compare_records(&grave, &newer).is_lt()); + assert!(compare_records(&grave, &older).is_gt()); + assert!(compare_records(&grave, &refresh).is_gt()); + assert_ne!(compare_records(&edited, &older), std::cmp::Ordering::Equal); + assert_eq!( + compare_records(&edited, &older), + compare_records(&older, &edited).reverse() + ); + assert!(supersedes_everywhere(&grave, &older)); + assert!(supersedes_everywhere(&newer, &older)); + assert!(!supersedes_everywhere(&refresh, &older)); + assert!(!supersedes_everywhere(&older, &grave)); +} + +// --------------------------------------------------------------------- cache + +async fn open_db(temp: &tempfile::TempDir) -> store::PodcastDhtDatabase { + store::PodcastDhtDatabase::open(&temp.path().join("podcast-dht.sqlite3"), network()) + .await + .unwrap() +} + +fn entry(record: &SignedPodcastRecord) -> (Vec, SignedPodcastRecord) { + (record.dht_keys(), record.clone()) +} + +#[tokio::test] +async fn cache_replay_of_an_older_revision_is_ignored() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let newer = card(&key, 2); + let older = card(&key, 1); + let id_key = PodcastDhtKey::podcast(&network(), newer.podcast_id()); + assert!(db.store(vec![entry(&newer)], NOW).await.unwrap() > 0); + assert_eq!(db.store(vec![entry(&older)], NOW).await.unwrap(), 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![newer.clone()] + ); + // Storing the same record again changes nothing. + assert_eq!(db.store(vec![entry(&newer)], NOW).await.unwrap(), 0); +} + +#[tokio::test] +async fn cache_converges_regardless_of_delivery_order() { + let key = secret(1); + let records = [ + card(&key, 1), + card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + 5), + card_with(spec("show.xml", "Rust Radio Weekly"), &key, 2, NOW), + tombstone(&key, 2, NOW + 1), + card_with(spec("show.xml", "Rust Radio"), &key, 3, NOW + 2), + ]; + let id_key = PodcastDhtKey::podcast(&network(), records[0].podcast_id()); + let mut results = Vec::new(); + for order in [ + [0, 1, 2, 3, 4], + [4, 3, 2, 1, 0], + [3, 0, 4, 2, 1], + [2, 4, 0, 3, 1], + ] { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + for index in order { + db.store(vec![entry(&records[index])], NOW).await.unwrap(); + } + results.push( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + ); + } + assert!( + results + .iter() + .all(|result| *result == vec![records[4].clone()]) + ); +} + +#[tokio::test] +async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let active = card(&key, 1); + db.store(vec![entry(&active)], NOW).await.unwrap(); + let token_key = PodcastDhtKey::token(&network(), "rust"); + assert_eq!( + db.records_by_key(token_key, vec!["rust".into()], NOW, 10) + .await + .unwrap(), + vec![active.clone()] + ); + + // A tombstone stored only under the id key still removes the card under + // the text keys of this node. + let grave = tombstone(&key, 2, NOW); + let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id()); + db.store(vec![(vec![id_key], grave.clone())], NOW) + .await + .unwrap(); + assert_eq!( + db.records_by_key(token_key, vec!["rust".into()], NOW, 10) + .await + .unwrap(), + Vec::::new() + ); + // An equal-revision card arriving later loses, under any key. + let same = card(&key, 2); + assert_eq!(db.store(vec![entry(&same)], NOW).await.unwrap(), 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![grave.clone()] + ); + // The tombstone is retained (and served) until it expires. + let card_ttl = PODCAST_CARD_TTL.as_millis() as u64; + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW + card_ttl + 1, 10) + .await + .unwrap(), + vec![grave.clone()] + ); + // Republishing with a higher revision wins over the tombstone. + let again = card(&key, 3); + assert!(db.store(vec![entry(&again)], NOW).await.unwrap() > 0); + assert_eq!( + db.records_by_key(id_key, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![again] + ); +} + +#[tokio::test] +async fn cache_drops_stale_text_keys_of_older_revisions() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let old = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW); + let renamed = card_with(spec("show.xml", "Ferris Hour"), &key, 2, NOW); + db.store(vec![entry(&old), entry(&renamed)], NOW) + .await + .unwrap(); + let rust = PodcastDhtKey::token(&network(), "rust"); + let ferris = PodcastDhtKey::token(&network(), "ferris"); + assert!( + db.records_by_key(rust, Vec::new(), NOW, 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + db.records_by_key(ferris, Vec::new(), NOW, 10) + .await + .unwrap(), + vec![renamed] + ); +} + +#[tokio::test] +async fn cache_read_path_filters_tokens_expiry_and_corruption() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let key = secret(1); + let record = card(&key, 1); + db.store(vec![entry(&record)], NOW).await.unwrap(); + let rust = PodcastDhtKey::token(&network(), "rust"); + assert_eq!( + db.records_by_key(rust, vec!["rust".into(), "radio".into()], NOW, 10) + .await + .unwrap() + .len(), + 1 + ); + assert!( + db.records_by_key(rust, vec!["rust".into(), "jazz".into()], NOW, 10) + .await + .unwrap() + .is_empty() + ); + let ttl = PODCAST_CARD_TTL.as_millis() as u64; + assert!( + db.records_by_key(rust, Vec::new(), NOW + ttl, 10) + .await + .unwrap() + .is_empty() + ); + + // A row whose payload no longer verifies is never served. + let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap(); + let mut tampered = record.clone(); + if let PodcastRecordBody::Card(card) = &mut tampered.body { + card.title = "Rust Radio Evil".into(); + } + conn.execute( + "UPDATE podcast_dht_records SET payload = ?1", + [postcard::to_stdvec(&tampered).unwrap()], + ) + .unwrap(); + assert!( + db.records_by_key(rust, Vec::new(), NOW, 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + db.delete_expired(NOW + ttl).await.unwrap(), + MAX_RECORD_DHT_KEYS.min(record.dht_keys().len()) + ); +} + +#[tokio::test] +async fn cache_bounds_records_per_key() { + let temp = tempfile::tempdir().unwrap(); + let db = open_db(&temp).await; + let token_key = PodcastDhtKey::token(&network(), "rust"); + let mut entries = Vec::new(); + for index in 0..=MAX_PODCAST_RECORDS_PER_KEY { + let key = SecretKey::from_bytes(&{ + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes()); + bytes + }); + // Later publishers expire later, so the first one is evicted. + let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64); + entries.push((vec![token_key], record)); + } + let first = entries[0].1.clone(); + for chunk in entries.chunks(64) { + db.store(chunk.to_vec(), NOW).await.unwrap(); + } + let cached = db + .records_by_key(token_key, Vec::new(), NOW, usize::MAX) + .await + .unwrap(); + assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY); + assert!(!cached.contains(&first)); + // A record expiring before every cached one is not admitted. + assert_eq!( + db.store(vec![(vec![token_key], first)], NOW).await.unwrap(), + 0 + ); +} + +#[tokio::test] +async fn cache_refuses_a_newer_schema_and_keeps_local_publications() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("podcast-dht.sqlite3"); + let key = secret(1); + let record = card(&key, 5); + { + let db = store::PodcastDhtDatabase::open(&path, network()) + .await + .unwrap(); + db.put_local_publication( + store::LocalPublication { + podcast_id: record.podcast_id().clone(), + revision: 5, + spec: Some(record.card().unwrap().spec()), + record: record.clone(), + }, + NOW, + ) + .await + .unwrap(); + } + let db = store::PodcastDhtDatabase::open(&path, network()) + .await + .unwrap(); + let local = db.local_publications().await.unwrap(); + assert_eq!(local.len(), 1); + assert_eq!(local[0].revision, 5); + assert_eq!(local[0].record, record); + drop(db); + + rusqlite::Connection::open(&path) + .unwrap() + .pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1) + .unwrap(); + assert!( + store::PodcastDhtDatabase::open(&path, network()) + .await + .is_err() + ); +} + +// ------------------------------------------------------------------ batches + +#[test] +fn store_batches_are_bounded_and_keys_must_belong() { + let key = secret(1); + let record = card(&key, 1); + let good = PodcastStoreEntry { + keys: record.dht_keys(), + record: record.clone(), + }; + validate_store_batch(vec![good.clone()], &network(), NOW).unwrap(); + assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err()); + assert!( + validate_store_batch( + vec![good.clone(); MAX_PODCAST_STORE_RECORDS], + &network(), + NOW + ) + .is_ok() + ); + assert!( + validate_store_batch( + vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1], + &network(), + NOW + ) + .is_err() + ); + let mut foreign = good.clone(); + foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")]; + assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err()); + let mut duplicated = good.clone(); + duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]]; + assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err()); + let mut forged = good; + resign(&mut forged.record, &secret(2)); + assert!(validate_store_batch(vec![forged], &network(), NOW).is_err()); +} + +#[test] +fn filter_tokens_are_bounded() { + let at: Vec = (0..MAX_PODCAST_QUERY_TOKENS) + .map(|i| format!("t{i}")) + .collect(); + validate_filter_tokens(&at).unwrap(); + let mut over = at.clone(); + over.push("x".into()); + assert!(validate_filter_tokens(&over).is_err()); + assert!(validate_filter_tokens(&["x".repeat(MAX_INDEX_TOKEN_BYTES + 1)]).is_err()); + assert!(validate_filter_tokens(&[String::new()]).is_err()); +} + +// ------------------------------------------------------------------ privacy + +#[test] +fn card_from_subscription_is_a_public_projection_only() { + let base = spec("show.xml", "Rust Radio"); + let mut subscription = subscription(&base); + let spec = PodcastCardSpec::from_subscription(&subscription).unwrap(); + assert_eq!(spec.feed_url, base.feed_url); + assert_eq!(spec.id_feed_url, None); + let record = card_with(spec, &secret(1), 1, NOW); + let bytes = postcard::to_stdvec(&record).unwrap(); + let private = b"private-op-id-123"; + assert!(!bytes.windows(private.len()).any(|window| window == private)); + assert!( + !record + .signing_bytes() + .windows(private.len()) + .any(|window| window == private) + ); + + // After a feed move, the card binds the id to the original URL. + subscription.feed_aliases = vec![base.feed_url.clone()]; + subscription.podcast.feed_url = feed("moved.xml"); + let moved = PodcastCardSpec::from_subscription(&subscription).unwrap(); + assert_eq!(moved.feed_url, feed("moved.xml")); + assert_eq!(moved.id_feed_url, Some(base.feed_url.clone())); + moved.validate().unwrap(); + + // Without the original URL the id cannot be proven and nothing is built. + subscription.feed_aliases.clear(); + assert!(PodcastCardSpec::from_subscription(&subscription).is_err()); +} + +#[test] +fn privacy_warnings_cover_both_feed_urls() { + let mut card = spec("show.xml", "Rust Radio"); + assert!(card.privacy_warnings().is_empty()); + card.id_feed_url = + Some(FeedUrl::parse("https://feeds.example.test/show.xml?token=abc").unwrap()); + assert_eq!(card.privacy_warnings().len(), 1); +} + +// ------------------------------------------------------------------- search + +fn music(name: &str, kind: ItemKind) -> LibraryItem { + LibraryItem { + id: ItemId::from_bytes([1; 32]), + owner: secret(5).public(), + kind, + name: name.to_string(), + normalized_name: crate::normalize_name(name), + artist_names: Vec::new(), + featured_artist_names: Vec::new(), + year: None, + release_type: None, + release_title: None, + track_number: None, + disc_number: None, + duration_seconds: None, + content_id: None, + revision: 1, + deleted: false, + updated_at_ms: NOW, + } +} + +#[test] +fn hits_merge_publishers_and_drop_tombstoned_ones() { + let a = secret(1); + let b = secret(2); + let c = secret(3); + let mut moved = spec("moved.xml", "Rust Radio"); + moved.id_feed_url = Some(feed("show.xml")); + moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml")); + let records = vec![ + card(&a, 1), + card(&a, 2), + card_with(moved, &b, 1, NOW + 10), + card(&c, 1), + tombstone(&c, 2, NOW), + card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW), + ]; + let hits = rank_hits( + aggregate_hits(records), + "rust radio", + &podcast_query_tokens("rust radio"), + ); + assert_eq!(hits.len(), 1, "Rust Hour lacks the token radio"); + let hit = &hits[0]; + assert_eq!(hit.kind(), SearchHitKind::Podcast); + let mut expected = vec![a.public(), b.public()]; + expected.sort_by_key(|key| *key.as_bytes()); + assert_eq!(hit.publishers, expected); + // Equal votes per feed URL: the most recently issued card represents. + assert_eq!(hit.card.publisher, b.public()); +} + +#[test] +fn federated_results_are_typed_and_exact_matches_lead() { + let podcast = aggregate_hits(vec![card(&secret(1), 1)]).remove(0); + let merged = merge_search_results( + "rust radio", + vec![ + music("Rust Radio Theme", ItemKind::Track), + music("Rust", ItemKind::Artist), + ], + vec![podcast], + ); + let kinds: Vec<_> = merged.iter().map(FederatedSearchHit::kind).collect(); + assert_eq!( + kinds, + vec![ + SearchHitKind::Podcast, + SearchHitKind::Track, + SearchHitKind::Artist + ] + ); + let json = serde_json::to_value(&merged[0]).unwrap(); + assert_eq!(json["type"], "podcast"); + assert_eq!(json["hit"]["card"]["title"], "Rust Radio"); + let json = serde_json::to_value(&merged[1]).unwrap(); + assert_eq!(json["type"], "music"); + assert_eq!(json["hit"]["kind"], "track"); + assert_eq!(SearchHitKind::Podcast.as_str(), "podcast"); +} + +#[test] +fn query_tokens_are_normalized_distinct_and_bounded() { + assert_eq!( + podcast_query_tokens(" Rust, RUST radio!"), + vec!["rust", "radio"] + ); + let long: Vec = (0..20).map(|i| format!("w{i}")).collect(); + assert_eq!( + podcast_query_tokens(&long.join(" ")).len(), + MAX_PODCAST_QUERY_TOKENS + ); + assert!(podcast_query_tokens("!!!").is_empty()); +} + +#[test] +fn error_responses_are_bounded() { + let response = PodcastDhtResponse::error("é".repeat(MAX_ERROR_BYTES)); + assert!(response.error.as_ref().unwrap().len() <= MAX_ERROR_BYTES); + validate_response(&response).unwrap(); +}