From 38e49c492623c1bdb9d083670f6368b0a69a004b Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 15:39:50 +0200 Subject: [PATCH 1/7] Add shared podcast contract to music-dht Introduce music_dht::podcast, a pure module (no SQLite, HTTP or clock) that every Furumi client uses for podcast identity and personal state: - FeedUrl canonicalization that keeps the path and every query parameter byte-for-byte, PodcastId/EpisodeId derivation with domain-separated BLAKE3, feed aliases and the documented no-GUID episode identity rule; - bounded PodcastMeta, EpisodeMeta and the self-contained PodcastEpisodeRef queue item; - convergent merge rules for subscriptions (LWW, unsubscribe is state), share decisions (false is a tombstone), progress registers ordered by a Lamport generation, and listening sessions (G-set, one completion per session). Tests cover every delivery permutation of small operation sets, late checkpoints of old sessions, concurrent offline sessions and RSS metadata updates that must not touch progress. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/lib.rs | 1 + crates/music-dht/src/podcast/feed_url.rs | 507 +++++++++++++++++ crates/music-dht/src/podcast/id.rs | 343 ++++++++++++ crates/music-dht/src/podcast/identity.rs | 512 ++++++++++++++++++ crates/music-dht/src/podcast/meta.rs | 480 +++++++++++++++++ crates/music-dht/src/podcast/mod.rs | 298 ++++++++++ crates/music-dht/src/podcast/state.rs | 445 +++++++++++++++ crates/music-dht/src/podcast/state_tests.rs | 568 ++++++++++++++++++++ 8 files changed, 3154 insertions(+) create mode 100644 crates/music-dht/src/podcast/feed_url.rs create mode 100644 crates/music-dht/src/podcast/id.rs create mode 100644 crates/music-dht/src/podcast/identity.rs create mode 100644 crates/music-dht/src/podcast/meta.rs create mode 100644 crates/music-dht/src/podcast/mod.rs create mode 100644 crates/music-dht/src/podcast/state.rs create mode 100644 crates/music-dht/src/podcast/state_tests.rs diff --git a/crates/music-dht/src/lib.rs b/crates/music-dht/src/lib.rs index f9b8500..669e751 100644 --- a/crates/music-dht/src/lib.rs +++ b/crates/music-dht/src/lib.rs @@ -92,6 +92,7 @@ pub mod jam; mod message; mod node; mod normalization; +pub mod podcast; mod record; mod request; mod routing; diff --git a/crates/music-dht/src/podcast/feed_url.rs b/crates/music-dht/src/podcast/feed_url.rs new file mode 100644 index 0000000..ea13774 --- /dev/null +++ b/crates/music-dht/src/podcast/feed_url.rs @@ -0,0 +1,507 @@ +//! Feed URL canonicalization and feed aliases. + +use std::collections::BTreeSet; +use std::fmt; + +use serde::{Deserialize, Serialize}; + +use super::{MAX_FEED_ALIASES, MAX_URL_BYTES, PodcastError, PodcastId, PodcastResult}; + +/// Canonical podcast feed URL. +/// +/// Canonicalization ([`FeedUrl::parse`]) is deliberately small so that every +/// client, whatever its URL library, produces the same bytes: +/// +/// 1. Only `http://` and `https://` (any case) are accepted; the scheme is +/// lowercased. +/// 2. Userinfo (`user:pass@`), whitespace, control characters and `\` are +/// rejected. +/// 3. The host is lowercased. Non-ASCII hosts are rejected: pass the IDNA +/// (`xn--`) form, as produced by WHATWG URL parsers. IPv6 literals are +/// accepted in brackets. +/// 4. An empty port and the scheme's default port (80, 443) are dropped; +/// other ports lose leading zeros. +/// 5. The fragment is dropped. +/// 6. An empty path becomes `/`. The path and the **whole** query string are +/// otherwise kept byte-for-byte, including parameter order, case and +/// percent-encoding; only non-ASCII bytes are percent-encoded (`%XX`, +/// uppercase hex) so the result is ASCII. +/// +/// The canonical form is at most [`MAX_URL_BYTES`] bytes and canonicalizing it +/// again is the identity. Deserialization accepts only canonical strings. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct FeedUrl(String); + +impl FeedUrl { + /// Canonicalizes an absolute `http(s)` feed URL. + pub fn parse(input: &str) -> PodcastResult { + canonicalize(input).map(Self) + } + + /// Canonicalizes a URL typed or pasted by a user. + /// + /// Surrounding whitespace is trimmed, `feed:`, `podcast:`, `pcast:` and + /// `itpc:` prefixes map to `https://` (or to the embedded `http(s)` URL), + /// and input without a scheme defaults to `https://`. + pub fn from_user_input(input: &str) -> PodcastResult { + let trimmed = input.trim(); + if trimmed.is_empty() { + return Err(invalid("feed URL is empty")); + } + let lower = trimmed.to_ascii_lowercase(); + let mut candidate = trimmed.to_string(); + for scheme in ["feed:", "podcast:", "pcast:", "itpc:"] { + if lower.starts_with(scheme) { + let rest = &trimmed[scheme.len()..]; + let rest_lower = rest.to_ascii_lowercase(); + candidate = + if rest_lower.starts_with("http://") || rest_lower.starts_with("https://") { + rest.to_string() + } else { + format!("https://{}", rest.trim_start_matches('/')) + }; + break; + } + } + if !has_scheme(&candidate) { + candidate = format!("https://{candidate}"); + } + Self::parse(&candidate) + } + + /// Canonical string form. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for FeedUrl { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl TryFrom for FeedUrl { + type Error = PodcastError; + + fn try_from(value: String) -> PodcastResult { + if canonicalize(&value)? == value { + Ok(Self(value)) + } else { + Err(invalid("feed URL is not in canonical form")) + } + } +} + +impl From for String { + fn from(value: FeedUrl) -> String { + value.0 + } +} + +impl AsRef for FeedUrl { + fn as_ref(&self) -> &str { + &self.0 + } +} + +fn invalid(reason: &str) -> PodcastError { + PodcastError::InvalidFeedUrl(reason.to_string()) +} + +fn has_scheme(text: &str) -> bool { + let Some((scheme, rest)) = text.split_once(':') else { + return false; + }; + let mut chars = scheme.chars(); + chars.next().is_some_and(|c| c.is_ascii_alphabetic()) + && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + && rest.starts_with("//") +} + +fn canonicalize(input: &str) -> PodcastResult { + if input.len() > MAX_URL_BYTES * 3 { + return Err(invalid("feed URL is too long")); + } + if input.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err(invalid("feed URL must not contain whitespace")); + } + if input.contains('\\') { + return Err(invalid("feed URL must not contain '\\'")); + } + let (scheme, rest) = input + .split_once("://") + .ok_or_else(|| invalid("feed URL must start with http:// or https://"))?; + let (scheme, default_port) = if scheme.eq_ignore_ascii_case("http") { + ("http", 80) + } else if scheme.eq_ignore_ascii_case("https") { + ("https", 443) + } else { + return Err(invalid("only http and https feed URLs are supported")); + }; + let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + if authority.contains('@') { + return Err(invalid("feed URL must not contain credentials")); + } + let (host, port) = split_host_port(authority)?; + let host = canonical_host(host)?; + let port = match port { + None | Some("") => None, + Some(port) => { + if port.len() > 5 || !port.bytes().all(|b| b.is_ascii_digit()) { + return Err(invalid("invalid feed URL port")); + } + match port.parse::() { + Ok(value) if (1..=65_535).contains(&value) => { + (value != default_port).then_some(value) + } + _ => return Err(invalid("invalid feed URL port")), + } + } + }; + let tail = tail.split('#').next().unwrap_or_default(); + let (path, query) = match tail.find('?') { + Some(at) => tail.split_at(at), + None => (tail, ""), + }; + let mut out = String::with_capacity(input.len() + 8); + out.push_str(scheme); + out.push_str("://"); + out.push_str(&host); + if let Some(port) = port { + out.push(':'); + out.push_str(&port.to_string()); + } + if path.is_empty() { + out.push('/'); + } else { + push_ascii(&mut out, path); + } + push_ascii(&mut out, query); + if out.len() > MAX_URL_BYTES { + return Err(invalid("feed URL is too long")); + } + Ok(out) +} + +fn split_host_port(authority: &str) -> PodcastResult<(&str, Option<&str>)> { + if authority.starts_with('[') { + let close = authority + .find(']') + .ok_or_else(|| invalid("unterminated IPv6 host"))?; + let (host, after) = authority.split_at(close + 1); + return match after { + "" => Ok((host, None)), + _ => after + .strip_prefix(':') + .map(|port| (host, Some(port))) + .ok_or_else(|| invalid("invalid feed URL host")), + }; + } + Ok(match authority.rsplit_once(':') { + Some((host, port)) => (host, Some(port)), + None => (authority, None), + }) +} + +fn canonical_host(host: &str) -> PodcastResult { + if host.is_empty() { + return Err(invalid("feed URL has no host")); + } + if !host.is_ascii() { + return Err(invalid( + "feed URL host must be ASCII (use the IDNA xn-- form)", + )); + } + if let Some(inner) = host.strip_prefix('[').and_then(|h| h.strip_suffix(']')) { + if inner.is_empty() + || !inner + .bytes() + .all(|b| b.is_ascii_hexdigit() || matches!(b, b':' | b'.')) + { + return Err(invalid("invalid IPv6 host")); + } + return Ok(host.to_ascii_lowercase()); + } + if !host + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_')) + || !host.bytes().any(|b| b.is_ascii_alphanumeric()) + { + return Err(invalid("invalid feed URL host")); + } + Ok(host.to_ascii_lowercase()) +} + +fn push_ascii(out: &mut String, value: &str) { + for ch in value.chars() { + if ch.is_ascii() { + out.push(ch); + } else { + let mut buf = [0u8; 4]; + for byte in ch.encode_utf8(&mut buf).bytes() { + out.push_str(&format!("%{byte:02X}")); + } + } + } +} + +/// Resolves the podcast a canonical feed URL belongs to. +/// +/// `known` yields `(podcast_id, url)` pairs of every URL already associated +/// with a podcast: the URL of its first subscription, its current fetch URL +/// and its feed aliases. When `url` matches, the existing id is returned (the +/// smallest id if, after a concurrent offline subscription, several podcasts +/// share the URL). Otherwise a new id is derived from `url`. +/// +/// Re-subscribing through a redirect target or an `itunes:new-feed-url` +/// therefore reuses the original podcast, its progress and its history. +pub fn resolve_podcast_id<'a>( + url: &FeedUrl, + known: impl IntoIterator, +) -> PodcastId { + known + .into_iter() + .filter(|(_, alias)| *alias == url) + .map(|(id, _)| id) + .min() + .cloned() + .unwrap_or_else(|| PodcastId::from_feed_url(url)) +} + +/// Merges two feed alias sets of `podcast` as a bounded grow-only set. +/// +/// The result is sorted and duplicate-free. The URL from which `podcast` was +/// derived is always kept; of the other URLs the [`MAX_FEED_ALIASES`] +/// lexicographically smallest are kept. Taking the smallest `k` elements of a +/// union is commutative, associative and idempotent, so devices converge +/// regardless of delivery order. +pub fn merge_feed_aliases(podcast: &PodcastId, a: &[FeedUrl], b: &[FeedUrl]) -> Vec { + let mut origin = None; + let mut rest = BTreeSet::new(); + for url in a.iter().chain(b) { + if PodcastId::from_feed_url(url) == *podcast { + origin = Some(url.clone()); + } else { + rest.insert(url.clone()); + } + } + let mut out: Vec = rest.into_iter().take(MAX_FEED_ALIASES).collect(); + if let Some(origin) = origin { + out.push(origin); + out.sort(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn canon(input: &str) -> String { + FeedUrl::parse(input).unwrap().as_str().to_string() + } + + #[test] + fn canonicalization_rules() { + let cases = [ + ( + "https://example.com/feed.xml", + "https://example.com/feed.xml", + ), + ( + "HTTPS://Example.COM/Feed.XML", + "https://example.com/Feed.XML", + ), + ("http://example.com:80/rss", "http://example.com/rss"), + ("https://example.com:443/rss", "https://example.com/rss"), + ("http://example.com:443/rss", "http://example.com:443/rss"), + ( + "https://example.com:08080/rss", + "https://example.com:8080/rss", + ), + ("https://example.com:/rss", "https://example.com/rss"), + ("https://example.com", "https://example.com/"), + ("https://example.com?x=1", "https://example.com/?x=1"), + ("https://example.com/rss#latest", "https://example.com/rss"), + ("https://example.com/rss?#x", "https://example.com/rss?"), + ( + "https://example.com/a/../b/./c?b=2&a=1&A=%2f&token=XyZ", + "https://example.com/a/../b/./c?b=2&a=1&A=%2f&token=XyZ", + ), + ( + "https://example.com/rss?url=https://other.test/x?y#frag", + "https://example.com/rss?url=https://other.test/x?y", + ), + ( + "https://example.com/подкаст?q=ё", + "https://example.com/%D0%BF%D0%BE%D0%B4%D0%BA%D0%B0%D1%81%D1%82?q=%D1%91", + ), + ( + "https://[2001:DB8::1]:443/feed", + "https://[2001:db8::1]/feed", + ), + ("https://[::1]:8443/feed", "https://[::1]:8443/feed"), + ( + "https://xn--d1acufc.xn--p1ai/rss", + "https://xn--d1acufc.xn--p1ai/rss", + ), + ( + "https://my_host.example./rss", + "https://my_host.example./rss", + ), + ]; + for (input, expected) in cases { + assert_eq!(canon(input), expected, "{input:?}"); + // Idempotent. + assert_eq!(canon(expected), expected, "{expected:?}"); + } + } + + #[test] + fn rejected_urls() { + for bad in [ + "", + "example.com/feed", + "ftp://example.com/feed", + "file:///etc/passwd", + "javascript:alert(1)", + "https://", + "https:///path", + "https://user:pass@example.com/feed", + "https://token@example.com/feed", + "https://example.com/my feed", + " https://example.com/feed", + "https://example.com/feed\n", + "https://example.com\\feed", + "https://пример.рф/rss", + "https://example.com:0/rss", + "https://example.com:65536/rss", + "https://example.com:12a/rss", + "https://[::1/feed", + "https://[zz]/feed", + "https://exa%6dple.com/feed", + "https://.../feed", + ] { + assert!(FeedUrl::parse(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn length_limit_is_inclusive() { + let prefix = "https://a.test/"; + let at = format!("{prefix}{}", "x".repeat(MAX_URL_BYTES - prefix.len())); + assert_eq!(canon(&at).len(), MAX_URL_BYTES); + let over = format!("{at}x"); + assert!(FeedUrl::parse(&over).is_err()); + } + + #[test] + fn user_input_forms() { + let ok = [ + ( + "https://podster.test/rss.xml?pid=778", + "https://podster.test/rss.xml?pid=778", + ), + (" http://a.test/feed ", "http://a.test/feed"), + ("a.test/feed.xml", "https://a.test/feed.xml"), + ("feed://a.test/rss", "https://a.test/rss"), + ("feed:https://a.test/rss", "https://a.test/rss"), + ("podcast://a.test/rss", "https://a.test/rss"), + ("itpc://a.test/rss", "https://a.test/rss"), + ("HTTPS://A.TEST/Feed", "https://a.test/Feed"), + ("a.test:8080/rss", "https://a.test:8080/rss"), + ( + "example.test/rss?src=https://other.test", + "https://example.test/rss?src=https://other.test", + ), + ( + "localhost:8080/feed?u=http://x.test/", + "https://localhost:8080/feed?u=http://x.test/", + ), + ]; + for (input, expected) in ok { + assert_eq!( + FeedUrl::from_user_input(input).unwrap().as_str(), + expected, + "{input:?}" + ); + } + for bad in [ + "", + " ", + "ftp://a.test/feed", + "https://a.test/my feed", + "https://", + "file:///etc/passwd", + "javascript:alert(1)", + ] { + assert!(FeedUrl::from_user_input(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn deserialization_requires_canonical_form() { + let url = FeedUrl::parse("https://example.com/rss").unwrap(); + let json = serde_json::to_string(&url).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), url); + assert!(serde_json::from_str::("\"HTTPS://example.com/rss\"").is_err()); + assert!(serde_json::from_str::("\"https://example.com/rss#x\"").is_err()); + assert!(serde_json::from_str::("\"ftp://example.com/rss\"").is_err()); + } + + #[test] + fn query_parameters_change_identity() { + let a = PodcastId::from_feed_url(&FeedUrl::parse("https://a.test/rss?id=1").unwrap()); + let b = PodcastId::from_feed_url(&FeedUrl::parse("https://a.test/rss?id=2").unwrap()); + let c = PodcastId::from_feed_url(&FeedUrl::parse("HTTPS://A.test:443/rss?id=1#x").unwrap()); + assert_ne!(a, b); + assert_eq!(a, c); + } + + #[test] + fn aliases_resolve_to_the_original_podcast() { + let original = FeedUrl::parse("http://old.test/feed").unwrap(); + let moved = FeedUrl::parse("https://new.test/feed").unwrap(); + let id = PodcastId::from_feed_url(&original); + let known = [(&id, &original), (&id, &moved)]; + assert_eq!(resolve_podcast_id(&moved, known), id); + assert_eq!(resolve_podcast_id(&original, known), id); + let unrelated = FeedUrl::parse("https://other.test/feed").unwrap(); + assert_eq!( + resolve_podcast_id(&unrelated, known), + PodcastId::from_feed_url(&unrelated) + ); + // Ambiguity after concurrent offline subscriptions resolves to the + // smallest id, independent of iteration order. + let other = PodcastId::from_feed_url(&moved); + let ambiguous = [(&id, &moved), (&other, &moved)]; + let reversed = [(&other, &moved), (&id, &moved)]; + let expected = (&id).min(&other).clone(); + assert_eq!(resolve_podcast_id(&moved, ambiguous), expected); + assert_eq!(resolve_podcast_id(&moved, reversed), expected); + } + + #[test] + fn alias_merge_is_a_bounded_semilattice() { + let origin = FeedUrl::parse("https://zzz.test/origin").unwrap(); + let id = PodcastId::from_feed_url(&origin); + let urls: Vec = (0..(MAX_FEED_ALIASES + 4)) + .map(|i| FeedUrl::parse(&format!("https://a.test/{i:02}")).unwrap()) + .collect(); + let a: Vec = urls[..5].to_vec(); + let b: Vec = urls[3..].iter().cloned().chain([origin.clone()]).collect(); + let c: Vec = urls[7..].to_vec(); + let ab = merge_feed_aliases(&id, &a, &b); + assert_eq!(ab, merge_feed_aliases(&id, &b, &a)); + assert_eq!(ab, merge_feed_aliases(&id, &ab, &ab)); + let left = merge_feed_aliases(&id, &ab, &c); + let right = merge_feed_aliases(&id, &a, &merge_feed_aliases(&id, &b, &c)); + assert_eq!(left, right); + assert_eq!(left.len(), MAX_FEED_ALIASES + 1); + assert!(left.contains(&origin)); + assert!(left.windows(2).all(|w| w[0] < w[1])); + } +} diff --git a/crates/music-dht/src/podcast/id.rs b/crates/music-dht/src/podcast/id.rs new file mode 100644 index 0000000..976ab3d --- /dev/null +++ b/crates/music-dht/src/podcast/id.rs @@ -0,0 +1,343 @@ +//! Cross-device podcast and episode identifiers. + +use std::fmt; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; + +use super::{FeedUrl, MAX_GUID_BYTES, PodcastError, PodcastResult}; + +/// BLAKE3 derivation domain of [`PodcastId`]. Changing it changes every id. +pub const PODCAST_ID_DOMAIN: &str = "furumi-podcast:podcast:v1"; +/// BLAKE3 derivation domain of [`EpisodeId`]. Changing it changes every id. +pub const EPISODE_ID_DOMAIN: &str = "furumi-podcast:episode:v1"; +/// String prefix of a [`PodcastId`]. +pub const PODCAST_ID_PREFIX: &str = "pod1:"; +/// String prefix of an [`EpisodeId`]. +pub const EPISODE_ID_PREFIX: &str = "ep1:"; + +/// Length of the base32 body: 32 bytes, RFC 4648 lowercase, no padding. +const DIGEST_CHARS: usize = 52; + +fn digest(domain: &str, parts: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain.as_bytes()); + hasher.update(&[0]); + for part in parts { + hasher.update(part); + } + *hasher.finalize().as_bytes() +} + +fn encode(prefix: &str, digest: &[u8; 32]) -> String { + let mut body = data_encoding::BASE32_NOPAD.encode(digest); + body.make_ascii_lowercase(); + format!("{prefix}{body}") +} + +fn decode(prefix: &str, value: &str) -> Option<[u8; 32]> { + let body = value.strip_prefix(prefix)?; + if body.len() != DIGEST_CHARS + || !body + .bytes() + .all(|b| b.is_ascii_lowercase() || (b'2'..=b'7').contains(&b)) + { + return None; + } + let bytes = data_encoding::BASE32_NOPAD + .decode(body.to_ascii_uppercase().as_bytes()) + .ok()?; + let digest: [u8; 32] = bytes.try_into().ok()?; + // Reject non-canonical trailing bits so every id has one spelling. + (encode(prefix, &digest) == value).then_some(digest) +} + +macro_rules! string_id { + ($name:ident, $prefix:expr, $kind:literal) => { + impl $name { + /// Parses and validates the canonical string form. + pub fn parse(value: &str) -> PodcastResult { + decode($prefix, value) + .map(|_| Self(value.to_string())) + .ok_or(PodcastError::InvalidId { kind: $kind }) + } + + /// Canonical string form. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The 32-byte BLAKE3 digest encoded in the id. + pub fn digest(&self) -> [u8; 32] { + decode($prefix, &self.0).expect("validated on construction") + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } + } + + impl FromStr for $name { + type Err = PodcastError; + + fn from_str(value: &str) -> PodcastResult { + Self::parse(value) + } + } + + impl TryFrom for $name { + type Error = PodcastError; + + fn try_from(value: String) -> PodcastResult { + decode($prefix, &value) + .map(|_| Self(value)) + .ok_or(PodcastError::InvalidId { kind: $kind }) + } + } + + impl From<$name> for String { + fn from(value: $name) -> String { + value.0 + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + }; +} + +/// Stable cross-device podcast identity: `pod1:<52 lowercase base32 chars>`. +/// +/// Derived as `BLAKE3("furumi-podcast:podcast:v1" || 0x00 || canonical_feed_url)` +/// from the URL used by the **first** subscription. The id never changes when +/// the feed moves; new URLs are recorded as aliases (see +/// [`super::resolve_podcast_id`]). Deserialization validates the format. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct PodcastId(String); + +string_id!(PodcastId, PODCAST_ID_PREFIX, "podcast"); + +impl PodcastId { + /// Derives the id of a podcast first subscribed through `url`. + pub fn from_feed_url(url: &FeedUrl) -> Self { + Self(encode( + PODCAST_ID_PREFIX, + &digest(PODCAST_ID_DOMAIN, &[url.as_str().as_bytes()]), + )) + } +} + +/// Stable cross-device episode identity: `ep1:<52 lowercase base32 chars>`. +/// +/// Derived as `BLAKE3("furumi-podcast:episode:v1" || 0x00 || podcast_id || key)` +/// where `key` is the material of an [`EpisodeKey`]. Title, artwork or audio +/// URL changes never change the id of an existing episode. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct EpisodeId(String); + +string_id!(EpisodeId, EPISODE_ID_PREFIX, "episode"); + +impl EpisodeId { + /// Derives an episode id from a key inside `podcast`. + pub fn derive(podcast: &PodcastId, key: &EpisodeKey) -> Self { + let material = key.material(); + Self(encode( + EPISODE_ID_PREFIX, + &digest( + EPISODE_ID_DOMAIN, + &[podcast.as_str().as_bytes(), material.as_bytes()], + ), + )) + } + + /// Derives the id of an episode with an RSS GUID (or Atom ``). + /// + /// Returns `None` when the GUID is blank or longer than [`MAX_GUID_BYTES`]. + pub fn from_guid(podcast: &PodcastId, guid: &str) -> Option { + EpisodeKey::guid(guid).map(|key| Self::derive(podcast, &key)) + } +} + +/// Key material from which an [`EpisodeId`] is derived. +/// +/// | Variant | Material | +/// | --- | --- | +/// | `Guid` | `guid:` + GUID trimmed of surrounding whitespace | +/// | `Link` | `link:` + `` trimmed | +/// | `DateTitle` | `date-title:` + unix seconds + `\0` + [`super::normalize_episode_title`] | +/// | `Enclosure` | `enclosure:` + enclosure URL trimmed | +/// +/// GUIDs and URLs are otherwise byte-for-byte: GUIDs are case-sensitive. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum EpisodeKey { + /// RSS `` or Atom ``. + Guid(String), + /// Item ``. + Link(String), + /// Publication time (whole unix seconds) and normalized title. + DateTitle { + /// Publication time in unix seconds. + published_at_secs: i64, + /// Title normalized by [`super::normalize_episode_title`]. + normalized_title: String, + }, + /// Enclosure (audio) URL. + Enclosure(String), +} + +impl EpisodeKey { + /// GUID key; `None` for a blank or oversized GUID. + pub fn guid(guid: &str) -> Option { + let guid = guid.trim(); + (!guid.is_empty() && guid.len() <= MAX_GUID_BYTES).then(|| Self::Guid(guid.to_string())) + } + + /// Link key; `None` for a blank or oversized link. + pub fn link(link: &str) -> Option { + let link = link.trim(); + (!link.is_empty() && link.len() <= super::MAX_URL_BYTES) + .then(|| Self::Link(link.to_string())) + } + + /// Date + title key; `None` when the title normalizes to nothing. + pub fn date_title(published_at_ms: i64, title: &str) -> Option { + let normalized_title = super::normalize_episode_title(title); + (!normalized_title.is_empty()).then(|| Self::DateTitle { + published_at_secs: published_at_ms.div_euclid(1_000), + normalized_title, + }) + } + + /// Enclosure key; `None` for a blank or oversized URL. + pub fn enclosure(url: &str) -> Option { + let url = url.trim(); + (!url.is_empty() && url.len() <= super::MAX_URL_BYTES) + .then(|| Self::Enclosure(url.to_string())) + } + + /// Exact bytes hashed after the podcast id. + pub fn material(&self) -> String { + match self { + Self::Guid(guid) => format!("guid:{guid}"), + Self::Link(link) => format!("link:{link}"), + Self::DateTitle { + published_at_secs, + normalized_title, + } => format!("date-title:{published_at_secs}\0{normalized_title}"), + Self::Enclosure(url) => format!("enclosure:{url}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn podcast() -> PodcastId { + PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed.xml").unwrap()) + } + + #[test] + fn podcast_id_is_domain_separated_blake3() { + let url = FeedUrl::parse("https://example.test/feed.xml").unwrap(); + let mut input = b"furumi-podcast:podcast:v1\0".to_vec(); + input.extend_from_slice(url.as_str().as_bytes()); + let expected = blake3::hash(&input); + let id = PodcastId::from_feed_url(&url); + assert_eq!(&id.digest(), expected.as_bytes()); + assert_eq!(id.as_str().len(), PODCAST_ID_PREFIX.len() + DIGEST_CHARS); + assert!(id.as_str()[5..].bytes().all(|b| !b.is_ascii_uppercase())); + } + + #[test] + fn podcast_id_golden_value() { + // Golden vector: changing it breaks every stored and synced id. + let id = podcast(); + let mut input = b"furumi-podcast:podcast:v1\0".to_vec(); + input.extend_from_slice(b"https://example.test/feed.xml"); + let mut body = data_encoding::BASE32_NOPAD.encode(blake3::hash(&input).as_bytes()); + body.make_ascii_lowercase(); + assert_eq!(id.as_str(), format!("pod1:{body}")); + } + + #[test] + fn episode_id_follows_the_plan_formula() { + let podcast = podcast(); + let id = EpisodeId::from_guid(&podcast, " abc-123 ").unwrap(); + let mut input = b"furumi-podcast:episode:v1\0".to_vec(); + input.extend_from_slice(podcast.as_str().as_bytes()); + input.extend_from_slice(b"guid:abc-123"); + assert_eq!(&id.digest(), blake3::hash(&input).as_bytes()); + // GUIDs are case-sensitive and podcast-scoped. + assert_ne!(EpisodeId::from_guid(&podcast, "ABC-123"), Some(id.clone())); + let other = PodcastId::from_feed_url(&FeedUrl::parse("https://other.test/").unwrap()); + assert_ne!(EpisodeId::from_guid(&other, "abc-123"), Some(id)); + } + + #[test] + fn keys_are_distinct_per_kind() { + let podcast = podcast(); + let value = "https://example.test/e1"; + let link = EpisodeId::derive(&podcast, &EpisodeKey::link(value).unwrap()); + let guid = EpisodeId::derive(&podcast, &EpisodeKey::guid(value).unwrap()); + let enclosure = EpisodeId::derive(&podcast, &EpisodeKey::enclosure(value).unwrap()); + assert_ne!(link, guid); + assert_ne!(link, enclosure); + assert_ne!(guid, enclosure); + } + + #[test] + fn blank_and_oversized_keys_are_rejected() { + assert!(EpisodeKey::guid(" ").is_none()); + assert!(EpisodeKey::guid(&"g".repeat(MAX_GUID_BYTES)).is_some()); + assert!(EpisodeKey::guid(&"g".repeat(MAX_GUID_BYTES + 1)).is_none()); + assert!(EpisodeKey::date_title(0, " -- ").is_none()); + assert!(EpisodeKey::link("").is_none()); + } + + #[test] + fn date_title_uses_whole_seconds_and_normalized_title() { + let a = EpisodeKey::date_title(1_700_000_000_123, "Hello, World!").unwrap(); + let b = EpisodeKey::date_title(1_700_000_000_999, "hello world").unwrap(); + assert_eq!(a, b); + let c = EpisodeKey::date_title(-1, "x").unwrap(); + assert_eq!( + c, + EpisodeKey::DateTitle { + published_at_secs: -1, + normalized_title: "x".to_string() + } + ); + } + + #[test] + fn parsing_validates_and_round_trips() { + let id = podcast(); + assert_eq!(PodcastId::parse(id.as_str()).unwrap(), id); + let json = serde_json::to_string(&id).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), id); + + let upper = id.as_str().to_ascii_uppercase().replace("POD1:", "pod1:"); + assert!(PodcastId::parse(&upper).is_err()); + assert!(PodcastId::parse(&id.as_str()[..id.as_str().len() - 1]).is_err()); + assert!(PodcastId::parse(&format!("{id}a")).is_err()); + assert!(PodcastId::parse("pod1:").is_err()); + assert!(PodcastId::parse("").is_err()); + assert!(EpisodeId::parse(id.as_str()).is_err()); + assert!(serde_json::from_str::("\"pod1:xyz\"").is_err()); + + // Non-canonical trailing bits: last char encodes 1 data bit + 4 zero bits. + let mut chars: Vec = id.as_str().chars().collect(); + let last = chars.len() - 1; + chars[last] = if chars[last] == 'b' { 'c' } else { 'b' }; + let tampered: String = chars.into_iter().collect(); + assert!(PodcastId::parse(&tampered).is_err()); + } +} diff --git a/crates/music-dht/src/podcast/identity.rs b/crates/music-dht/src/podcast/identity.rs new file mode 100644 index 0000000..af91112 --- /dev/null +++ b/crates/music-dht/src/podcast/identity.rs @@ -0,0 +1,512 @@ +//! Episode identity resolution, including the documented no-GUID rule. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use super::{EpisodeId, EpisodeKey, PodcastId}; + +/// Normalizes an episode title for the `(date, title)` identity key. +/// +/// This is [`crate::normalize_name`]: NFKC, lowercase, non-alphanumeric runs +/// collapsed to one space, trimmed. It is part of the episode-id derivation. +pub fn normalize_episode_title(title: &str) -> String { + crate::normalize_name(title) +} + +/// Fields of a feed item (or of a stored episode) that identify it. +/// +/// The parser in `podcast::feed` produces these hints; stores keep them for +/// known episodes so that a later refresh can recognize items without GUID. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct EpisodeIdentityHints { + /// RSS `` / Atom ``, when present. + #[serde(default)] + pub guid: Option, + /// Enclosure (audio) URL. + #[serde(default)] + pub enclosure_url: Option, + /// Item ``. + #[serde(default)] + pub link: Option, + /// Publication time in unix milliseconds. + #[serde(default)] + pub published_at_ms: Option, + /// Item title. + #[serde(default)] + pub title: Option, +} + +impl EpisodeIdentityHints { + fn guid_key(&self) -> Option { + self.guid.as_deref().and_then(EpisodeKey::guid) + } + + fn enclosure(&self) -> Option<&str> { + trimmed(self.enclosure_url.as_deref()) + } + + fn link(&self) -> Option<&str> { + trimmed(self.link.as_deref()) + } + + fn date_title(&self) -> Option<(i64, String)> { + let published = self.published_at_ms?.div_euclid(1_000); + let title = normalize_episode_title(self.title.as_deref()?); + (!title.is_empty()).then_some((published, title)) + } +} + +struct Usable<'a> { + link: Option<&'a str>, + enclosure: Option<&'a str>, + date_title: Option<(i64, String)>, +} + +fn trimmed(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +/// An episode already stored for the podcast, with the hints it was last seen with. +/// +/// Include episodes learned through sync (progress operations, queue items) +/// as well as episodes from earlier RSS refreshes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct KnownEpisode { + /// Existing stable id. + pub episode_id: EpisodeId, + /// Identity hints recorded for it. + pub hints: EpisodeIdentityHints, +} + +/// Resolves the id of a single feed item. See [`resolve_episode_ids`]. +/// +/// A single item has no feed context, so its `` is only used when it +/// differs from `channel_link`. Prefer [`resolve_episode_ids`] for a whole +/// feed refresh. +pub fn resolve_episode_id( + podcast: &PodcastId, + channel_link: Option<&str>, + item: &EpisodeIdentityHints, + known: &[KnownEpisode], +) -> Option { + resolve_episode_ids(podcast, channel_link, std::slice::from_ref(item), known) + .pop() + .flatten() +} + +/// Resolves stable ids for the items of one feed document. +/// +/// Returns one entry per item, in order; `None` marks an item that cannot be +/// identified (no usable key) or that duplicates an earlier item of the same +/// document. The rule (normative, see `PODCASTS.md`): +/// +/// 1. An item with a GUID gets `EpisodeId::from_guid`. A GUID never matches +/// a different known episode, so the id is independent of local history. +/// 2. An item without a GUID first looks for a known episode, in this order: +/// same enclosure URL, then same ``, then same publication second and +/// [`normalize_episode_title`]. The first stage with candidates wins; among +/// candidates the smallest id is reused. A known episode already claimed by +/// another item of this document is not a candidate. +/// 3. Otherwise a new id is derived from the first available key: ``, +/// then `(date, title)`, then enclosure URL. +/// +/// A `` is only identifying when it is unique within the document and +/// differs from the channel link; many feeds repeat the show homepage in every +/// item. The same applies to enclosure URLs and `(date, title)` pairs shared +/// by several GUID-less items. +pub fn resolve_episode_ids( + podcast: &PodcastId, + channel_link: Option<&str>, + items: &[EpisodeIdentityHints], + known: &[KnownEpisode], +) -> Vec> { + let channel_link = trimmed(channel_link); + let mut link_counts: BTreeMap<&str, usize> = BTreeMap::new(); + let mut enclosure_counts: BTreeMap<&str, usize> = BTreeMap::new(); + let mut date_title_counts: BTreeMap<(i64, String), usize> = BTreeMap::new(); + for item in items { + if let Some(link) = item.link() { + *link_counts.entry(link).or_default() += 1; + } + if item.guid_key().is_none() { + if let Some(url) = item.enclosure() { + *enclosure_counts.entry(url).or_default() += 1; + } + if let Some(pair) = item.date_title() { + *date_title_counts.entry(pair).or_default() += 1; + } + } + } + let usable: Vec> = items + .iter() + .map(|item| Usable { + link: item + .link() + .filter(|link| Some(*link) != channel_link && link_counts.get(link) == Some(&1)), + enclosure: item + .enclosure() + .filter(|url| enclosure_counts.get(url) == Some(&1)), + date_title: item + .date_title() + .filter(|pair| date_title_counts.get(pair) == Some(&1)), + }) + .collect(); + + let mut claimed: BTreeSet = BTreeSet::new(); + let mut out: Vec> = vec![None; items.len()]; + + // Pass 1: GUID items claim their derived ids first. + for (index, item) in items.iter().enumerate() { + if let Some(key) = item.guid_key() { + let id = EpisodeId::derive(podcast, &key); + if claimed.insert(id.clone()) { + out[index] = Some(id); + } + } + } + + // Pass 2: GUID-less items reuse a matching known episode. + let mut pending = Vec::new(); + for (index, item) in items.iter().enumerate() { + if item.guid_key().is_some() { + continue; + } + let own = &usable[index]; + let stages: [&dyn Fn(&EpisodeIdentityHints) -> bool; 3] = [ + &|candidate| { + own.enclosure + .is_some_and(|url| candidate.enclosure() == Some(url)) + }, + &|candidate| own.link.is_some_and(|link| candidate.link() == Some(link)), + &|candidate| { + own.date_title + .as_ref() + .is_some_and(|pair| candidate.date_title().as_ref() == Some(pair)) + }, + ]; + let reused = stages.iter().find_map(|matches| { + known + .iter() + .filter(|known| !claimed.contains(&known.episode_id) && matches(&known.hints)) + .map(|known| &known.episode_id) + .min() + .cloned() + }); + match reused { + Some(id) => { + claimed.insert(id.clone()); + out[index] = Some(id); + } + None => pending.push(index), + } + } + + // Pass 3: derive new ids from the first available fallback key. + for index in pending { + let own = &usable[index]; + let key = own + .link + .and_then(EpisodeKey::link) + .or_else(|| { + own.date_title + .as_ref() + .map(|(secs, title)| EpisodeKey::DateTitle { + published_at_secs: *secs, + normalized_title: title.clone(), + }) + }) + .or_else(|| own.enclosure.and_then(EpisodeKey::enclosure)); + if let Some(key) = key { + let id = EpisodeId::derive(podcast, &key); + if claimed.insert(id.clone()) { + out[index] = Some(id); + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::podcast::FeedUrl; + + fn podcast() -> PodcastId { + PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed").unwrap()) + } + + fn hints( + guid: Option<&str>, + enclosure: Option<&str>, + link: Option<&str>, + date: Option, + title: Option<&str>, + ) -> EpisodeIdentityHints { + EpisodeIdentityHints { + guid: guid.map(str::to_string), + enclosure_url: enclosure.map(str::to_string), + link: link.map(str::to_string), + published_at_ms: date, + title: title.map(str::to_string), + } + } + + fn derived(key: EpisodeKey) -> Option { + Some(EpisodeId::derive(&podcast(), &key)) + } + + #[test] + fn guid_wins_and_ignores_history() { + let item = hints(Some("g1"), Some("https://cdn/1.mp3"), None, None, Some("T")); + let known = [KnownEpisode { + episode_id: EpisodeId::from_guid(&podcast(), "other").unwrap(), + hints: hints(None, Some("https://cdn/1.mp3"), None, None, None), + }]; + assert_eq!( + resolve_episode_id(&podcast(), None, &item, &known), + derived(EpisodeKey::guid("g1").unwrap()) + ); + } + + #[test] + fn title_cover_and_audio_url_changes_keep_the_guid_id() { + let before = hints( + Some("g1"), + Some("https://cdn/a.mp3"), + None, + Some(1), + Some("Old"), + ); + let after = hints( + Some("g1"), + Some("https://cdn/b.mp3"), + None, + Some(2), + Some("New"), + ); + assert_eq!( + resolve_episode_id(&podcast(), None, &before, &[]), + resolve_episode_id(&podcast(), None, &after, &[]) + ); + } + + #[test] + fn fallback_key_order_for_new_items() { + let p = podcast(); + let with_link = hints( + None, + Some("https://cdn/1.mp3"), + Some("https://e/1"), + Some(5_000), + Some("A"), + ); + assert_eq!( + resolve_episode_id(&p, None, &with_link, &[]), + derived(EpisodeKey::link("https://e/1").unwrap()) + ); + let date_title = hints( + None, + Some("https://cdn/1.mp3"), + None, + Some(5_000), + Some("A"), + ); + assert_eq!( + resolve_episode_id(&p, None, &date_title, &[]), + derived(EpisodeKey::date_title(5_000, "A").unwrap()) + ); + let enclosure_only = hints(None, Some("https://cdn/1.mp3"), None, None, Some("A")); + assert_eq!( + resolve_episode_id(&p, None, &enclosure_only, &[]), + derived(EpisodeKey::enclosure("https://cdn/1.mp3").unwrap()) + ); + assert_eq!( + resolve_episode_id(&p, None, &EpisodeIdentityHints::default(), &[]), + None + ); + } + + #[test] + fn guidless_episode_keeps_its_id_after_title_and_url_changes() { + let p = podcast(); + // First seen: no link, so the id comes from (date, title). + let first = hints( + None, + Some("https://cdn/a.mp3"), + None, + Some(9_000), + Some("Pilot"), + ); + let id = resolve_episode_id(&p, None, &first, &[]).unwrap(); + let known = [KnownEpisode { + episode_id: id.clone(), + hints: first.clone(), + }]; + // Title changed: matched by enclosure. + let retitled = hints( + None, + Some("https://cdn/a.mp3"), + None, + Some(9_000), + Some("Pilot (remastered)"), + ); + assert_eq!( + resolve_episode_id(&p, None, &retitled, &known), + Some(id.clone()) + ); + // Audio URL changed: matched by (date, title). + let moved = hints( + None, + Some("https://cdn2/a.mp3"), + None, + Some(9_999), + Some("PILOT!"), + ); + assert_eq!( + resolve_episode_id(&p, None, &moved, &known), + Some(id.clone()) + ); + // Link appeared later: the enclosure still matches first. + let linked = hints( + None, + Some("https://cdn/a.mp3"), + Some("https://e/pilot"), + None, + None, + ); + assert_eq!(resolve_episode_id(&p, None, &linked, &known), Some(id)); + } + + #[test] + fn shared_channel_links_are_not_identifying() { + let p = podcast(); + let home = "https://show.test/"; + let items = [ + hints(None, Some("https://cdn/1.mp3"), Some(home), None, None), + hints(None, Some("https://cdn/2.mp3"), Some(home), None, None), + ]; + let ids = resolve_episode_ids(&p, Some(home), &items, &[]); + assert_eq!( + ids[0], + derived(EpisodeKey::enclosure("https://cdn/1.mp3").unwrap()) + ); + assert_eq!( + ids[1], + derived(EpisodeKey::enclosure("https://cdn/2.mp3").unwrap()) + ); + // The same holds without the channel link: the link repeats. + assert_eq!(resolve_episode_ids(&p, None, &items, &[]), ids); + // A single item whose link equals the channel link. + assert_eq!(resolve_episode_id(&p, Some(home), &items[0], &[]), ids[0]); + } + + #[test] + fn duplicates_in_one_document_are_dropped() { + let p = podcast(); + let items = [ + hints(Some("g"), Some("https://cdn/1.mp3"), None, None, None), + hints(Some("g"), Some("https://cdn/2.mp3"), None, None, None), + hints(None, Some("https://cdn/3.mp3"), None, None, None), + hints(None, Some("https://cdn/3.mp3"), None, None, None), + ]; + let ids = resolve_episode_ids(&p, None, &items, &[]); + assert!(ids[0].is_some()); + assert_eq!(ids[1], None); + // The shared enclosure is not identifying: both GUID-less items have + // no usable key. + assert_eq!(ids[2], None); + assert_eq!(ids[3], None); + } + + #[test] + fn a_known_episode_is_claimed_once() { + let p = podcast(); + let known_id = derived(EpisodeKey::enclosure("https://cdn/old.mp3").unwrap()).unwrap(); + let known = [KnownEpisode { + episode_id: known_id.clone(), + hints: hints( + None, + Some("https://cdn/old.mp3"), + None, + Some(1_000), + Some("Same"), + ), + }]; + // Two new items both match by (date, title) only if unique; here + // they share it, so neither matches; the enclosure one matches. + let items = [ + hints( + None, + Some("https://cdn/x.mp3"), + None, + Some(1_000), + Some("Same"), + ), + hints( + None, + Some("https://cdn/old.mp3"), + None, + Some(1_000), + Some("Same"), + ), + ]; + let ids = resolve_episode_ids(&p, None, &items, &known); + assert_eq!(ids[1], Some(known_id.clone())); + assert_eq!( + ids[0], + derived(EpisodeKey::enclosure("https://cdn/x.mp3").unwrap()) + ); + assert_ne!(ids[0], ids[1]); + } + + #[test] + fn ambiguous_known_matches_pick_the_smallest_id_in_any_order() { + let p = podcast(); + let a = EpisodeId::from_guid(&p, "a").unwrap(); + let b = EpisodeId::from_guid(&p, "b").unwrap(); + let same = hints(None, Some("https://cdn/1.mp3"), None, None, None); + let k1 = KnownEpisode { + episode_id: a.clone(), + hints: same.clone(), + }; + let k2 = KnownEpisode { + episode_id: b.clone(), + hints: same.clone(), + }; + let expected = Some(a.clone().min(b.clone())); + assert_eq!( + resolve_episode_id(&p, None, &same, &[k1.clone(), k2.clone()]), + expected + ); + assert_eq!(resolve_episode_id(&p, None, &same, &[k2, k1]), expected); + } + + #[test] + fn result_is_independent_of_known_order() { + let p = podcast(); + let items: Vec = (0..6) + .map(|i| { + hints( + None, + Some(&format!("https://cdn/{i}.mp3")), + Some(&format!("https://e/{i}")), + Some(i * 1_000), + Some(&format!("Ep {i}")), + ) + }) + .collect(); + let mut known: Vec = items + .iter() + .enumerate() + .map(|(i, h)| KnownEpisode { + episode_id: EpisodeId::from_guid(&p, &format!("legacy-{i}")).unwrap(), + hints: h.clone(), + }) + .collect(); + let forward = resolve_episode_ids(&p, None, &items, &known); + known.reverse(); + assert_eq!(resolve_episode_ids(&p, None, &items, &known), forward); + assert!(forward.iter().all(Option::is_some)); + } +} diff --git a/crates/music-dht/src/podcast/meta.rs b/crates/music-dht/src/podcast/meta.rs new file mode 100644 index 0000000..1f36ff7 --- /dev/null +++ b/crates/music-dht/src/podcast/meta.rs @@ -0,0 +1,480 @@ +//! Podcast and episode metadata DTOs. + +use serde::{Deserialize, Serialize}; + +use super::{ + EpisodeId, EpisodeIdentityHints, FeedUrl, MAX_DESCRIPTION_CHARS, MAX_GUID_BYTES, + MAX_MIME_BYTES, MAX_TITLE_CHARS, MAX_URL_BYTES, PodcastError, PodcastId, PodcastResult, + bounded_text, check_opt_text, check_opt_url, check_text, check_url, +}; + +/// `itunes:episodeType`. +/// +/// Unknown values deserialize (and parse) as [`EpisodeType::Full`], the +/// iTunes default, so publishers cannot break the wire format. +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, +)] +#[serde(rename_all = "snake_case", from = "String")] +pub enum EpisodeType { + /// Regular episode. + #[default] + Full, + /// Trailer or teaser. + Trailer, + /// Bonus content. + Bonus, +} + +impl EpisodeType { + /// Parses an `itunes:episodeType` value, case-insensitively. + pub fn parse(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "trailer" => Self::Trailer, + "bonus" => Self::Bonus, + _ => Self::Full, + } + } +} + +impl From for EpisodeType { + fn from(value: String) -> Self { + Self::parse(&value) + } +} + +/// Channel metadata of a podcast, as last fetched from its feed. +/// +/// Synced inside subscription operations so that another trusted device can +/// show and refresh the podcast without the user typing the URL again. It is +/// private user data: never publish it outside the trusted-device group. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct PodcastMeta { + /// Canonical URL the feed is currently fetched from. + pub feed_url: FeedUrl, + /// Podcast title. + pub title: String, + /// Author. + #[serde(default)] + pub author: Option, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + #[serde(default)] + pub description: Option, + /// Publisher web page. + #[serde(default)] + pub link: Option, + /// Artwork URL. + #[serde(default)] + pub image_url: Option, +} + +impl PodcastMeta { + /// Validates contract bounds. + pub fn validate(&self) -> PodcastResult<()> { + check_text("podcast title", &self.title, MAX_TITLE_CHARS, true)?; + check_opt_text("podcast author", self.author.as_deref(), MAX_TITLE_CHARS)?; + check_opt_text( + "podcast description", + self.description.as_deref(), + MAX_DESCRIPTION_CHARS, + )?; + if self + .link + .as_ref() + .is_some_and(|link| link.len() > MAX_URL_BYTES) + { + return Err(PodcastError::InvalidField { + field: "podcast link", + reason: "is too long", + }); + } + check_opt_url("podcast image_url", self.image_url.as_deref()) + } + + /// Returns a copy that fits the contract bounds. + /// + /// Text is truncated with [`bounded_text`]; invalid optional URLs are + /// dropped; an empty title becomes `"Untitled podcast"`. + pub fn bounded(&self) -> Self { + Self { + feed_url: self.feed_url.clone(), + title: bounded_title(&self.title, "Untitled podcast"), + author: bounded_opt(self.author.as_deref(), MAX_TITLE_CHARS), + description: bounded_opt(self.description.as_deref(), MAX_DESCRIPTION_CHARS), + link: self + .link + .clone() + .filter(|link| link.len() <= MAX_URL_BYTES && !link.trim().is_empty()), + image_url: valid_url(self.image_url.as_deref()), + } + } +} + +/// Full episode metadata, as stored by clients and refreshed from RSS. +/// +/// Publisher metadata never touches personal progress or history: those are +/// separate records keyed by [`EpisodeId`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EpisodeMeta { + /// Stable episode id. + pub episode_id: EpisodeId, + /// Owning podcast. + pub podcast_id: PodcastId, + /// RSS GUID / Atom id, when present. + #[serde(default)] + pub guid: Option, + /// Episode title. + pub title: String, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + #[serde(default)] + pub description: Option, + /// `itunes:season`. Episodes without a season form their own group. + #[serde(default)] + pub season: Option, + /// `itunes:episode`. + #[serde(default)] + pub episode_number: Option, + /// `itunes:episodeType`. + #[serde(default)] + pub episode_type: EpisodeType, + /// Publication time in unix milliseconds. + #[serde(default)] + pub published_at_ms: Option, + /// Duration announced by the publisher, in milliseconds. + #[serde(default)] + pub duration_ms: Option, + /// Absolute http(s) audio URL at the publisher. + pub audio_url: String, + /// Enclosure MIME type. + #[serde(default)] + pub mime: Option, + /// Enclosure length in bytes. + #[serde(default)] + pub byte_len: Option, + /// Episode artwork URL. + #[serde(default)] + pub image_url: Option, + /// Episode web page. + #[serde(default)] + pub link: Option, +} + +impl EpisodeMeta { + /// Validates contract bounds. + pub fn validate(&self) -> PodcastResult<()> { + if self + .guid + .as_ref() + .is_some_and(|guid| guid.len() > MAX_GUID_BYTES) + { + return Err(PodcastError::InvalidField { + field: "episode guid", + reason: "is too long", + }); + } + check_text("episode title", &self.title, MAX_TITLE_CHARS, true)?; + check_opt_text( + "episode description", + self.description.as_deref(), + MAX_DESCRIPTION_CHARS, + )?; + check_duration(self.duration_ms)?; + check_url("episode audio_url", &self.audio_url)?; + check_mime(self.mime.as_deref())?; + check_opt_url("episode image_url", self.image_url.as_deref())?; + if self + .link + .as_ref() + .is_some_and(|link| link.len() > MAX_URL_BYTES) + { + return Err(PodcastError::InvalidField { + field: "episode link", + reason: "is too long", + }); + } + Ok(()) + } + + /// Identity hints to keep for the no-GUID matching rule. + pub fn identity_hints(&self) -> EpisodeIdentityHints { + EpisodeIdentityHints { + guid: self.guid.clone(), + enclosure_url: Some(self.audio_url.clone()), + link: self.link.clone(), + published_at_ms: self.published_at_ms, + title: Some(self.title.clone()), + } + } + + /// Builds the self-contained queue reference for this episode. + /// + /// The episode artwork falls back to the podcast artwork. + pub fn to_ref(&self, podcast: &PodcastMeta) -> PodcastEpisodeRef { + PodcastEpisodeRef { + podcast_id: self.podcast_id.clone(), + episode_id: self.episode_id.clone(), + feed_url: podcast.feed_url.clone(), + podcast_title: bounded_title(&podcast.title, "Untitled podcast"), + episode_title: bounded_title(&self.title, "Untitled episode"), + audio_url: self.audio_url.clone(), + mime: self + .mime + .clone() + .filter(|mime| check_mime(Some(mime)).is_ok()), + duration_ms: self.duration_ms.filter(|ms| *ms > 0), + published_at_ms: self.published_at_ms, + season: self.season, + episode_number: self.episode_number, + image_url: valid_url(self.image_url.as_deref()) + .or_else(|| valid_url(podcast.image_url.as_deref())), + } + } +} + +/// Self-contained, bounded reference to a podcast episode. +/// +/// Carried by playback queues (`PlaybackSource::PodcastEpisode`) and by +/// progress operations, so a receiving trusted device can display, resume and +/// play the episode without being subscribed and without its own copy of the +/// feed. It contains no personal state. It is private: Jam snapshots replace it +/// with an opaque placeholder. +/// +/// If the audio URL went stale, refresh the feed at `feed_url` and look the +/// episode up by `episode_id`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PodcastEpisodeRef { + /// Owning podcast. + pub podcast_id: PodcastId, + /// Episode. + pub episode_id: EpisodeId, + /// Canonical feed URL used to refresh the episode. + pub feed_url: FeedUrl, + /// Podcast title for display. + pub podcast_title: String, + /// Episode title for display. + pub episode_title: String, + /// Absolute http(s) audio URL at the publisher. + pub audio_url: String, + /// Enclosure MIME type. + #[serde(default)] + pub mime: Option, + /// Announced duration in milliseconds. + #[serde(default)] + pub duration_ms: Option, + /// Publication time in unix milliseconds. + #[serde(default)] + pub published_at_ms: Option, + /// `itunes:season`. + #[serde(default)] + pub season: Option, + /// `itunes:episode`. + #[serde(default)] + pub episode_number: Option, + /// Episode or podcast artwork URL. + #[serde(default)] + pub image_url: Option, +} + +impl PodcastEpisodeRef { + /// Validates contract bounds. Receivers must drop invalid references. + pub fn validate(&self) -> PodcastResult<()> { + check_text("podcast title", &self.podcast_title, MAX_TITLE_CHARS, true)?; + check_text("episode title", &self.episode_title, MAX_TITLE_CHARS, true)?; + check_url("episode audio_url", &self.audio_url)?; + check_mime(self.mime.as_deref())?; + check_duration(self.duration_ms)?; + check_opt_url("episode image_url", self.image_url.as_deref()) + } + + /// Identity hints known from this reference. + /// + /// Add them to the known-episode set before resolving ids from a feed so + /// a GUID-less episode learned through sync keeps its id. + pub fn identity_hints(&self) -> EpisodeIdentityHints { + EpisodeIdentityHints { + guid: None, + enclosure_url: Some(self.audio_url.clone()), + link: None, + published_at_ms: self.published_at_ms, + title: Some(self.episode_title.clone()), + } + } +} + +fn check_duration(duration_ms: Option) -> PodcastResult<()> { + if duration_ms.is_some_and(|ms| ms < 0) { + return Err(PodcastError::InvalidField { + field: "duration_ms", + reason: "must not be negative", + }); + } + Ok(()) +} + +fn check_mime(mime: Option<&str>) -> PodcastResult<()> { + let Some(mime) = mime else { return Ok(()) }; + if mime.is_empty() || mime.len() > MAX_MIME_BYTES || !mime.bytes().all(|b| b.is_ascii_graphic()) + { + return Err(PodcastError::InvalidField { + field: "mime", + reason: "must be a short printable ASCII token", + }); + } + Ok(()) +} + +fn bounded_title(title: &str, fallback: &str) -> String { + let title = title.trim(); + if title.is_empty() { + fallback.to_string() + } else { + bounded_text(title, MAX_TITLE_CHARS) + } +} + +fn bounded_opt(value: Option<&str>, max_chars: usize) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| bounded_text(value, max_chars)) +} + +fn valid_url(value: Option<&str>) -> Option { + value + .filter(|value| super::is_http_url(value)) + .map(str::to_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn podcast_meta() -> PodcastMeta { + PodcastMeta { + feed_url: FeedUrl::parse("https://example.test/feed").unwrap(), + title: "Show".to_string(), + author: Some("Host".to_string()), + description: None, + link: Some("https://example.test/".to_string()), + image_url: Some("https://example.test/cover.jpg".to_string()), + } + } + + fn episode() -> EpisodeMeta { + let podcast_id = PodcastId::from_feed_url(&podcast_meta().feed_url); + EpisodeMeta { + episode_id: EpisodeId::from_guid(&podcast_id, "g1").unwrap(), + podcast_id, + guid: Some("g1".to_string()), + title: "Pilot".to_string(), + description: Some("Notes".to_string()), + season: Some(2), + episode_number: Some(7), + episode_type: EpisodeType::Bonus, + published_at_ms: Some(1_700_000_000_000), + duration_ms: Some(3_600_000), + audio_url: "https://cdn.example.test/1.mp3?sig=abc".to_string(), + mime: Some("audio/mpeg".to_string()), + byte_len: Some(1234), + image_url: None, + link: Some("https://example.test/1".to_string()), + } + } + + #[test] + fn episode_type_is_tolerant() { + assert_eq!(EpisodeType::parse(" Trailer "), EpisodeType::Trailer); + assert_eq!(EpisodeType::parse("BONUS"), EpisodeType::Bonus); + assert_eq!(EpisodeType::parse("full"), EpisodeType::Full); + assert_eq!(EpisodeType::parse("teaser"), EpisodeType::Full); + assert_eq!( + serde_json::to_string(&EpisodeType::Trailer).unwrap(), + "\"trailer\"" + ); + assert_eq!( + serde_json::from_str::("\"something-new\"").unwrap(), + EpisodeType::Full + ); + } + + #[test] + fn metadata_round_trips_and_validates() { + let episode = episode(); + episode.validate().unwrap(); + let json = serde_json::to_string(&episode).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), episode); + podcast_meta().validate().unwrap(); + + let mut bad = episode.clone(); + bad.audio_url = "file:///tmp/x.mp3".to_string(); + assert!(bad.validate().is_err()); + let mut bad = episode.clone(); + bad.description = Some("x".repeat(MAX_DESCRIPTION_CHARS + 1)); + assert!(bad.validate().is_err()); + let mut bad = episode; + bad.mime = Some("audio/mpeg; codecs=\"mp3\"".to_string()); + assert!(bad.validate().is_err()); + } + + #[test] + fn minimal_legacy_json_uses_defaults() { + let episode = episode(); + let json = format!( + r#"{{"episode_id":"{}","podcast_id":"{}","title":"T","audio_url":"https://a.test/1.mp3"}}"#, + episode.episode_id, episode.podcast_id + ); + let parsed: EpisodeMeta = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.episode_type, EpisodeType::Full); + assert_eq!(parsed.season, None); + } + + #[test] + fn episode_ref_is_self_contained_and_bounded() { + let podcast = podcast_meta(); + let mut episode = episode(); + episode.title = "x".repeat(MAX_TITLE_CHARS + 10); + let reference = episode.to_ref(&podcast); + reference.validate().unwrap(); + assert_eq!(reference.episode_title.chars().count(), MAX_TITLE_CHARS); + assert_eq!(reference.podcast_title, "Show"); + assert_eq!( + reference.image_url.as_deref(), + Some("https://example.test/cover.jpg") + ); + assert_eq!(reference.feed_url, podcast.feed_url); + let json = serde_json::to_string(&reference).unwrap(); + assert!(!json.contains("description")); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + reference + ); + + let mut bad = reference.clone(); + bad.episode_title = " ".to_string(); + assert!(bad.validate().is_err()); + let mut bad = reference.clone(); + bad.audio_url = "javascript:alert(1)".to_string(); + assert!(bad.validate().is_err()); + let mut bad = reference; + bad.duration_ms = Some(-1); + assert!(bad.validate().is_err()); + } + + #[test] + fn refs_reject_invalid_ids_on_deserialization() { + let json = r#"{"podcast_id":"pod1:bad","episode_id":"ep1:bad","feed_url":"https://a.test/","podcast_title":"P","episode_title":"E","audio_url":"https://a.test/1.mp3"}"#; + assert!(serde_json::from_str::(json).is_err()); + } + + #[test] + fn bounded_podcast_meta_fits_the_contract() { + let mut meta = podcast_meta(); + meta.title = " ".to_string(); + meta.description = Some("d".repeat(MAX_DESCRIPTION_CHARS * 2)); + meta.image_url = Some("ftp://x".to_string()); + let bounded = meta.bounded(); + bounded.validate().unwrap(); + assert_eq!(bounded.title, "Untitled podcast"); + assert_eq!(bounded.image_url, None); + } +} diff --git a/crates/music-dht/src/podcast/mod.rs b/crates/music-dht/src/podcast/mod.rs new file mode 100644 index 0000000..a9d6e43 --- /dev/null +++ b/crates/music-dht/src/podcast/mod.rs @@ -0,0 +1,298 @@ +//! Shared podcast contract for Furumi clients. +//! +//! This module is pure: it has no SQLite, HTTP, clock, audio or UI +//! dependency. Every Furumi client (TUI, web, desktop) uses it so that +//! podcast identity, normalization, merge rules and wire DTOs are defined +//! exactly once. The normative description lives in the workspace +//! `PODCASTS.md`; this documentation summarizes the API. +//! +//! # Identity +//! +//! * [`FeedUrl`] is a canonical feed URL. [`FeedUrl::parse`] lowercases the +//! scheme and host, drops a default port and the fragment, and keeps the +//! path and **every** query parameter byte-for-byte (only non-ASCII bytes +//! are percent-encoded). Non-`http(s)` URLs, userinfo and whitespace are +//! rejected. [`FeedUrl::from_user_input`] additionally accepts the +//! `feed:`/`podcast:`/`pcast:`/`itpc:` forms and a missing scheme. +//! * [`PodcastId`] is `pod1:` followed by the lowercase base32 BLAKE3 hash of +//! `"furumi-podcast:podcast:v1\0"` and the canonical feed URL. It is +//! derived once, from the URL of the first subscription; later URLs +//! (redirects, `itunes:new-feed-url`) become feed aliases resolved by +//! [`resolve_podcast_id`]. +//! * [`EpisodeId`] is `ep1:` followed by the lowercase base32 BLAKE3 hash of +//! `"furumi-podcast:episode:v1\0"`, the podcast id and a key. The key is +//! the RSS GUID when present. Without a GUID, [`resolve_episode_ids`] first +//! reuses the id of a known episode that matches by enclosure URL, `` +//! or (date, normalized title) and only then derives a new id from the +//! first available fallback key. +//! +//! # Personal state +//! +//! * [`SubscriptionRecord`]: last-writer-wins by `(hlc_ms, op_id)`; +//! unsubscribing is state, not a tombstone. Feed aliases merge as a bounded +//! grow-only set. +//! * [`ShareRecord`]: last-writer-wins; `shared = false` is a tombstone. See +//! [`effective_share`] for the rule that a (re)subscription is private. +//! * [`ProgressRecord`]: register ordered by +//! `(generation, hlc_ms, device_id, op_id)`. Explicit user actions bump the +//! generation to `max_seen + 1`; periodic checkpoints keep the generation of +//! their session, so a late checkpoint of an old session always loses. +//! * [`ListenSession`]: grow-only set keyed by `session_id`; one completion per +//! session, so re-delivery never double counts. +//! +//! # Queue items +//! +//! [`PodcastEpisodeRef`] is the bounded, self-contained reference carried by +//! playback queues and progress operations. A receiving device can show and +//! play the episode without being subscribed to the podcast. +//! +//! ``` +//! use music_dht::podcast::{EpisodeId, FeedUrl, PodcastId}; +//! +//! let url = FeedUrl::parse("HTTPS://Example.COM:443/feed.xml?token=Ab#top").unwrap(); +//! assert_eq!(url.as_str(), "https://example.com/feed.xml?token=Ab"); +//! let podcast = PodcastId::from_feed_url(&url); +//! assert!(podcast.as_str().starts_with("pod1:")); +//! let episode = EpisodeId::from_guid(&podcast, "episode-42").unwrap(); +//! assert!(episode.as_str().starts_with("ep1:")); +//! ``` + +mod feed_url; +mod id; +mod identity; +mod meta; +mod state; + +pub use feed_url::{FeedUrl, merge_feed_aliases, resolve_podcast_id}; +pub use id::{ + EPISODE_ID_DOMAIN, EPISODE_ID_PREFIX, EpisodeId, EpisodeKey, PODCAST_ID_DOMAIN, + PODCAST_ID_PREFIX, PodcastId, +}; +pub use identity::{ + EpisodeIdentityHints, KnownEpisode, normalize_episode_title, resolve_episode_id, + resolve_episode_ids, +}; +pub use meta::{EpisodeMeta, EpisodeType, PodcastEpisodeRef, PodcastMeta}; +pub use state::{ + COMPLETION_TAIL_MAX_MS, ListenSession, MIN_RECORDED_EPISODE_LISTEN_MS, ProgressRecord, + ProgressStatus, ShareRecord, SubscriptionRecord, completed_listen_count, effective_share, + merge_listen_session, merge_progress, merge_share, merge_subscription, reaches_completion, +}; + +/// Maximum characters of a title (podcast, episode, author). +pub const MAX_TITLE_CHARS: usize = 1_000; +/// Maximum characters of a podcast or episode description. +pub const MAX_DESCRIPTION_CHARS: usize = 4_000; +/// Maximum bytes of any URL carried by the contract. +pub const MAX_URL_BYTES: usize = 2_048; +/// Maximum bytes of an RSS GUID. Longer GUIDs are treated as absent. +pub const MAX_GUID_BYTES: usize = 2_048; +/// Maximum bytes of a MIME type. +pub const MAX_MIME_BYTES: usize = 128; +/// Maximum bytes of a device, session or operation id. +pub const MAX_TOKEN_BYTES: usize = 256; +/// Maximum feed aliases kept per podcast, besides the URL the id was derived from. +pub const MAX_FEED_ALIASES: usize = 8; + +/// Maximum subscription records in one sync snapshot. +pub const MAX_SNAPSHOT_SUBSCRIPTIONS: usize = 1_000; +/// Maximum share records in one sync snapshot. +pub const MAX_SNAPSHOT_SHARES: usize = 1_000; +/// Maximum progress registers in one sync snapshot. +pub const MAX_SNAPSHOT_PROGRESS: usize = 5_000; +/// Maximum listening sessions in one sync snapshot. +pub const MAX_SNAPSHOT_LISTEN_SESSIONS: usize = 2_000; +/// Serialized JSON budget shared by all podcast snapshot sections. +/// +/// It leaves room for music state inside the 8 MiB sync line limit. +pub const MAX_PODCAST_SNAPSHOT_BYTES: usize = 4 * 1024 * 1024; + +/// Errors produced by podcast contract validation and parsing. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum PodcastError { + /// A feed URL cannot be canonicalized. + #[error("invalid feed URL: {0}")] + InvalidFeedUrl(String), + /// A string is not a valid podcast or episode id. + #[error("invalid {kind} id")] + InvalidId { + /// `podcast` or `episode`. + kind: &'static str, + }, + /// A field violates the contract (missing, too long, malformed). + #[error("invalid {field}: {reason}")] + InvalidField { + /// Field name. + field: &'static str, + /// Human-readable reason. + reason: &'static str, + }, + /// Two records that must describe the same entity do not. + #[error("cannot merge records of different {0}")] + Mismatch(&'static str), + /// The document is not an RSS or Atom feed. + #[error("not an RSS or Atom feed{0}")] + NotAFeed(String), + /// The feed is malformed and nothing usable could be parsed. + #[error("malformed feed: {0}")] + MalformedFeed(String), +} + +/// Result alias for this module. +pub type PodcastResult = std::result::Result; + +/// Truncates `text` to at most `max_chars` characters, ending with `…` when cut. +/// +/// Clients use it to fit publisher metadata into the contract bounds. +pub fn bounded_text(text: &str, max_chars: usize) -> String { + if text.chars().count() <= max_chars { + return text.to_string(); + } + if max_chars == 0 { + return String::new(); + } + let mut out: String = text.chars().take(max_chars - 1).collect(); + out.push('…'); + out +} + +pub(crate) fn check_text( + field: &'static str, + value: &str, + max_chars: usize, + required: bool, +) -> PodcastResult<()> { + if required && value.trim().is_empty() { + return Err(PodcastError::InvalidField { + field, + reason: "must not be empty", + }); + } + if value.len() > max_chars * 4 || value.chars().count() > max_chars { + return Err(PodcastError::InvalidField { + field, + reason: "is too long", + }); + } + Ok(()) +} + +pub(crate) fn check_opt_text( + field: &'static str, + value: Option<&str>, + max_chars: usize, +) -> PodcastResult<()> { + value.map_or(Ok(()), |value| check_text(field, value, max_chars, false)) +} + +pub(crate) fn check_token(field: &'static str, value: &str) -> PodcastResult<()> { + if value.is_empty() { + return Err(PodcastError::InvalidField { + field, + reason: "must not be empty", + }); + } + if value.len() > MAX_TOKEN_BYTES { + return Err(PodcastError::InvalidField { + field, + reason: "is too long", + }); + } + if value.chars().any(|c| c.is_control()) { + return Err(PodcastError::InvalidField { + field, + reason: "contains control characters", + }); + } + Ok(()) +} + +/// Returns whether `value` is an absolute `http`/`https` URL with a host, +/// without whitespace or control characters and within [`MAX_URL_BYTES`]. +/// +/// Media URLs keep the publisher's spelling; only feed URLs are canonicalized. +pub fn is_http_url(value: &str) -> bool { + if value.is_empty() + || value.len() > MAX_URL_BYTES + || value.chars().any(|c| c.is_whitespace() || c.is_control()) + { + return false; + } + let lower_prefix = |prefix: &str| { + value.len() >= prefix.len() && value[..prefix.len()].eq_ignore_ascii_case(prefix) + }; + let rest = if lower_prefix("http://") { + &value[7..] + } else if lower_prefix("https://") { + &value[8..] + } else { + return false; + }; + let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = authority.rsplit('@').next().unwrap_or_default(); + !host.is_empty() && !host.starts_with(':') +} + +pub(crate) fn check_url(field: &'static str, value: &str) -> PodcastResult<()> { + if is_http_url(value) { + Ok(()) + } else { + Err(PodcastError::InvalidField { + field, + reason: "must be an absolute http(s) URL", + }) + } +} + +pub(crate) fn check_opt_url(field: &'static str, value: Option<&str>) -> PodcastResult<()> { + value.map_or(Ok(()), |value| check_url(field, value)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bounded_text_cuts_on_char_boundaries() { + assert_eq!(bounded_text("hello", 5), "hello"); + assert_eq!(bounded_text("hello!", 5), "hell…"); + assert_eq!(bounded_text("привет", 3), "пр…"); + assert_eq!(bounded_text("x", 0), ""); + } + + #[test] + fn text_limits_accept_the_limit_and_reject_one_more() { + let at = "я".repeat(MAX_TITLE_CHARS); + let over = "я".repeat(MAX_TITLE_CHARS + 1); + assert!(check_text("title", &at, MAX_TITLE_CHARS, true).is_ok()); + assert!(check_text("title", &over, MAX_TITLE_CHARS, true).is_err()); + assert!(check_text("title", " ", MAX_TITLE_CHARS, true).is_err()); + assert!(check_text("title", "", MAX_TITLE_CHARS, false).is_ok()); + } + + #[test] + fn token_limits() { + assert!(check_token("op_id", &"a".repeat(MAX_TOKEN_BYTES)).is_ok()); + assert!(check_token("op_id", &"a".repeat(MAX_TOKEN_BYTES + 1)).is_err()); + assert!(check_token("op_id", "").is_err()); + assert!(check_token("op_id", "a\nb").is_err()); + } + + #[test] + fn http_urls() { + assert!(is_http_url("https://cdn.example.test/a.mp3?x=1")); + assert!(is_http_url("HTTP://cdn.example.test")); + assert!(!is_http_url("ftp://cdn.example.test/a.mp3")); + assert!(!is_http_url("file:///etc/passwd")); + assert!(!is_http_url("https://")); + assert!(!is_http_url("https:///path")); + assert!(!is_http_url("https://a.test/my file.mp3")); + assert!(!is_http_url("")); + let long = format!("https://a.test/{}", "x".repeat(MAX_URL_BYTES)); + assert!(!is_http_url(&long)); + let at = format!("https://a.test/{}", "x".repeat(MAX_URL_BYTES - 15)); + assert_eq!(at.len(), MAX_URL_BYTES); + assert!(is_http_url(&at)); + } +} diff --git a/crates/music-dht/src/podcast/state.rs b/crates/music-dht/src/podcast/state.rs new file mode 100644 index 0000000..c258eb7 --- /dev/null +++ b/crates/music-dht/src/podcast/state.rs @@ -0,0 +1,445 @@ +//! Personal podcast state and its convergent merge rules. +//! +//! Every merge function here is commutative, associative and idempotent over +//! records of the same entity, so trusted devices that receive the same set of +//! operations converge to the same state regardless of delivery order and +//! duplication. All ties are broken by a total order over every field. + +use std::cmp::Ordering; +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +use super::{ + EpisodeId, FeedUrl, MAX_FEED_ALIASES, PodcastError, PodcastId, PodcastMeta, PodcastResult, + check_token, merge_feed_aliases, +}; + +/// Largest tail of an episode that still counts as finished, in milliseconds. +/// +/// See [`reaches_completion`]. +pub const COMPLETION_TAIL_MAX_MS: i64 = 30_000; + +/// Minimum actual listening time for a non-completed session to enter history. +pub const MIN_RECORDED_EPISODE_LISTEN_MS: i64 = 30_000; + +/// Shared automatic completion criterion for every client. +/// +/// A position completes an episode when it is within the smaller of 30 s and +/// 5 % of the duration from the end. Natural end of stream always completes, +/// whatever the announced duration. Without a known positive duration only the +/// end of stream completes. +pub fn reaches_completion(position_ms: i64, duration_ms: Option) -> bool { + let Some(duration) = duration_ms.filter(|duration| *duration > 0) else { + return false; + }; + let tail = COMPLETION_TAIL_MAX_MS.min(duration / 20); + position_ms >= duration - tail +} + +/// Listening status of one episode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProgressStatus { + /// Never started, or explicitly marked unplayed. + NotStarted, + /// Started and not finished. + InProgress, + /// Finished, automatically or by an explicit "mark played". + Completed, +} + +/// Durable resume register of one episode. +/// +/// The winner among records of the same episode is the greatest +/// `(generation, hlc_ms, device_id, op_id)`; remaining fields only break ties +/// between malformed duplicates. Progress is **never** the maximum position: +/// seeking backwards and "start over" are real changes. +/// +/// * Explicit user actions (start of a new listening session, seek, "start +/// over", "mark played", "mark unplayed", handoff to another device) use +/// [`ProgressRecord::explicit_generation`]: one more than the greatest +/// generation the device has seen for the episode. This is a Lamport clock, +/// so wall-clock skew cannot undo an explicit action that was observed. +/// * Periodic checkpoints keep the generation of the latest explicit action of +/// their session. A late checkpoint of an older session therefore loses +/// against any newer explicit action, whatever its `hlc_ms`. +/// * Concurrent explicit actions on offline devices receive the same +/// generation and are ordered by `hlc_ms`, then `device_id`, then `op_id`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProgressRecord { + /// Episode the register belongs to. + pub episode_id: EpisodeId, + /// Owning podcast. + pub podcast_id: PodcastId, + /// Resume position in milliseconds. + pub position_ms: i64, + /// Listening status. + pub status: ProgressStatus, + /// Lamport generation of the latest explicit action (see type docs). + pub generation: u64, + /// Listening session (or explicit action) that wrote the record. + pub session_id: String, + /// Logical write time, normally the sync operation `hlc_ms`. + pub hlc_ms: i64, + /// Writing device, normally the sync operation origin. + pub device_id: String, + /// Writing operation id. + pub op_id: String, +} + +impl ProgressRecord { + /// Generation for an explicit user action, given the greatest generation + /// already seen for the episode (`None` when none was seen). + pub fn explicit_generation(max_seen: Option) -> u64 { + max_seen.map_or(1, |generation| generation.saturating_add(1)) + } + + /// Primary ordering key. + pub fn order_key(&self) -> (u64, i64, &str, &str) { + ( + self.generation, + self.hlc_ms, + self.device_id.as_str(), + self.op_id.as_str(), + ) + } + + fn total_cmp(&self, other: &Self) -> Ordering { + self.order_key() + .cmp(&other.order_key()) + .then_with(|| self.position_ms.cmp(&other.position_ms)) + .then_with(|| self.status.cmp(&other.status)) + .then_with(|| self.session_id.cmp(&other.session_id)) + .then_with(|| self.podcast_id.cmp(&other.podcast_id)) + } + + /// Returns whether this record wins over `other` (same episode). + pub fn supersedes(&self, other: &Self) -> bool { + self.total_cmp(other) == Ordering::Greater + } + + /// Position from which ordinary Play resumes. + /// + /// Completed and not-started episodes start from the beginning; use the + /// explicit "start over" action to restart an in-progress episode. + pub fn resume_position_ms(&self) -> i64 { + match self.status { + ProgressStatus::InProgress => self.position_ms, + ProgressStatus::NotStarted | ProgressStatus::Completed => 0, + } + } + + /// Validates contract bounds. + pub fn validate(&self) -> PodcastResult<()> { + if self.position_ms < 0 { + return Err(PodcastError::InvalidField { + field: "position_ms", + reason: "must not be negative", + }); + } + check_token("session_id", &self.session_id)?; + check_token("device_id", &self.device_id)?; + check_token("op_id", &self.op_id) + } +} + +/// Merges two progress registers of the same episode. +pub fn merge_progress( + current: &ProgressRecord, + incoming: &ProgressRecord, +) -> PodcastResult { + if current.episode_id != incoming.episode_id { + return Err(PodcastError::Mismatch("episodes")); + } + Ok(if incoming.supersedes(current) { + incoming.clone() + } else { + current.clone() + }) +} + +/// Personal subscription state of one podcast. +/// +/// `subscribed` and `podcast` form one last-writer-wins register ordered by +/// `(hlc_ms, op_id)`. Unsubscribing writes `subscribed = false`: it is state, +/// not a tombstone, and never deletes progress or history, so subscribing +/// again continues where the user stopped. +/// +/// Only explicit user actions (subscribe, unsubscribe) write this register. +/// Background RSS refreshes keep their metadata locally and must not emit a +/// subscription operation; otherwise a stale device could resurrect an +/// unsubscribed podcast. +/// +/// `feed_aliases` is a bounded grow-only set merged independently of the +/// register (see [`merge_feed_aliases`]); it always contains every +/// `podcast.feed_url` seen. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SubscriptionRecord { + /// Podcast. + pub podcast_id: PodcastId, + /// Whether the user is subscribed. + pub subscribed: bool, + /// Channel metadata at the time of the write. + pub podcast: PodcastMeta, + /// Known feed URLs of the podcast. + #[serde(default)] + pub feed_aliases: Vec, + /// Logical write time. + pub hlc_ms: i64, + /// Writing operation id. + pub op_id: String, +} + +impl SubscriptionRecord { + fn total_cmp(&self, other: &Self) -> Ordering { + (self.hlc_ms, self.op_id.as_str()) + .cmp(&(other.hlc_ms, other.op_id.as_str())) + .then_with(|| self.subscribed.cmp(&other.subscribed)) + .then_with(|| self.podcast.cmp(&other.podcast)) + } + + /// Returns whether this record's register wins over `other`'s. + pub fn supersedes(&self, other: &Self) -> bool { + self.total_cmp(other) == Ordering::Greater + } + + /// Ensures `feed_aliases` is sorted, bounded and contains `podcast.feed_url`. + pub fn normalize(&mut self) { + self.feed_aliases = merge_feed_aliases( + &self.podcast_id, + &self.feed_aliases, + std::slice::from_ref(&self.podcast.feed_url), + ); + } + + /// Validates contract bounds. + pub fn validate(&self) -> PodcastResult<()> { + self.podcast.validate()?; + if self.feed_aliases.len() > MAX_FEED_ALIASES + 1 { + return Err(PodcastError::InvalidField { + field: "feed_aliases", + reason: "has too many entries", + }); + } + check_token("op_id", &self.op_id) + } +} + +/// Merges two subscription records of the same podcast. +/// +/// The result is normalized (see [`SubscriptionRecord::normalize`]). +pub fn merge_subscription( + current: &SubscriptionRecord, + incoming: &SubscriptionRecord, +) -> PodcastResult { + if current.podcast_id != incoming.podcast_id { + return Err(PodcastError::Mismatch("podcasts")); + } + let mut merged = if incoming.supersedes(current) { + incoming.clone() + } else { + current.clone() + }; + let mut left = current.feed_aliases.clone(); + left.push(current.podcast.feed_url.clone()); + let mut right = incoming.feed_aliases.clone(); + right.push(incoming.podcast.feed_url.clone()); + merged.feed_aliases = merge_feed_aliases(&merged.podcast_id, &left, &right); + Ok(merged) +} + +/// Decision to publish a podcast card to the federation. +/// +/// Last-writer-wins by `(hlc_ms, op_id)`; `shared = false` is the tombstone +/// that revokes the publication. Unsubscribing must also write +/// `shared = false`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ShareRecord { + /// Podcast. + pub podcast_id: PodcastId, + /// Whether the user chose to share the podcast. + pub shared: bool, + /// Logical write time. + pub hlc_ms: i64, + /// Writing operation id. + pub op_id: String, +} + +impl ShareRecord { + fn total_cmp(&self, other: &Self) -> Ordering { + (self.hlc_ms, self.op_id.as_str(), self.shared).cmp(&( + other.hlc_ms, + other.op_id.as_str(), + other.shared, + )) + } + + /// Returns whether this record wins over `other`. + pub fn supersedes(&self, other: &Self) -> bool { + self.total_cmp(other) == Ordering::Greater + } + + /// Validates contract bounds. + pub fn validate(&self) -> PodcastResult<()> { + check_token("op_id", &self.op_id) + } +} + +/// Merges two share records of the same podcast. +pub fn merge_share(current: &ShareRecord, incoming: &ShareRecord) -> PodcastResult { + if current.podcast_id != incoming.podcast_id { + return Err(PodcastError::Mismatch("podcasts")); + } + Ok(if incoming.supersedes(current) { + incoming.clone() + } else { + current.clone() + }) +} + +/// Whether a podcast must currently be published. +/// +/// True only when the user is subscribed, chose to share, and the share +/// decision is newer than the latest subscription write. A new subscription +/// (including re-subscribing, or subscribing from someone else's public card) +/// is therefore always private until the user shares it again, and a stale +/// device cannot republish a podcast through an old share decision. +pub fn effective_share( + subscription: Option<&SubscriptionRecord>, + share: Option<&ShareRecord>, +) -> bool { + match (subscription, share) { + (Some(subscription), Some(share)) => { + subscription.subscribed + && share.shared + && subscription.podcast_id == share.podcast_id + && (share.hlc_ms, share.op_id.as_str()) + > (subscription.hlc_ms, subscription.op_id.as_str()) + } + _ => false, + } +} + +/// One listening session of an episode on one device. +/// +/// Sessions form a grow-only set keyed by `session_id`, generated when +/// playback of the episode starts on a device. A device may re-send its +/// session with more listening time (for example after a crash); versions of +/// one session merge by [`merge_listen_session`]. +/// +/// * A session enters history when it completed or lasted at least +/// [`MIN_RECORDED_EPISODE_LISTEN_MS`] ([`ListenSession::should_record`]). +/// * `completed` is set when the session reached [`reaches_completion`] or the +/// natural end of the stream. Each session counts at most once, so the play +/// count of an episode is the number of distinct completed sessions +/// ([`completed_listen_count`]); duplicate delivery never double counts. +/// * "Mark played" changes [`ProgressRecord::status`] only. It does not create +/// a session and does not invent listening time. +/// * Sessions listened concurrently on offline devices are all kept. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ListenSession { + /// Unique session id, normally a UUID. + pub session_id: String, + /// Episode. + pub episode_id: EpisodeId, + /// Owning podcast. + pub podcast_id: PodcastId, + /// Device that played the session. + pub device_id: String, + /// Session start in unix milliseconds. + pub started_at_ms: i64, + /// Latest update of the session in unix milliseconds. + pub ended_at_ms: i64, + /// Position when the session started. + pub start_position_ms: i64, + /// Position when the session ended (or was last reported). + pub end_position_ms: i64, + /// Actual accumulated listening time, excluding pauses and seeks. + pub listened_ms: i64, + /// Whether the session completed the episode. + pub completed: bool, +} + +impl ListenSession { + fn total_cmp(&self, other: &Self) -> Ordering { + ( + self.completed, + self.listened_ms, + self.ended_at_ms, + self.end_position_ms, + self.started_at_ms, + self.start_position_ms, + ) + .cmp(&( + other.completed, + other.listened_ms, + other.ended_at_ms, + other.end_position_ms, + other.started_at_ms, + other.start_position_ms, + )) + } + + /// Whether the session belongs in listening history. + pub fn should_record(&self) -> bool { + self.completed || self.listened_ms >= MIN_RECORDED_EPISODE_LISTEN_MS + } + + /// Validates contract bounds. + pub fn validate(&self) -> PodcastResult<()> { + check_token("session_id", &self.session_id)?; + check_token("device_id", &self.device_id)?; + if self.started_at_ms < 0 || self.ended_at_ms < self.started_at_ms { + return Err(PodcastError::InvalidField { + field: "ended_at_ms", + reason: "must not precede started_at_ms", + }); + } + if self.start_position_ms < 0 || self.end_position_ms < 0 || self.listened_ms < 0 { + return Err(PodcastError::InvalidField { + field: "listened_ms", + reason: "positions and listening time must not be negative", + }); + } + Ok(()) + } +} + +/// Merges two versions of the same listening session. +/// +/// The version with the greatest `(completed, listened_ms, ended_at_ms, ...)` +/// wins, so a session can only grow. +pub fn merge_listen_session( + current: &ListenSession, + incoming: &ListenSession, +) -> PodcastResult { + if current.session_id != incoming.session_id { + return Err(PodcastError::Mismatch("sessions")); + } + if current.episode_id != incoming.episode_id + || current.podcast_id != incoming.podcast_id + || current.device_id != incoming.device_id + { + return Err(PodcastError::Mismatch("session identities")); + } + Ok(if incoming.total_cmp(current) == Ordering::Greater { + incoming.clone() + } else { + current.clone() + }) +} + +/// Number of distinct sessions that completed an episode. +pub fn completed_listen_count<'a>(sessions: impl IntoIterator) -> usize { + sessions + .into_iter() + .filter(|session| session.completed) + .map(|session| session.session_id.as_str()) + .collect::>() + .len() +} + +#[cfg(test)] +#[path = "state_tests.rs"] +mod tests; diff --git a/crates/music-dht/src/podcast/state_tests.rs b/crates/music-dht/src/podcast/state_tests.rs new file mode 100644 index 0000000..2bbf60a --- /dev/null +++ b/crates/music-dht/src/podcast/state_tests.rs @@ -0,0 +1,568 @@ +use std::collections::BTreeMap; + +use super::*; +use crate::podcast::{EpisodeIdentityHints, EpisodeMeta, EpisodeType, resolve_episode_ids}; + +fn permutations(items: &[T]) -> Vec> { + if items.len() <= 1 { + return vec![items.to_vec()]; + } + let mut out = Vec::new(); + for index in 0..items.len() { + let mut rest = items.to_vec(); + let head = rest.remove(index); + for mut tail in permutations(&rest) { + tail.insert(0, head.clone()); + out.push(tail); + } + } + out +} + +fn feed_url(path: &str) -> FeedUrl { + FeedUrl::parse(&format!("https://example.test/{path}")).unwrap() +} + +fn podcast_id() -> PodcastId { + PodcastId::from_feed_url(&feed_url("feed")) +} + +fn episode_id(guid: &str) -> EpisodeId { + EpisodeId::from_guid(&podcast_id(), guid).unwrap() +} + +fn progress( + generation: u64, + hlc_ms: i64, + device: &str, + op: &str, + position_ms: i64, + status: ProgressStatus, + session: &str, +) -> ProgressRecord { + ProgressRecord { + episode_id: episode_id("e1"), + podcast_id: podcast_id(), + position_ms, + status, + generation, + session_id: session.to_string(), + hlc_ms, + device_id: device.to_string(), + op_id: op.to_string(), + } +} + +fn fold(items: &[T], merge: impl Fn(&T, &T) -> PodcastResult) -> T { + let mut iter = items.iter(); + let first = iter.next().expect("non-empty").clone(); + iter.fold(first, |acc, item| merge(&acc, item).unwrap()) +} + +fn assert_converges( + items: &[T], + merge: impl Fn(&T, &T) -> PodcastResult + Copy, +) -> T { + let reference = fold(items, merge); + for order in permutations(items) { + assert_eq!(fold(&order, merge), reference, "order: {order:?}"); + } + // Merge laws on every pair and triple. + for a in items { + assert_eq!( + merge(&merge(a, a).unwrap(), a).unwrap(), + merge(a, a).unwrap() + ); + for b in items { + assert_eq!(merge(a, b).unwrap(), merge(b, a).unwrap()); + for c in items { + assert_eq!( + merge(&merge(a, b).unwrap(), c).unwrap(), + merge(a, &merge(b, c).unwrap()).unwrap() + ); + } + } + } + reference +} + +#[test] +fn progress_converges_for_every_permutation() { + use ProgressStatus::*; + let ops = [ + progress(1, 100, "dev-a", "dev-a:1", 10_000, InProgress, "s1"), + progress(1, 160, "dev-a", "dev-a:2", 70_000, InProgress, "s1"), + progress(2, 150, "dev-b", "dev-b:1", 0, InProgress, "s2"), + progress(2, 150, "dev-c", "dev-c:1", 5_000, InProgress, "s3"), + progress(2, 140, "dev-b", "dev-b:2", 0, Completed, "s2"), + // Duplicate delivery of the same operation. + progress(1, 160, "dev-a", "dev-a:2", 70_000, InProgress, "s1"), + ]; + let winner = assert_converges(&ops, merge_progress); + assert_eq!(winner.op_id, "dev-c:1"); +} + +#[test] +fn late_old_generation_checkpoint_loses() { + use ProgressStatus::*; + // Device A keeps checkpointing session s1 (generation 3) with a clock far + // ahead; device B explicitly seeks back (generation 4) with a slow clock. + let seek_back = progress(4, 1_000, "dev-b", "dev-b:9", 60_000, InProgress, "s2"); + let late = progress( + 3, 9_000_000, "dev-a", "dev-a:50", 3_000_000, InProgress, "s1", + ); + for order in [[&seek_back, &late], [&late, &seek_back]] { + let merged = merge_progress(order[0], order[1]).unwrap(); + assert_eq!(merged, seek_back); + } + assert!(seek_back.supersedes(&late)); + assert!(!late.supersedes(&seek_back)); +} + +#[test] +fn restart_beats_later_checkpoint_of_the_old_session() { + use ProgressStatus::*; + let checkpoint = progress(2, 100, "dev-a", "dev-a:1", 2_400_000, InProgress, "s1"); + let generation = ProgressRecord::explicit_generation(Some(checkpoint.generation)); + assert_eq!(generation, 3); + let restart = progress(generation, 110, "dev-a", "dev-a:2", 0, InProgress, "s2"); + // A checkpoint of s1 that was queued before the restart and delivered + // later, even stamped after the restart. + let late = progress(2, 120, "dev-a", "dev-a:3", 2_460_000, InProgress, "s1"); + let merged = assert_converges(&[checkpoint, restart.clone(), late], merge_progress); + assert_eq!(merged, restart); + assert_eq!(merged.resume_position_ms(), 0); +} + +#[test] +fn progress_is_not_the_maximum_position() { + use ProgressStatus::*; + let ahead = progress(1, 100, "dev-a", "dev-a:1", 3_000_000, InProgress, "s1"); + let back = progress(2, 101, "dev-a", "dev-a:2", 60_000, InProgress, "s1"); + assert_eq!(merge_progress(&ahead, &back).unwrap().position_ms, 60_000); +} + +#[test] +fn concurrent_offline_sessions_converge_and_keep_both_sessions() { + use ProgressStatus::*; + // Both devices last saw generation 5 and start listening offline. + let generation = ProgressRecord::explicit_generation(Some(5)); + let a_start = progress( + generation, 200, "dev-a", "dev-a:1", 300_000, InProgress, "sa", + ); + let a_cp = progress( + generation, 300, "dev-a", "dev-a:2", 900_000, InProgress, "sa", + ); + let b_start = progress( + generation, 210, "dev-b", "dev-b:1", 300_000, InProgress, "sb", + ); + let b_cp = progress( + generation, 280, "dev-b", "dev-b:2", 1_500_000, InProgress, "sb", + ); + let merged = assert_converges(&[a_start, a_cp.clone(), b_start, b_cp], merge_progress); + // Deterministic: the latest write of the shared generation wins. + assert_eq!(merged, a_cp); + + let session = |id: &str, device: &str, listened: i64, completed: bool| ListenSession { + session_id: id.to_string(), + episode_id: episode_id("e1"), + podcast_id: podcast_id(), + device_id: device.to_string(), + started_at_ms: 200, + ended_at_ms: 300, + start_position_ms: 300_000, + end_position_ms: 900_000, + listened_ms: listened, + completed, + }; + let sessions = [ + session("sa", "dev-a", 600_000, false), + session("sb", "dev-b", 1_200_000, true), + session("sb", "dev-b", 1_200_000, true), + ]; + let mut set: BTreeMap = BTreeMap::new(); + for order in permutations(&sessions) { + set.clear(); + for item in order { + let merged = match set.get(&item.session_id) { + Some(current) => merge_listen_session(current, &item).unwrap(), + None => item.clone(), + }; + set.insert(item.session_id.clone(), merged); + } + assert_eq!(set.len(), 2); + assert_eq!(completed_listen_count(set.values()), 1); + } +} + +#[test] +fn listen_session_versions_only_grow() { + let base = ListenSession { + session_id: "s".to_string(), + episode_id: episode_id("e1"), + podcast_id: podcast_id(), + device_id: "dev-a".to_string(), + started_at_ms: 1_000, + ended_at_ms: 2_000, + start_position_ms: 0, + end_position_ms: 60_000, + listened_ms: 60_000, + completed: false, + }; + let longer = ListenSession { + ended_at_ms: 3_000, + end_position_ms: 120_000, + listened_ms: 120_000, + ..base.clone() + }; + let finished = ListenSession { + ended_at_ms: 3_500, + end_position_ms: 3_600_000, + listened_ms: 125_000, + completed: true, + ..base.clone() + }; + let merged = assert_converges( + &[base.clone(), longer, finished.clone(), base.clone()], + merge_listen_session, + ); + assert_eq!(merged, finished); + assert!(merged.should_record()); + assert_eq!(completed_listen_count([&merged, &merged, &finished]), 1); + + let mut other_episode = base.clone(); + other_episode.episode_id = episode_id("e2"); + assert!(merge_listen_session(&base, &other_episode).is_err()); + let short = ListenSession { + listened_ms: MIN_RECORDED_EPISODE_LISTEN_MS - 1, + ..base.clone() + }; + assert!(!short.should_record()); + assert!( + ListenSession { + listened_ms: MIN_RECORDED_EPISODE_LISTEN_MS, + ..base + } + .should_record() + ); +} + +#[test] +fn completion_criterion() { + // 60 min episode: tail is 30 s. + assert!(!reaches_completion(3_569_999, Some(3_600_000))); + assert!(reaches_completion(3_570_000, Some(3_600_000))); + // 5 min episode: tail is 5 % = 15 s. + assert!(!reaches_completion(284_999, Some(300_000))); + assert!(reaches_completion(285_000, Some(300_000))); + assert!(!reaches_completion(1_000_000, None)); + assert!(!reaches_completion(1_000_000, Some(0))); +} + +fn podcast_meta(title: &str, url: &str) -> PodcastMeta { + PodcastMeta { + feed_url: feed_url(url), + title: title.to_string(), + author: None, + description: None, + link: None, + image_url: None, + } +} + +fn subscription( + subscribed: bool, + hlc_ms: i64, + op: &str, + title: &str, + url: &str, +) -> SubscriptionRecord { + let mut record = SubscriptionRecord { + podcast_id: podcast_id(), + subscribed, + podcast: podcast_meta(title, url), + feed_aliases: Vec::new(), + hlc_ms, + op_id: op.to_string(), + }; + record.normalize(); + record +} + +#[test] +fn subscriptions_converge_for_every_permutation() { + let ops = [ + // Same podcast added on two devices. + subscription(true, 100, "dev-a:1", "Show", "feed"), + subscription(true, 101, "dev-b:1", "Show (B)", "feed"), + // Unsubscribed on A while B was offline. + subscription(false, 200, "dev-a:2", "Show", "feed"), + // B comes back and re-delivers its old state after the feed moved. + subscription(true, 150, "dev-b:2", "Show", "moved"), + subscription(true, 101, "dev-b:1", "Show (B)", "feed"), + ]; + let merged = assert_converges(&ops, merge_subscription); + assert!( + !merged.subscribed, + "unsubscribe survives the offline device" + ); + assert_eq!(merged.op_id, "dev-a:2"); + assert!(merged.feed_aliases.contains(&feed_url("moved"))); + assert!(merged.feed_aliases.contains(&feed_url("feed"))); + merged.validate().unwrap(); +} + +#[test] +fn resubscribing_keeps_progress_and_history() { + use ProgressStatus::*; + let mut progress_register = BTreeMap::new(); + let saved = progress(4, 500, "dev-a", "dev-a:5", 1_234_000, InProgress, "s1"); + progress_register.insert(saved.episode_id.clone(), saved.clone()); + let subscribed = subscription(true, 100, "dev-a:1", "Show", "feed"); + let unsubscribed = subscription(false, 600, "dev-a:6", "Show", "feed"); + let again = subscription(true, 700, "dev-b:1", "Show", "feed"); + let merged = fold(&[subscribed, unsubscribed, again], merge_subscription); + assert!(merged.subscribed); + assert_eq!(progress_register.get(&saved.episode_id), Some(&saved)); +} + +fn share(shared: bool, hlc_ms: i64, op: &str) -> ShareRecord { + ShareRecord { + podcast_id: podcast_id(), + shared, + hlc_ms, + op_id: op.to_string(), + } +} + +#[test] +fn shares_converge_and_false_is_a_tombstone() { + let ops = [ + share(true, 100, "dev-a:1"), + share(false, 200, "dev-a:2"), + share(true, 150, "dev-b:1"), + share(false, 200, "dev-a:2"), + ]; + let merged = assert_converges(&ops, merge_share); + assert!(!merged.shared, "late stale share cannot republish"); +} + +#[test] +fn effective_share_rules() { + let subscribed = subscription(true, 100, "dev-a:1", "Show", "feed"); + let shared = share(true, 150, "dev-a:2"); + assert!(effective_share(Some(&subscribed), Some(&shared))); + assert!(!effective_share(Some(&subscribed), None)); + assert!(!effective_share(None, Some(&shared))); + // New subscriptions are private: a share decision older than the + // latest subscription write does not apply. + let resubscribed = subscription(true, 300, "dev-a:4", "Show", "feed"); + assert!(!effective_share(Some(&resubscribed), Some(&shared))); + // Unsubscribing revokes publication even if a stale share wins. + let unsubscribed = subscription(false, 200, "dev-a:3", "Show", "feed"); + let stale = share(true, 250, "dev-b:1"); + assert!(!effective_share(Some(&unsubscribed), Some(&stale))); +} + +/// Minimal client store used to show that whole operation sets converge and +/// that publisher metadata never reaches personal progress. +#[derive(Debug, Default, Clone, PartialEq)] +struct Store { + subscriptions: BTreeMap, + shares: BTreeMap, + progress: BTreeMap, + sessions: BTreeMap, + catalog: BTreeMap, +} + +#[derive(Debug, Clone)] +enum Op { + Subscription(SubscriptionRecord), + Share(ShareRecord), + Progress(ProgressRecord), + Session(ListenSession), + /// Local RSS refresh: title and audio URL changed at the publisher. + Rss { + title: &'static str, + audio_url: &'static str, + }, +} + +fn upsert( + map: &mut BTreeMap, + key: K, + value: &V, + merge: impl Fn(&V, &V) -> PodcastResult, +) { + let merged = match map.get(&key) { + Some(current) => merge(current, value).unwrap(), + None => value.clone(), + }; + map.insert(key, merged); +} + +impl Store { + fn apply(&mut self, op: &Op) { + match op { + Op::Subscription(record) => upsert( + &mut self.subscriptions, + record.podcast_id.clone(), + record, + merge_subscription, + ), + Op::Share(record) => upsert( + &mut self.shares, + record.podcast_id.clone(), + record, + merge_share, + ), + Op::Progress(record) => upsert( + &mut self.progress, + record.episode_id.clone(), + record, + merge_progress, + ), + Op::Session(session) => upsert( + &mut self.sessions, + session.session_id.clone(), + session, + merge_listen_session, + ), + Op::Rss { title, audio_url } => { + let hints = EpisodeIdentityHints { + guid: Some("e1".to_string()), + enclosure_url: Some(audio_url.to_string()), + link: None, + published_at_ms: Some(1_000), + title: Some(title.to_string()), + }; + let id = resolve_episode_ids(&podcast_id(), None, &[hints], &[])[0] + .clone() + .unwrap(); + // Metadata only: last refresh on this device wins locally. + let meta = EpisodeMeta { + episode_id: id.clone(), + podcast_id: podcast_id(), + guid: Some("e1".to_string()), + title: title.to_string(), + description: None, + season: None, + episode_number: None, + episode_type: EpisodeType::Full, + published_at_ms: Some(1_000), + duration_ms: None, + audio_url: audio_url.to_string(), + mime: None, + byte_len: None, + image_url: None, + link: None, + }; + self.catalog.insert(id, meta); + } + } + } + + fn personal(&self) -> Store { + Store { + catalog: BTreeMap::new(), + ..self.clone() + } + } +} + +#[test] +fn whole_operation_sets_converge_and_rss_never_touches_progress() { + use ProgressStatus::*; + let session = ListenSession { + session_id: "s2".to_string(), + episode_id: episode_id("e1"), + podcast_id: podcast_id(), + device_id: "dev-b".to_string(), + started_at_ms: 10, + ended_at_ms: 20, + start_position_ms: 0, + end_position_ms: 400_000, + listened_ms: 400_000, + completed: false, + }; + let ops = [ + Op::Subscription(subscription(true, 100, "dev-a:1", "Show", "feed")), + Op::Share(share(true, 120, "dev-a:2")), + Op::Progress(progress( + 1, 130, "dev-a", "dev-a:3", 900_000, InProgress, "s1", + )), + Op::Progress(progress( + 2, 125, "dev-b", "dev-b:1", 400_000, InProgress, "s2", + )), + Op::Session(session), + Op::Rss { + title: "Old title", + audio_url: "https://cdn.test/old.mp3", + }, + Op::Rss { + title: "New title", + audio_url: "https://cdn.test/new.mp3", + }, + ]; + let mut reference: Option = None; + for order in permutations(&ops) { + let mut store = Store::default(); + for op in &order { + store.apply(op); + } + // One episode id across both metadata versions. + assert_eq!(store.catalog.len(), 1); + let progress = store.progress.values().next().unwrap(); + assert_eq!(progress.op_id, "dev-b:1"); + assert_eq!(progress.position_ms, 400_000); + let personal = store.personal(); + match &reference { + Some(reference) => assert_eq!(&personal, reference), + None => reference = Some(personal), + } + } +} + +#[test] +fn validation_bounds() { + use ProgressStatus::*; + let ok = progress(1, 1, "dev-a", "dev-a:1", 0, NotStarted, "s"); + ok.validate().unwrap(); + let mut bad = ok.clone(); + bad.position_ms = -1; + assert!(bad.validate().is_err()); + let mut bad = ok.clone(); + bad.session_id = String::new(); + assert!(bad.validate().is_err()); + let mut bad = ok; + bad.device_id = "d".repeat(crate::podcast::MAX_TOKEN_BYTES + 1); + assert!(bad.validate().is_err()); + + let mut record = subscription(true, 1, "dev-a:1", "Show", "feed"); + record.feed_aliases = (0..=crate::podcast::MAX_FEED_ALIASES + 1) + .map(|i| feed_url(&format!("alias-{i}"))) + .collect(); + assert!(record.validate().is_err()); + record.normalize(); + record.validate().unwrap(); +} + +#[test] +fn progress_json_shape() { + let record = progress( + 3, + 42, + "dev-a", + "dev-a:7", + 1_000, + ProgressStatus::InProgress, + "s", + ); + let json = serde_json::to_value(&record).unwrap(); + assert_eq!(json["status"], "in_progress"); + assert_eq!(json["generation"], 3); + assert_eq!( + serde_json::from_value::(json).unwrap(), + record + ); +} -- 2.54.0 From 31d132bb3f90e1425379ba045682713ea25259ee Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 15:40:19 +0200 Subject: [PATCH 2/7] Add podcast-feed RSS and Atom parser Port the tolerant RSS 2.0 / RSS 1.0 / Atom parser of the original furumi-tui podcast work into music_dht::podcast::feed behind the optional podcast-feed feature (quick-xml, encoding_rs, url). Episode identity depends on what the parser extracts, so it must be shared by all clients. The port keeps its robustness (BOM and declared encodings, entities, mismatched tags, depth and size limits, lenient dates, http(s)-only media) and adds itunes:season, itunes:episode, itunes:episodeType, guid isPermaLink and itunes:new-feed-url. Missing GUIDs stay absent instead of falling back to the enclosure URL, and parsed items produce identity hints and contract metadata. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 29 + Cargo.toml | 3 + crates/music-dht/Cargo.toml | 7 + crates/music-dht/src/podcast/feed.rs | 1775 ++++++++++++++++++++ crates/music-dht/src/podcast/feed_tests.rs | 665 ++++++++ crates/music-dht/src/podcast/mod.rs | 9 + 6 files changed, 2488 insertions(+) create mode 100644 crates/music-dht/src/podcast/feed.rs create mode 100644 crates/music-dht/src/podcast/feed_tests.rs diff --git a/Cargo.lock b/Cargo.lock index fe24c1d..9c9ccc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,6 +389,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -790,6 +796,20 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "enum-assoc" version = "1.3.0" @@ -2071,6 +2091,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "music-dht" version = "0.5.0" @@ -2079,10 +2105,12 @@ dependencies = [ "async-trait", "blake3", "data-encoding", + "encoding_rs", "federation-net", "futures", "iroh", "postcard", + "quick-xml", "rand 0.9.5", "rusqlite", "serde", @@ -2092,6 +2120,7 @@ dependencies = [ "tokio", "tracing", "unicode-normalization", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 568c7b7..86aa1c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,3 +38,6 @@ lofty = "0.22" data-encoding = "2" rand = "0.9" tempfile = "3" +quick-xml = "0.41" +encoding_rs = "0.8.35" +url = "2.5" diff --git a/crates/music-dht/Cargo.toml b/crates/music-dht/Cargo.toml index a376ffd..cec4116 100644 --- a/crates/music-dht/Cargo.toml +++ b/crates/music-dht/Cargo.toml @@ -26,6 +26,13 @@ thiserror = { workspace = true } tracing = { workspace = true } rand = { workspace = true } data-encoding = { workspace = true } +quick-xml = { workspace = true, optional = true } +encoding_rs = { workspace = true, optional = true } +url = { workspace = true, optional = true } + +[features] +# RSS 2.0 / RSS 1.0 / Atom podcast feed parser (`music_dht::podcast::feed`). +podcast-feed = ["dep:quick-xml", "dep:encoding_rs", "dep:url"] [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/music-dht/src/podcast/feed.rs b/crates/music-dht/src/podcast/feed.rs new file mode 100644 index 0000000..8da47b8 --- /dev/null +++ b/crates/music-dht/src/podcast/feed.rs @@ -0,0 +1,1775 @@ +//! Shared tolerant RSS 2.0 / RSS 1.0 / Atom parser for podcast feeds. +//! +//! Available with the `podcast-feed` cargo feature. Every Furumi client uses +//! this parser because episode identity depends on what it extracts. +//! +//! Real-world feeds are frequently sloppy: undeclared namespace prefixes, +//! HTML entities that XML does not define, mismatched closing tags, legacy +//! single-byte encodings and creative date formats. The parser is a single +//! iterative pass over `quick-xml` events (no recursion), keeps only the +//! fields Furumi needs, bounds every captured value and never panics on +//! malformed input. It performs no I/O: fetching, redirects, conditional +//! requests and scheduling belong to the client runtime. +//! +//! ``` +//! use music_dht::podcast::feed::parse_feed_with_base; +//! use music_dht::podcast::{FeedUrl, PodcastId}; +//! +//! let xml = br#"Show +//! Pilotep-1 +//! +//! "#; +//! let url = FeedUrl::parse("https://example.test/feed.xml").unwrap(); +//! let feed = parse_feed_with_base(xml, Some(url.as_str())).unwrap(); +//! let podcast = PodcastId::from_feed_url(&url); +//! let ids = feed.resolve_episode_ids(&podcast, &[]); +//! let episode = feed.episodes[0].to_meta(&podcast, ids[0].clone().unwrap()); +//! assert_eq!(episode.audio_url, "https://example.test/audio/1.mp3"); +//! ``` + +use std::borrow::Cow; +use std::collections::HashSet; + +use quick_xml::events::{BytesStart, Event}; +use quick_xml::reader::Reader; +use url::Url; + +use super::{ + EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode, + MAX_DESCRIPTION_CHARS, MAX_GUID_BYTES, MAX_TITLE_CHARS, PodcastError, PodcastId, PodcastMeta, + PodcastResult, is_http_url, resolve_episode_ids, +}; + +/// Upper bound on episodes taken from a single feed document. +pub const MAX_EPISODES: usize = 20_000; +/// Raw text captured for one element is capped to keep memory bounded on +/// hostile or broken documents. +const MAX_CAPTURE_BYTES: usize = 512 * 1024; +/// Elements nested deeper than this are skipped (their events are ignored). +const MAX_DEPTH: usize = 256; +/// Durations longer than this (about 115 days) are treated as garbage. +const MAX_DURATION_SECONDS: f64 = 10_000_000.0; +const UNTITLED_EPISODE: &str = "Untitled episode"; +const UNTITLED_PODCAST: &str = "Untitled podcast"; + +/// Channel-level result of [`parse_feed`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedFeed { + /// Podcast title (never empty). + pub title: String, + /// Author (`itunes:author`, Atom author, `dc:creator`, `managingEditor`). + pub author: Option, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + pub description: Option, + /// Publisher web page. + pub link: Option, + /// Absolute http(s) artwork URL. + pub image_url: Option, + /// `itunes:new-feed-url`, verbatim. Clients canonicalize it with + /// [`FeedUrl::parse`] and record it as a feed alias. + pub new_feed_url: Option, + /// Playable items in document order, duplicates removed. + pub episodes: Vec, +} + +/// One playable feed item. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedEpisode { + /// RSS `` / Atom ``, trimmed; `None` when absent, blank or + /// longer than [`MAX_GUID_BYTES`]. + pub guid: Option, + /// RSS `guid isPermaLink` (default `true` per RSS 2.0); `None` without + /// an RSS GUID. Informational: identity uses the GUID text either way. + pub guid_is_permalink: Option, + /// Title (never empty). + pub title: String, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + pub description: Option, + /// Publication time in unix milliseconds. + pub published_at_ms: Option, + /// Announced duration in milliseconds. + pub duration_ms: Option, + /// `itunes:season`. + pub season: Option, + /// `itunes:episode`. + pub episode_number: Option, + /// `itunes:episodeType`. + pub episode_type: EpisodeType, + /// Absolute http(s) audio URL (never a local file). + pub audio_url: String, + /// Enclosure MIME type, lowercased, without parameters. + pub mime_type: Option, + /// Enclosure length in bytes. + pub byte_len: Option, + /// Item web page. + pub link: Option, + /// Absolute http(s) artwork URL. + pub image_url: Option, +} + +impl ParsedEpisode { + /// Identity hints for [`resolve_episode_ids`]. + pub fn identity_hints(&self) -> EpisodeIdentityHints { + EpisodeIdentityHints { + guid: self.guid.clone(), + enclosure_url: Some(self.audio_url.clone()), + link: self.link.clone(), + published_at_ms: self.published_at_ms, + title: Some(self.title.clone()), + } + } + + /// Contract metadata for this item under a resolved id. + pub fn to_meta(&self, podcast_id: &PodcastId, episode_id: EpisodeId) -> EpisodeMeta { + EpisodeMeta { + episode_id, + podcast_id: podcast_id.clone(), + guid: self.guid.clone(), + title: self.title.clone(), + description: self.description.clone(), + season: self.season, + episode_number: self.episode_number, + episode_type: self.episode_type, + published_at_ms: self.published_at_ms, + duration_ms: self.duration_ms, + audio_url: self.audio_url.clone(), + mime: self.mime_type.clone(), + byte_len: self.byte_len, + image_url: self.image_url.clone(), + link: self + .link + .clone() + .filter(|link| link.len() <= super::MAX_URL_BYTES), + } + } +} + +impl ParsedFeed { + /// Identity hints of every episode, in order. + pub fn identity_hints(&self) -> Vec { + self.episodes + .iter() + .map(ParsedEpisode::identity_hints) + .collect() + } + + /// Resolves stable ids for every episode (see [`resolve_episode_ids`]). + pub fn resolve_episode_ids( + &self, + podcast_id: &PodcastId, + known: &[KnownEpisode], + ) -> Vec> { + resolve_episode_ids( + podcast_id, + self.link.as_deref(), + &self.identity_hints(), + known, + ) + } + + /// Contract channel metadata for a feed fetched from `feed_url`. + pub fn to_podcast_meta(&self, feed_url: FeedUrl) -> PodcastMeta { + PodcastMeta { + feed_url, + title: self.title.clone(), + author: self.author.clone(), + description: self.description.clone(), + link: self.link.clone(), + image_url: self.image_url.clone(), + } + .bounded() + } +} + +/// Parses an RSS or Atom document. Fails for documents that are not feeds +/// (for example an HTML page) with "not an RSS or Atom feed". +/// +/// Without a base URL, relative media URLs cannot be resolved and such +/// episodes are dropped; prefer [`parse_feed_with_base`] when the feed URL is +/// known. +pub fn parse_feed(bytes: &[u8]) -> PodcastResult { + parse_feed_with_base(bytes, None) +} + +/// [`parse_feed`] resolving relative enclosure and image URLs against +/// `base_url` (the final URL the feed was fetched from). Episodes whose audio +/// URL does not resolve to an absolute `http`/`https` URL are dropped, so a +/// feed can never point playback at a local file. +pub fn parse_feed_with_base(bytes: &[u8], base_url: Option<&str>) -> PodcastResult { + let text = decode_document(bytes); + let parser = Parser { + base: base_url.and_then(|base| Url::parse(base).ok()), + ..Parser::default() + }; + parser.run(&text) +} + +fn not_a_feed(detail: impl std::fmt::Display) -> PodcastError { + PodcastError::NotAFeed(format!(" ({detail})")) +} + +// --------------------------------------------------------------------------- +// Character encoding + +/// Decodes the raw document to UTF-8. A BOM wins; otherwise an ASCII-compatible +/// encoding named in the XML declaration (windows-1251, koi8-r, ...) is +/// honoured; everything else is decoded as lossy UTF-8. +fn decode_document(bytes: &[u8]) -> Cow<'_, str> { + if let Some((encoding, bom_len)) = encoding_rs::Encoding::for_bom(bytes) { + let (text, _) = encoding.decode_without_bom_handling(&bytes[bom_len..]); + return text; + } + if let Some(encoding) = declared_encoding(bytes).and_then(encoding_rs::Encoding::for_label) + && encoding != encoding_rs::UTF_8 + && encoding.is_ascii_compatible() + { + let (text, _) = encoding.decode_without_bom_handling(bytes); + return text; + } + String::from_utf8_lossy(bytes) +} + +/// The `encoding="..."` label of a leading XML declaration, if any. +fn declared_encoding(bytes: &[u8]) -> Option<&[u8]> { + let head = &bytes[..bytes.len().min(512)]; + let start = head.iter().position(|b| !b.is_ascii_whitespace())?; + let head = &head[start..]; + if !head.starts_with(b"")?; + let decl = &head[..end]; + let at = find(decl, b"encoding")? + b"encoding".len(); + let rest = &decl[at..]; + let rest = &rest[rest.iter().position(|b| !b.is_ascii_whitespace())?..]; + let rest = rest.strip_prefix(b"=")?; + let rest = &rest[rest.iter().position(|b| !b.is_ascii_whitespace())?..]; + let quote = *rest.first()?; + if quote != b'"' && quote != b'\'' { + return None; + } + let rest = &rest[1..]; + let len = rest.iter().position(|&b| b == quote)?; + Some(&rest[..len]) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +// --------------------------------------------------------------------------- +// Namespaces + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ns { + /// No namespace, or the RSS 0.9x/1.0 namespace. + Plain, + Atom, + Itunes, + Content, + Media, + Dc, + Other, +} + +fn classify_uri(uri: &str) -> Ns { + let uri = uri.trim().to_ascii_lowercase(); + if uri.is_empty() { + Ns::Plain + } else if uri.contains("w3.org/2005/atom") { + Ns::Atom + } else if uri.contains("itunes.com/dtds/podcast") { + Ns::Itunes + } else if uri.contains("purl.org/rss/1.0/modules/content") { + Ns::Content + } else if uri.contains("search.yahoo.com/mrss") { + Ns::Media + } else if uri.contains("purl.org/dc/elements") { + Ns::Dc + } else if uri.contains("purl.org/rss/1.0") + || uri.contains("my.netscape.com/rdf") + || uri.contains("backend.userland.com/rss") + { + Ns::Plain + } else { + Ns::Other + } +} + +/// Classification of a prefix nobody declared (common in sloppy feeds). +fn classify_prefix(prefix: &str) -> Ns { + match prefix.to_ascii_lowercase().as_str() { + "" => Ns::Plain, + "itunes" => Ns::Itunes, + "content" => Ns::Content, + "media" => Ns::Media, + "dc" => Ns::Dc, + "atom" | "atom10" => Ns::Atom, + _ => Ns::Other, + } +} + +// --------------------------------------------------------------------------- +// Parser state + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Rss, + Atom, +} + +#[derive(Debug)] +struct Elem { + ns: Ns, + /// ASCII-lowercased local name. + local: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Field { + FeedTitle, + FeedDescription, + FeedItunesSummary, + FeedLink, + FeedItunesAuthor, + FeedAtomAuthor, + FeedDcCreator, + FeedManagingEditor, + FeedImageUrl, + FeedLogo, + FeedIcon, + FeedNewFeedUrl, + ItemTitle, + ItemItunesTitle, + ItemDescription, + ItemContent, + ItemSummary, + ItemGuid, + ItemPubDate, + ItemDcDate, + ItemUpdated, + ItemDuration, + ItemLink, + ItemSeason, + ItemEpisode, + ItemEpisodeType, +} + +#[derive(Debug)] +struct Capture { + /// Stack depth of the captured element. + depth: usize, + field: Field, + text: String, +} + +#[derive(Debug, Default)] +struct FeedAcc { + title: Option, + description: Option, + itunes_summary: Option, + link: Option, + itunes_author: Option, + atom_author: Option, + dc_creator: Option, + managing_editor: Option, + itunes_image: Option, + image_url: Option, + logo: Option, + icon: Option, + new_feed_url: Option, +} + +#[derive(Debug, Clone)] +struct Media { + url: String, + mime: Option, + len: Option, +} + +#[derive(Debug, Default)] +struct ItemAcc { + title: Option, + itunes_title: Option, + description: Option, + content: Option, + summary: Option, + guid: Option, + guid_is_permalink: Option, + pub_date: Option, + dc_date: Option, + updated: Option, + duration: Option, + season: Option, + episode: Option, + episode_type: Option, + link: Option, + enclosure: Option, + media_content: Option, + itunes_image: Option, + thumbnail: Option, +} + +#[derive(Debug, Default)] +struct Parser { + kind: Option, + stack: Vec, + /// Namespace declarations: (depth of declaring element, prefix, uri). + ns_decls: Vec<(usize, String, String)>, + /// Start tags ignored because they exceeded `MAX_DEPTH`. + skipped_depth: usize, + capture: Option, + feed: FeedAcc, + item: Option, + item_depth: usize, + episodes: Vec, + seen_guids: HashSet, + full: bool, + /// URL the document was fetched from, for relative media URLs. + base: Option, +} + +/// Sets `slot` unless it already holds a value (first occurrence wins). +fn set_once(slot: &mut Option, value: String) { + if slot.is_none() && !value.trim().is_empty() { + *slot = Some(value); + } +} + +impl Parser { + fn run(mut self, text: &str) -> PodcastResult { + let mut reader = Reader::from_str(text); + { + let config = reader.config_mut(); + config.check_end_names = false; + config.allow_unmatched_ends = true; + config.allow_dangling_amp = true; + config.expand_empty_elements = false; + config.trim_text(false); + } + let mut malformed = None; + loop { + match reader.read_event() { + Ok(Event::Eof) => break, + Ok(event) => { + self.handle(event)?; + if self.full { + break; + } + } + Err(err) => { + if self.kind.is_none() { + return Err(not_a_feed(format_args!("invalid XML: {err}"))); + } + malformed = Some(err.to_string()); + break; + } + } + } + if self.kind.is_none() { + return Err(PodcastError::NotAFeed(String::new())); + } + if let Some(err) = malformed { + if self.episodes.is_empty() { + return Err(PodcastError::MalformedFeed(err)); + } + tracing::warn!("podcast feed is malformed, keeping what parsed: {err}"); + } + Ok(self.finish()) + } + + fn handle(&mut self, event: Event<'_>) -> PodcastResult<()> { + match event { + Event::Start(start) => self.on_start(&start, false)?, + Event::Empty(start) => self.on_start(&start, true)?, + Event::End(end) => { + let name = end.name(); + let local = local_name(name.as_ref()); + self.on_end(&local); + } + Event::Text(text) => { + if let Some(capture) = self.capture.as_mut() { + match text.xml10_content() { + Ok(value) => push_capped(&mut capture.text, &value), + Err(_) => push_capped(&mut capture.text, &String::from_utf8_lossy(&text)), + } + } + } + Event::CData(data) => { + if let Some(capture) = self.capture.as_mut() { + match data.decode() { + Ok(value) => push_capped(&mut capture.text, &value), + Err(_) => push_capped(&mut capture.text, &String::from_utf8_lossy(&data)), + } + } + } + Event::GeneralRef(reference) => { + if let Some(capture) = self.capture.as_mut() { + let raw = String::from_utf8_lossy(&reference); + match resolve_entity(&raw) { + Some(ch) => push_capped(&mut capture.text, ch.encode_utf8(&mut [0; 4])), + None => { + // Keep unknown entities verbatim instead of failing. + push_capped(&mut capture.text, "&"); + push_capped(&mut capture.text, &raw); + push_capped(&mut capture.text, ";"); + } + } + } + } + Event::Comment(_) | Event::Decl(_) | Event::PI(_) | Event::DocType(_) | Event::Eof => {} + } + Ok(()) + } + + fn resolve_ns(&self, prefix: &str) -> Ns { + for (_, declared, uri) in self.ns_decls.iter().rev() { + if declared == prefix { + return classify_uri(uri); + } + } + classify_prefix(prefix) + } + + fn on_start(&mut self, start: &BytesStart<'_>, empty: bool) -> PodcastResult<()> { + if self.stack.len() >= MAX_DEPTH { + if !empty { + self.skipped_depth += 1; + } + return Ok(()); + } + let depth = self.stack.len() + 1; + let attrs = Attrs::collect(start); + for (key, value) in &attrs.raw { + if key == "xmlns" { + self.ns_decls.push((depth, String::new(), value.clone())); + } else if let Some(prefix) = key.strip_prefix("xmlns:") { + self.ns_decls + .push((depth, prefix.to_string(), value.clone())); + } + } + let qname = start.name(); + let (prefix, local) = split_qname(qname.as_ref()); + let mut ns = self.resolve_ns(&prefix); + if self.kind == Some(Kind::Rss) && prefix.is_empty() && matches!(ns, Ns::Atom | Ns::Other) { + // RSS has no namespace of its own; an unprefixed element is a core + // RSS element whatever default namespace the publisher declared. + ns = Ns::Plain; + } + self.stack.push(Elem { ns, local }); + + if self.kind.is_none() { + let local = self.stack[0].local.as_str(); + self.kind = match local { + "rss" | "rdf" => Some(Kind::Rss), + "feed" => Some(Kind::Atom), + _ => return Err(not_a_feed(format_args!("root element <{local}>"))), + }; + } else if let Some(capture) = self.capture.as_mut() { + // Markup nested in a captured field (e.g. Atom xhtml content): + // keep words apart. + push_capped(&mut capture.text, " "); + } else { + self.dispatch_start(&attrs); + } + + if empty { + self.close_top(); + } + Ok(()) + } + + fn dispatch_start(&mut self, attrs: &Attrs) { + let Some(kind) = self.kind else { return }; + let depth = self.stack.len(); + let local_at = |back: usize| { + depth + .checked_sub(back) + .and_then(|index| self.stack.get(index)) + .map(|elem| elem.local.clone()) + .unwrap_or_default() + }; + let ns = self.stack[depth - 1].ns; + let local = local_at(1); + let parent = local_at(2); + let grandparent = local_at(3); + let core = match kind { + Kind::Rss => ns == Ns::Plain, + Kind::Atom => matches!(ns, Ns::Plain | Ns::Atom), + }; + + if self.item.is_some() { + let direct = depth == self.item_depth + 1; + if let Some(field) = self.item_field(kind, ns, core, &local, direct, attrs) { + self.begin_capture(field); + } + return; + } + + let is_item = match kind { + Kind::Rss => core && local == "item", + Kind::Atom => core && local == "entry", + }; + if is_item { + self.item = Some(ItemAcc::default()); + self.item_depth = depth; + return; + } + + let feed_level = match kind { + Kind::Rss => parent == "channel", + Kind::Atom => depth == 2, + }; + let field = if feed_level { + self.feed_field(kind, ns, core, &local, attrs) + } else if kind == Kind::Rss + && core + && local == "url" + && parent == "image" + && matches!(grandparent.as_str(), "channel" | "rdf") + { + Some(Field::FeedImageUrl) + } else if kind == Kind::Atom && core && local == "name" && parent == "author" && depth == 3 + { + Some(Field::FeedAtomAuthor) + } else { + None + }; + if let Some(field) = field { + self.begin_capture(field); + } + } + + fn feed_field( + &mut self, + kind: Kind, + ns: Ns, + core: bool, + local: &str, + attrs: &Attrs, + ) -> Option { + if core { + return match (kind, local) { + (_, "title") => Some(Field::FeedTitle), + (Kind::Rss, "description") | (Kind::Atom, "subtitle") => { + Some(Field::FeedDescription) + } + (Kind::Rss, "link") => Some(Field::FeedLink), + (Kind::Rss, "managingeditor") => Some(Field::FeedManagingEditor), + (Kind::Atom, "link") => { + let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); + if rel.is_empty() || rel == "alternate" { + let href = attrs.get("href").unwrap_or_default(); + set_once(&mut self.feed.link, clean_short(&href)); + } + None + } + (Kind::Atom, "logo") => Some(Field::FeedLogo), + (Kind::Atom, "icon") => Some(Field::FeedIcon), + _ => None, + }; + } + match (ns, local) { + (Ns::Itunes, "author") => Some(Field::FeedItunesAuthor), + (Ns::Itunes, "summary") => Some(Field::FeedItunesSummary), + (Ns::Itunes, "image") => { + if let Some(href) = attrs.get("href") { + set_once(&mut self.feed.itunes_image, clean_short(&href)); + } + None + } + (Ns::Itunes, "new-feed-url") => Some(Field::FeedNewFeedUrl), + (Ns::Dc, "creator") => Some(Field::FeedDcCreator), + _ => None, + } + } + + fn item_field( + &mut self, + kind: Kind, + ns: Ns, + core: bool, + local: &str, + direct: bool, + attrs: &Attrs, + ) -> Option { + let item = self.item.as_mut()?; + // Media RSS may wrap content in , so accept any depth. + if ns == Ns::Media { + match local { + "content" => { + let url = attrs.get("url").unwrap_or_default(); + let mime = attrs.get("type"); + let medium = attrs.get("medium"); + if item.media_content.is_none() + && is_playable(&url, mime.as_deref(), medium.as_deref(), false) + { + item.media_content = Some(Media { + url: url.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("filesize")), + }); + } + } + "thumbnail" => { + if let Some(url) = attrs.get("url") { + set_once(&mut item.thumbnail, clean_short(&url)); + } + } + _ => {} + } + return None; + } + if !direct { + return None; + } + if core { + return match (kind, local) { + (_, "title") => Some(Field::ItemTitle), + (Kind::Rss, "description") => Some(Field::ItemDescription), + (Kind::Rss, "guid") => { + if item.guid_is_permalink.is_none() { + let permalink = attrs.get("ispermalink").unwrap_or_default(); + item.guid_is_permalink = + Some(!permalink.trim().eq_ignore_ascii_case("false")); + } + Some(Field::ItemGuid) + } + (Kind::Atom, "id") => Some(Field::ItemGuid), + (Kind::Rss, "pubdate") | (Kind::Atom, "published") => Some(Field::ItemPubDate), + (Kind::Atom, "updated") => Some(Field::ItemUpdated), + (Kind::Rss, "link") => Some(Field::ItemLink), + (Kind::Atom, "summary") => Some(Field::ItemDescription), + (Kind::Atom, "content") => Some(Field::ItemContent), + (Kind::Rss, "enclosure") => { + let url = attrs.get("url").unwrap_or_default(); + let mime = attrs.get("type"); + if item.enclosure.is_none() && is_playable(&url, mime.as_deref(), None, true) { + item.enclosure = Some(Media { + url: url.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("length")), + }); + } + None + } + (Kind::Atom, "link") => { + let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); + let href = attrs.get("href").unwrap_or_default(); + let mime = attrs.get("type"); + if rel == "enclosure" { + if item.enclosure.is_none() + && is_playable(&href, mime.as_deref(), None, true) + { + item.enclosure = Some(Media { + url: href.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("length")), + }); + } + } else if rel.is_empty() || rel == "alternate" { + set_once(&mut item.link, clean_short(&href)); + } + None + } + _ => None, + }; + } + match (ns, local) { + (Ns::Itunes, "title") => Some(Field::ItemItunesTitle), + (Ns::Itunes, "summary") => Some(Field::ItemSummary), + (Ns::Itunes, "duration") => Some(Field::ItemDuration), + (Ns::Itunes, "season") => Some(Field::ItemSeason), + (Ns::Itunes, "episode") => Some(Field::ItemEpisode), + (Ns::Itunes, "episodetype") => Some(Field::ItemEpisodeType), + (Ns::Itunes, "image") => { + if let Some(href) = attrs.get("href") { + set_once(&mut item.itunes_image, clean_short(&href)); + } + None + } + (Ns::Content, "encoded") => Some(Field::ItemContent), + (Ns::Dc, "date") => Some(Field::ItemDcDate), + _ => None, + } + } + + fn begin_capture(&mut self, field: Field) { + self.capture = Some(Capture { + depth: self.stack.len(), + field, + text: String::new(), + }); + } + + fn on_end(&mut self, local: &str) { + if self.skipped_depth > 0 { + self.skipped_depth -= 1; + return; + } + // Tolerate mismatched end tags: close up to the nearest open element + // with the same local name, ignore the tag if none is open. + let Some(index) = self.stack.iter().rposition(|elem| elem.local == local) else { + return; + }; + while self.stack.len() > index { + self.close_top(); + } + } + + fn close_top(&mut self) { + let depth = self.stack.len(); + if depth == 0 { + return; + } + if self + .capture + .as_ref() + .is_some_and(|capture| capture.depth == depth) + && let Some(capture) = self.capture.take() + { + self.store_capture(capture); + } + if depth == self.item_depth + && let Some(item) = self.item.take() + { + self.finish_item(item); + } + while self + .ns_decls + .last() + .is_some_and(|(decl_depth, _, _)| *decl_depth >= depth) + { + self.ns_decls.pop(); + } + self.stack.pop(); + } + + fn store_capture(&mut self, capture: Capture) { + let text = capture.text; + if let Some(item) = self.item.as_mut() { + let slot = match capture.field { + Field::ItemTitle => &mut item.title, + Field::ItemItunesTitle => &mut item.itunes_title, + Field::ItemDescription => &mut item.description, + Field::ItemContent => &mut item.content, + Field::ItemSummary => &mut item.summary, + Field::ItemGuid => &mut item.guid, + Field::ItemPubDate => &mut item.pub_date, + Field::ItemDcDate => &mut item.dc_date, + Field::ItemUpdated => &mut item.updated, + Field::ItemDuration => &mut item.duration, + Field::ItemLink => &mut item.link, + Field::ItemSeason => &mut item.season, + Field::ItemEpisode => &mut item.episode, + Field::ItemEpisodeType => &mut item.episode_type, + _ => return, + }; + set_once(slot, text); + return; + } + let feed = &mut self.feed; + let slot = match capture.field { + Field::FeedTitle => &mut feed.title, + Field::FeedDescription => &mut feed.description, + Field::FeedItunesSummary => &mut feed.itunes_summary, + Field::FeedLink => &mut feed.link, + Field::FeedItunesAuthor => &mut feed.itunes_author, + Field::FeedAtomAuthor => &mut feed.atom_author, + Field::FeedDcCreator => &mut feed.dc_creator, + Field::FeedManagingEditor => &mut feed.managing_editor, + Field::FeedImageUrl => &mut feed.image_url, + Field::FeedLogo => &mut feed.logo, + Field::FeedIcon => &mut feed.icon, + Field::FeedNewFeedUrl => &mut feed.new_feed_url, + _ => return, + }; + set_once(slot, text); + } + + fn finish_item(&mut self, item: ItemAcc) { + if self.full { + return; + } + let base = self.base.as_ref(); + let Some(media) = [item.enclosure, item.media_content] + .into_iter() + .flatten() + .find_map(|media| { + let url = resolve_http_url(&media.url, base)?; + Some(Media { url, ..media }) + }) + else { + return; + }; + let guid = item + .guid + .as_deref() + .map(str::trim) + .filter(|guid| !guid.is_empty() && guid.len() <= MAX_GUID_BYTES) + .map(str::to_string); + // Duplicate items (same GUID, or same audio URL without GUID) keep + // the first occurrence. + let dedup_key = match &guid { + Some(guid) => format!("guid:{guid}"), + None => format!("enclosure:{}", media.url), + }; + if !self.seen_guids.insert(dedup_key) { + return; + } + let guid_is_permalink = if guid.is_some() { + item.guid_is_permalink + } else { + None + }; + let title = [item.title.as_deref(), item.itunes_title.as_deref()] + .into_iter() + .flatten() + .map(clean_short) + .find(|title| !title.is_empty()) + .unwrap_or_else(|| UNTITLED_EPISODE.to_string()); + let description = [ + item.description.as_deref(), + item.content.as_deref(), + item.summary.as_deref(), + ] + .into_iter() + .flatten() + .map(clean_long) + .find(|text| !text.is_empty()); + let published_at_ms = [ + item.pub_date.as_deref(), + item.dc_date.as_deref(), + item.updated.as_deref(), + ] + .into_iter() + .flatten() + .find_map(parse_date) + .map(|seconds| seconds.saturating_mul(1_000)); + self.episodes.push(ParsedEpisode { + guid, + guid_is_permalink, + title, + description, + published_at_ms, + duration_ms: item + .duration + .as_deref() + .and_then(parse_duration) + .filter(|seconds| *seconds <= MAX_DURATION_SECONDS) + .map(|seconds| (seconds * 1_000.0).round() as i64), + season: item.season.as_deref().and_then(parse_number), + episode_number: item.episode.as_deref().and_then(parse_number), + episode_type: item + .episode_type + .as_deref() + .map(EpisodeType::parse) + .unwrap_or_default(), + audio_url: media.url, + mime_type: media.mime, + byte_len: media.len, + link: item + .link + .as_deref() + .map(clean_short) + .filter(|s| !s.is_empty()), + image_url: [item.itunes_image, item.thumbnail] + .into_iter() + .flatten() + .find_map(|url| resolve_http_url(&url, base)), + }); + if self.episodes.len() >= MAX_EPISODES { + self.full = true; + } + } + + fn finish(self) -> ParsedFeed { + let feed = self.feed; + let title = feed + .title + .as_deref() + .map(clean_short) + .filter(|title| !title.is_empty()) + .unwrap_or_else(|| UNTITLED_PODCAST.to_string()); + let author = [ + feed.itunes_author.as_deref(), + feed.atom_author.as_deref(), + feed.dc_creator.as_deref(), + feed.managing_editor.as_deref(), + ] + .into_iter() + .flatten() + .map(clean_short) + .find(|author| !author.is_empty()); + let description = [feed.description.as_deref(), feed.itunes_summary.as_deref()] + .into_iter() + .flatten() + .map(clean_long) + .find(|text| !text.is_empty()); + let image_url = [feed.itunes_image, feed.image_url, feed.logo, feed.icon] + .into_iter() + .flatten() + .find_map(|url| resolve_http_url(&url, self.base.as_ref())); + ParsedFeed { + title, + author, + description, + link: feed + .link + .as_deref() + .map(clean_short) + .filter(|s| !s.is_empty()), + image_url, + new_feed_url: feed + .new_feed_url + .as_deref() + .map(|url| url.trim().to_string()) + .filter(|url| !url.is_empty() && url.len() <= super::MAX_URL_BYTES), + episodes: self.episodes, + } + } +} + +// --------------------------------------------------------------------------- +// Element and attribute helpers + +fn split_qname(name: &[u8]) -> (String, String) { + let name = String::from_utf8_lossy(name); + match name.split_once(':') { + Some((prefix, local)) => (prefix.to_string(), local.to_ascii_lowercase()), + None => (String::new(), name.to_ascii_lowercase()), + } +} + +fn local_name(name: &[u8]) -> String { + split_qname(name).1 +} + +/// Attributes of one element: raw keys with entity-decoded values. +struct Attrs { + raw: Vec<(String, String)>, +} + +impl Attrs { + fn collect(start: &BytesStart<'_>) -> Attrs { + let mut attributes = start.attributes(); + attributes.with_checks(false); + let raw = attributes + .flatten() + .map(|attr| { + let key = String::from_utf8_lossy(attr.key.as_ref()).into_owned(); + let value = decode_entities(&String::from_utf8_lossy(&attr.value)); + (key, value) + }) + .collect(); + Attrs { raw } + } + + /// Value of the attribute whose local name matches `name` (any prefix). + fn get(&self, name: &str) -> Option { + self.raw + .iter() + .find(|(key, _)| { + let local = key.rsplit_once(':').map_or(key.as_str(), |(_, l)| l); + local.eq_ignore_ascii_case(name) && !key.starts_with("xmlns") + }) + .map(|(_, value)| value.clone()) + } +} + +fn push_capped(text: &mut String, value: &str) { + if text.len() >= MAX_CAPTURE_BYTES { + return; + } + let room = MAX_CAPTURE_BYTES - text.len(); + if value.len() <= room { + text.push_str(value); + } else { + let mut end = room; + while !value.is_char_boundary(end) { + end -= 1; + } + text.push_str(&value[..end]); + } +} + +fn parse_len(value: Option) -> Option { + value?.trim().parse::().ok().filter(|len| *len > 0) +} + +fn clean_mime(mime: Option<&str>) -> Option { + let mime = mime?.split(';').next()?.trim().to_ascii_lowercase(); + (!mime.is_empty()).then_some(mime) +} + +const MEDIA_EXTENSIONS: &[&str] = &[ + "mp3", "m4a", "m4b", "aac", "ogg", "oga", "opus", "flac", "wav", "mp4", "m4v", "mov", "webm", + "mka", +]; + +fn has_media_extension(url: &str) -> bool { + let path = url.split(['?', '#']).next().unwrap_or_default(); + let file = path.rsplit('/').next().unwrap_or_default(); + file.rsplit_once('.').is_some_and(|(_, ext)| { + MEDIA_EXTENSIONS + .iter() + .any(|known| known.eq_ignore_ascii_case(ext)) + }) +} + +/// Whether an enclosure/media element points at something playable. +/// `lenient_untyped`: accept a URL without type information (RSS +/// enclosures); media:content without type must at least look like audio. +fn is_playable(url: &str, mime: Option<&str>, medium: Option<&str>, lenient_untyped: bool) -> bool { + let url = url.trim(); + if url.is_empty() { + return false; + } + if let Some(mime) = clean_mime(mime) { + return mime.starts_with("audio/") + || mime.starts_with("video/") + || matches!( + mime.as_str(), + "application/octet-stream" + | "binary/octet-stream" + | "application/ogg" + | "application/mp4" + | "application/x-mpegurl" + | "application/vnd.apple.mpegurl" + ) + || (mime == "application/x-unknown" && has_media_extension(url)); + } + if let Some(medium) = medium.map(|m| m.trim().to_ascii_lowercase()) + && !medium.is_empty() + { + return medium == "audio" || medium == "video"; + } + lenient_untyped || has_media_extension(url) +} + +/// An absolute `http`/`https` URL for a media or image reference: absolute +/// http(s) URLs are kept as written, relative ones (including +/// protocol-relative `//host/...`) are resolved against `base`, and anything +/// else (`file:`, `ftp:`, unresolvable relative paths) yields `None`. +fn resolve_http_url(raw: &str, base: Option<&Url>) -> Option { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + let has_host = |url: &Url| url.host_str().is_some_and(|host| !host.is_empty()); + if is_remote_url(raw) { + // Keep the publisher's spelling when it already satisfies the + // contract; otherwise use the serialized (percent-encoded) form. + let parsed = Url::parse(raw).ok().filter(has_host)?; + return if is_http_url(raw) { + Some(raw.to_string()) + } else { + Some(parsed.to_string()).filter(|url| is_http_url(url)) + }; + } + let resolved = match base { + Some(base) => base.join(raw).ok()?, + None if raw.starts_with("//") => Url::parse(&format!("https:{raw}")).ok()?, + None => return None, + }; + (matches!(resolved.scheme(), "http" | "https") && has_host(&resolved)) + .then(|| resolved.to_string()) + .filter(|url| is_http_url(url)) +} + +fn is_remote_url(value: &str) -> bool { + let value = value.as_bytes(); + let has_prefix = |prefix: &[u8]| { + value.len() >= prefix.len() && value[..prefix.len()].eq_ignore_ascii_case(prefix) + }; + has_prefix(b"http://") || has_prefix(b"https://") +} + +/// Parses a non-negative integer such as `itunes:season`; garbage yields `None`. +fn parse_number(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + text.parse().ok() +} + +// --------------------------------------------------------------------------- +// Text cleanup + +fn clean_short(text: &str) -> String { + html_to_text(text, MAX_TITLE_CHARS) +} + +fn clean_long(text: &str) -> String { + html_to_text(text, MAX_DESCRIPTION_CHARS) +} + +const BLOCK_TAGS: &[&str] = &[ + "br", + "p", + "div", + "li", + "ul", + "ol", + "tr", + "td", + "th", + "table", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "blockquote", + "hr", + "pre", + "section", + "article", + "header", + "footer", + "dd", + "dt", +]; + +/// Converts (possibly HTML) text to plain text: strips tags (dropping +/// `]]> + Jane Host + + https://example.test/small.pngignored"#, + r#" + Episode one + One + <p>First &amp; best</p> + ep-1 + Tue, 02 Jan 2024 10:00:00 GMT + 1:02:03 + + https://example.test/ep1 + + + + + Only itunes title + Long show notes]]> + ep-2 + 754 + + + + Summary only + ep-3 + +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Test Cast"); + assert_eq!(feed.author.as_deref(), Some("Jane Host")); + assert_eq!(feed.description.as_deref(), Some("Weekly talk show.")); + assert_eq!(feed.link.as_deref(), Some("https://example.test/cast")); + assert_eq!( + feed.image_url.as_deref(), + Some("https://example.test/cover.jpg") + ); + assert_eq!(feed.episodes.len(), 3); + + let first = &feed.episodes[0]; + assert_eq!(first.guid.as_deref(), Some("ep-1")); + assert_eq!(first.title, "Episode one"); + assert_eq!(first.description.as_deref(), Some("First & best")); + assert_eq!(first.published_at_ms, Some(1_704_189_600_000)); + assert_eq!(first.duration_ms, Some(3_723_000)); + assert_eq!(first.audio_url, "https://cdn.example.test/1.mp3"); + assert_eq!(first.mime_type.as_deref(), Some("audio/mpeg")); + assert_eq!(first.byte_len, Some(5000)); + assert_eq!(first.link.as_deref(), Some("https://example.test/ep1")); + assert_eq!( + first.image_url.as_deref(), + Some("https://example.test/ep1.jpg") + ); + + let second = &feed.episodes[1]; + assert_eq!(second.title, "Only itunes title"); + assert_eq!(second.description.as_deref(), Some("Long show notes")); + assert_eq!(second.duration_ms, Some(754_000)); + assert_eq!(second.byte_len, None); + assert_eq!(second.published_at_ms, None); + + let third = &feed.episodes[2]; + assert_eq!(third.title, "Untitled episode"); + assert_eq!(third.description.as_deref(), Some("Summary only")); + assert_eq!(third.mime_type, None); +} + +#[test] +fn entities_numeric_refs_and_unknown_html_entities() { + let xml = rss( + "", + r#" + A & B — AП — &bogus; R&D © + x < y && z + e + +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let episode = &feed.episodes[0]; + assert_eq!(episode.title, "A & B — AП — &bogus; R&D ©"); + assert_eq!(episode.description.as_deref(), Some("x < y && z")); + assert_eq!(episode.audio_url, "https://cdn.example.test/a.mp3?x=1&y=2"); +} + +#[test] +fn atom_feed() { + let xml = r#" + + Atom Cast + About <b>things</b> + + + https://example.test/logo.png + Atom Author + +Entry one +urn:uuid:1 +2024-03-01T12:30:00+02:00 +2024-03-05T00:00:00Z +<p>Entry summary</p> + + + + +Entry two +urn:uuid:2 +2024-03-06T00:00:00Z +

Inline

xhtml

+ +
+ +No audio +urn:uuid:3 + + +
"#; + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Atom Cast"); + assert_eq!(feed.description.as_deref(), Some("About things")); + assert_eq!(feed.link.as_deref(), Some("https://example.test/")); + assert_eq!( + feed.image_url.as_deref(), + Some("https://example.test/logo.png") + ); + assert_eq!(feed.author.as_deref(), Some("Atom Author")); + assert_eq!(feed.episodes.len(), 2); + let first = &feed.episodes[0]; + assert_eq!(first.guid.as_deref(), Some("urn:uuid:1")); + assert_eq!(first.published_at_ms, Some(1_709_289_000_000)); + assert_eq!(first.description.as_deref(), Some("Entry summary")); + assert_eq!(first.link.as_deref(), Some("https://example.test/e1")); + assert_eq!(first.audio_url, "https://cdn.example.test/e1.ogg"); + assert_eq!(first.byte_len, Some(42)); + let second = &feed.episodes[1]; + assert_eq!(second.published_at_ms, Some(1_709_683_200_000)); + assert_eq!(second.description.as_deref(), Some("Inline xhtml")); +} + +#[test] +fn items_without_audio_are_skipped_and_media_content_is_a_fallback() { + let xml = r#"T +No enclosurea +Image enclosureb +Empty urlc +Mediad + + + + +Videoe +"#; + let feed = parse_feed(xml.as_bytes()).unwrap(); + let titles: Vec<_> = feed.episodes.iter().map(|e| e.title.as_str()).collect(); + assert_eq!(titles, ["Media", "Video"]); + assert_eq!(feed.episodes[0].audio_url, "https://x.test/d.mp3"); + assert_eq!(feed.episodes[0].byte_len, Some(77)); + assert_eq!( + feed.episodes[0].image_url.as_deref(), + Some("https://x.test/d-thumb.jpg") + ); +} + +#[test] +fn duplicate_guids_keep_the_first_and_missing_guids_stay_absent() { + let items = [ + item("dup", "First", "Mon, 01 Jan 2024 00:00:00 +0000"), + item("dup", "Second", "Tue, 02 Jan 2024 00:00:00 +0000"), + r#"No guid"#.to_string(), + r#"Blank guid "#.to_string(), + ] + .concat(); + let feed = parse_feed(rss("", &items).as_bytes()).unwrap(); + let pairs: Vec<_> = feed + .episodes + .iter() + .map(|e| (e.guid.as_deref(), e.title.as_str(), e.audio_url.as_str())) + .collect(); + assert_eq!( + pairs, + [ + (Some("dup"), "First", "https://cdn.example.test/dup.mp3"), + (None, "No guid", "https://cdn.example.test/noguid.mp3"), + (None, "Blank guid", "https://cdn.example.test/blank.mp3"), + ] + ); +} + +#[test] +fn durations() { + let cases = [ + ("01:02:03", Some(3723.0)), + ("1:02:03", Some(3723.0)), + ("62:03", Some(3723.0)), + ("3723", Some(3723.0)), + ("3723.5", Some(3723.5)), + ("00:00:01.25", Some(1.25)), + (" 45:00 ", Some(2700.0)), + ("0", None), + ("00:00:00", None), + ("", None), + ("abc", None), + ("1:2:3:4", None), + ("-5", None), + ("1e3", None), + ("NaN", None), + ("12:", None), + ]; + for (input, expected) in cases { + assert_eq!(parse_duration(input), expected, "{input:?}"); + } +} + +#[test] +fn dates() { + let reference = Some(1_704_189_600); // 2024-01-02 10:00:00 UTC + let cases = [ + ("Tue, 02 Jan 2024 10:00:00 GMT", reference), + ("Tue, 02 Jan 2024 10:00:00 +0000", reference), + ("02 Jan 2024 10:00:00 UT", reference), + ("Tue, 2 Jan 2024 13:00:00 +0300", reference), + ("Tue, 02 Jan 2024 05:00:00 EST", reference), + ("Tue, 02 Jan 2024 02:00:00 PST", reference), + ("Tuesday, 02 January 2024 10:00 GMT", reference), + ("Tue, 02 Jan 24 10:00:00 GMT", reference), + ("Tue, 02 Jan 2024 13:00:00 +03:00", reference), + ("Tue, 02 Jan 2024 10:00:00", reference), + ("Вт, 02 янв 2024 13:00:00 +0300", reference), + ("2024-01-02T10:00:00Z", reference), + ("2024-01-02T13:00:00+03:00", reference), + ("2024-01-02T10:00:00.123Z", reference), + ("2024-01-02 10:00:00", reference), + ("2024-01-02", Some(1_704_153_600)), + ("not a date", None), + ("Tue, 31 Feb 2024 10:00:00 GMT", None), + ("Tue, 02 Foo 2024 10:00:00 GMT", None), + ("2024-13-02T10:00:00Z", None), + ("", None), + ]; + for (input, expected) in cases { + assert_eq!(parse_date(input), expected, "{input:?}"); + } +} + +#[test] +fn non_ascii_zones_do_not_panic() { + for input in [ + "Mon, 07 Nov 2022 10:00:00 +й0", + "Mon, 07 Nov 2022 10:00:00 -йй", + "Mon, 07 Nov 2022 10:00+й0", + "Mon, 07 Nov 2022 10:00:00 +0й00", + "2022-11-07T10:00:00+0й0", + "2022-11-07T10:00:00+й", + "2022-11-07T10:00:00-€:00", + "2022-11-07T10:00:00+日本", + ] { + assert_eq!(parse_date(input), None, "{input:?}"); + } + // Exhaustive-ish: every short suffix of mixed ASCII / multi-byte + // characters after a sign. + let alphabet = ['0', '9', ':', 'й', '€', '日', '+', '-']; + for a in alphabet { + for b in alphabet { + for c in alphabet { + for prefix in ["Mon, 07 Nov 2022 10:00:00 ", "2022-11-07T10:00:00"] { + for sign in ['+', '-'] { + let _ = parse_date(&format!("{prefix}{sign}{a}{b}{c}")); + } + } + } + } + } + // In a feed, the bad date only loses the date. + let xml = rss("", &item("z", "Zone", "Mon, 07 Nov 2022 10:00:00 +й0")); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); + assert_eq!(feed.episodes[0].published_at_ms, None); +} + +#[test] +fn media_urls_are_resolved_and_must_be_http() { + let xml = rss( + r#""#, + r#"relRelative + + +protoProtocol relative + +fileLocal file + +ftpFTP + +fallbackMedia fallback + + +absAbsolute + "#, + ); + let feed = + parse_feed_with_base(xml.as_bytes(), Some("https://host.test/feeds/rss.xml")).unwrap(); + let urls: Vec<(&str, &str)> = feed + .episodes + .iter() + .map(|e| (e.guid.as_deref().unwrap_or_default(), e.audio_url.as_str())) + .collect(); + assert_eq!( + urls, + [ + ("rel", "https://host.test/audio/1.mp3"), + ("proto", "https://cdn.example.test/2.mp3"), + ("fallback", "https://cdn.example.test/4.mp3"), + ("abs", "HTTPS://cdn.example.test/5.mp3"), + ] + ); + assert_eq!( + feed.image_url.as_deref(), + Some("https://host.test/img/cover.jpg") + ); + assert_eq!( + feed.episodes[0].image_url.as_deref(), + Some("https://host.test/feeds/ep.png") + ); + for episode in &feed.episodes { + assert!(is_remote_url(&episode.audio_url), "{}", episode.audio_url); + } + + // Without a base, relative media cannot be resolved and is dropped; + // protocol-relative URLs default to https. + let feed = parse_feed(xml.as_bytes()).unwrap(); + let guids: Vec<&str> = feed + .episodes + .iter() + .map(|e| e.guid.as_deref().unwrap_or_default()) + .collect(); + assert_eq!(guids, ["proto", "fallback", "abs"]); + assert_eq!(feed.image_url, None); +} + +#[test] +fn rss_with_an_unrelated_default_namespace() { + for ns in ["http://example.test/x", "http://www.w3.org/2005/Atom"] { + let xml = format!( + r#"T + +1One +"# + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "T", "{ns}"); + assert_eq!(feed.episodes.len(), 1, "{ns}"); + assert_eq!(feed.episodes[0].title, "One"); + } +} + +#[test] +fn html_pages_and_garbage_are_rejected() { + let html = br#"Vivat

Hi   there

"#; + let err = parse_feed(html).unwrap_err().to_string(); + assert!(err.contains("not an RSS or Atom feed"), "{err}"); + for garbage in [ + &b""[..], + b"hello world", + b"\x00\xff\xfe garbage", + b"{\"json\": true}", + b"<", + b"<<<>>>", + ] { + let err = parse_feed(garbage).unwrap_err().to_string(); + assert!( + err.contains("not an RSS or Atom feed"), + "{garbage:?}: {err}" + ); + } +} + +#[test] +fn truncated_and_malformed_xml_never_panics() { + let items: String = (0..5) + .map(|i| item(&format!("g{i}"), &format!("T {i} &  "), "bad date")) + .collect(); + let xml = rss( + "", + &items, + ); + let bytes = xml.as_bytes(); + for len in 0..=bytes.len() { + let _ = parse_feed(&bytes[..len]); + } + // Mismatched end tags are tolerated. + let sloppy = r#"SA</b>1"#; + let feed = parse_feed(sloppy.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); + assert_eq!(feed.episodes[0].title, "A"); + // Truncation after complete items keeps them. + let third = xml.find("T 3").unwrap(); + let cut = &bytes[..xml[..third].rfind("<item>").unwrap()]; + let feed = parse_feed(cut).unwrap(); + assert_eq!(feed.episodes.len(), 3); +} + +#[test] +fn deep_nesting_does_not_overflow() { + let depth = 100_000; + let mut xml = String::from("<rss><channel><title>Deep"); + xml.push_str(&"".repeat(depth)); + xml.push_str(&"".repeat(depth)); + xml.push_str(&item("after", "After", "")); + xml.push_str(""); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); +} + +#[test] +fn cyrillic_titles_in_utf8_and_windows_1251() { + let xml = rss( + "Иван Петров", + &item( + "1", + "Выпуск №1: «Привет»", + "Mon, 01 Jan 2024 00:00:00 +0300", + ), + ) + .replace("Test Cast", "Подкаст"); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Подкаст"); + assert_eq!(feed.author.as_deref(), Some("Иван Петров")); + assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»"); + + let cp1251 = xml.replace("encoding=\"UTF-8\"", "encoding=\"windows-1251\""); + let (encoded, _, unmappable) = encoding_rs::WINDOWS_1251.encode(&cp1251); + assert!(!unmappable); + let feed = parse_feed(&encoded).unwrap(); + assert_eq!(feed.title, "Подкаст"); + assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»"); + + // Invalid UTF-8 without a declaration is decoded lossily. + let mut broken = xml.into_bytes(); + let at = broken + .windows(7) + .position(|w| w == b"Test Ca") + .unwrap_or(10); + broken.insert(at, 0xff); + assert!(parse_feed(&broken).is_ok()); +} + +#[test] +fn namespace_prefixes_are_matched_tolerantly() { + // Undeclared `itunes:` prefix and a custom prefix bound to the + // iTunes namespace. + let xml = format!( + r#"PUndeclared +1Custom prefix90 +"# + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.author.as_deref(), Some("Undeclared")); + assert_eq!(feed.episodes[0].title, "Custom prefix"); + assert_eq!(feed.episodes[0].duration_ms, Some(90_000)); +} + +#[test] +fn long_descriptions_are_truncated() { + let long = "word ".repeat(2000); + let text = html_to_text(&long, 4000); + assert!(text.chars().count() <= 4000); + assert!(text.ends_with('…')); + let xml = rss(&format!("{long}"), ""); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!(feed.description.unwrap().chars().count() <= 4000); +} + +#[test] +fn itunes_season_episode_and_type() { + let xml = rss( + "", + r#"aA + 2 7 + Trailer + +bB + two-1 + mystery + +cCbonus + 99999999999 + "#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let a = &feed.episodes[0]; + assert_eq!((a.season, a.episode_number), (Some(2), Some(7))); + assert_eq!(a.episode_type, EpisodeType::Trailer); + let b = &feed.episodes[1]; + assert_eq!((b.season, b.episode_number), (None, None)); + assert_eq!(b.episode_type, EpisodeType::Full); + let c = &feed.episodes[2]; + assert_eq!((c.season, c.episode_number), (None, None)); + assert_eq!(c.episode_type, EpisodeType::Bonus); +} + +#[test] +fn guid_permalink_flag() { + let xml = rss( + "", + r#"x +https://e.test/y +z +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let flags: Vec> = feed.episodes.iter().map(|e| e.guid_is_permalink).collect(); + assert_eq!(flags, [Some(false), Some(true), Some(false), None]); +} + +#[test] +fn new_feed_url_is_reported() { + let xml = rss( + " https://new.example.test/feed?x=1 ", + "", + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!( + feed.new_feed_url.as_deref(), + Some("https://new.example.test/feed?x=1") + ); + let alias = FeedUrl::parse(feed.new_feed_url.as_deref().unwrap()).unwrap(); + assert_eq!(alias.as_str(), "https://new.example.test/feed?x=1"); +} + +#[test] +fn oversized_guid_is_treated_as_absent() { + let long = "g".repeat(MAX_GUID_BYTES + 1); + let xml = rss( + "", + &format!(r#"{long}"#), + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes[0].guid, None); +} + +#[test] +fn media_urls_with_spaces_are_percent_encoded() { + let xml = rss( + "", + r#"s"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes[0].audio_url, "https://c.test/my%20file.mp3"); +} + +#[test] +fn parsed_feed_maps_to_valid_contract_metadata() { + let base = FeedUrl::parse("https://example.test/feed.xml").unwrap(); + let items = [ + item("g1", "One", "Tue, 02 Jan 2024 10:00:00 GMT"), + r#"No guidhttps://example.test/no-guid + Wed, 03 Jan 2024 10:00:00 GMT + "# + .to_string(), + // Same page as the channel link: not identifying. + r#"Home linkhttps://example.test/cast + "# + .to_string(), + ] + .concat(); + let feed = parse_feed_with_base(rss("", &items).as_bytes(), Some(base.as_str())).unwrap(); + let podcast_id = PodcastId::from_feed_url(&base); + let ids = feed.resolve_episode_ids(&podcast_id, &[]); + assert_eq!(ids[0], EpisodeId::from_guid(&podcast_id, "g1")); + assert_eq!( + ids[1], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::link("https://example.test/no-guid").unwrap() + )) + ); + assert_eq!( + ids[2], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::enclosure("https://cdn.example.test/home.mp3").unwrap() + )) + ); + + let podcast = feed.to_podcast_meta(base.clone()); + podcast.validate().unwrap(); + for (episode, id) in feed.episodes.iter().zip(ids) { + let meta = episode.to_meta(&podcast_id, id.unwrap()); + meta.validate().unwrap(); + meta.to_ref(&podcast).validate().unwrap(); + } + + // A later refresh with a changed title and audio URL keeps every id, + // using the stored hints of the previous refresh. + let known: Vec = feed + .episodes + .iter() + .zip(feed.resolve_episode_ids(&podcast_id, &[])) + .map(|(episode, id)| KnownEpisode { + episode_id: id.unwrap(), + hints: episode.identity_hints(), + }) + .collect(); + let refreshed = items + .replace("Home link", "Home link (edited)") + .replace("cdn.example.test/ng.mp3", "cdn2.example.test/ng.mp3"); + let feed2 = parse_feed_with_base(rss("", &refreshed).as_bytes(), Some(base.as_str())).unwrap(); + let ids2 = feed2.resolve_episode_ids(&podcast_id, &known); + let expected: Vec> = + known.iter().map(|k| Some(k.episode_id.clone())).collect(); + assert_eq!(ids2, expected); +} + +#[test] +fn hostile_sizes_stay_bounded() { + let huge_title = "t".repeat(MAX_CAPTURE_BYTES * 2); + let xml = rss( + "", + &format!( + r#"h{huge_title}"# + ), + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!(feed.episodes[0].title.chars().count() <= MAX_TITLE_CHARS); + let duration = rss( + "", + r#"d99999999999999999999"#, + ); + assert_eq!( + parse_feed(duration.as_bytes()).unwrap().episodes[0].duration_ms, + None + ); +} diff --git a/crates/music-dht/src/podcast/mod.rs b/crates/music-dht/src/podcast/mod.rs index a9d6e43..d4b2bb6 100644 --- a/crates/music-dht/src/podcast/mod.rs +++ b/crates/music-dht/src/podcast/mod.rs @@ -46,6 +46,12 @@ //! playback queues and progress operations. A receiving device can show and //! play the episode without being subscribed to the podcast. //! +//! # Feed parser +//! +//! With the `podcast-feed` cargo feature, `feed` provides the shared tolerant +//! RSS 2.0 / RSS 1.0 / Atom parser that produces the identity hints consumed +//! by [`resolve_episode_ids`]. +//! //! ``` //! use music_dht::podcast::{EpisodeId, FeedUrl, PodcastId}; //! @@ -63,6 +69,9 @@ mod identity; mod meta; mod state; +#[cfg(feature = "podcast-feed")] +pub mod feed; + pub use feed_url::{FeedUrl, merge_feed_aliases, resolve_podcast_id}; pub use id::{ EPISODE_ID_DOMAIN, EPISODE_ID_PREFIX, EpisodeId, EpisodeKey, PODCAST_ID_DOMAIN, -- 2.54.0 From 2d4782819d3e112389d24476584c0ca4d74e9730 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 15:40:43 +0200 Subject: [PATCH 3/7] Add podcasts to device-sync protocol v4 Bump DEVICE_SYNC_PROTOCOL_VERSION to 4 (ALPN unchanged) and carry the podcast contract between trusted devices: - podcast_subscription_set, podcast_share_set, podcast_progress_set and podcast_listen_recorded operations with record materializers and validation; unsubscribe is LWW state, share=false is a tombstone; - bounded podcast sections in SyncSnapshot, the recovery path for peers that upgrade after operations were hidden from them; - PlaybackTrack.source with PlaybackSource::{Music, PodcastEpisode, Private}, reserved PlaybackStateWire.playback_rate and an optional self-reported protocol_version in DeviceProfileSet; - gating helpers for peers below v4 (op_allowed_for_peer, playback_allowed_for_peer, playback_command_allowed_from_peer, SyncSnapshot::for_peer) and Jam privacy (redact_for_jam, redact_jam_snapshot, jam_command_allowed, restore_jam_private_items). Tests show that a v3-shaped enum rejects a whole batch containing a new kind, which is why gating is mandatory, and that legacy JSON still parses and music-only messages keep their v3 shape. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/device_sync.rs | 1287 ++++++++++++++++++++++++++- crates/music-dht/src/jam.rs | 64 ++ 2 files changed, 1345 insertions(+), 6 deletions(-) diff --git a/crates/music-dht/src/device_sync.rs b/crates/music-dht/src/device_sync.rs index 6bf3ac6..679abff 100644 --- a/crates/music-dht/src/device_sync.rs +++ b/crates/music-dht/src/device_sync.rs @@ -3,6 +3,22 @@ //! The storage and UI policy live in applications. This module only defines //! the stable JSON-lines messages spoken over the auxiliary iroh stream ALPN, //! plus small helpers for opaque `frid://i/...` invites. +//! +//! # Versions +//! +//! | Profile `protocol_version` | Adds | +//! | --- | --- | +//! | 1 | likes, playlists, membership (`furumi/sync/1`) | +//! | 2 | [`SyncOpPayload::ListenRecorded`] (`furumi/sync/2`) | +//! | 3 | fenced personal playback coordination | +//! | 4 | podcasts: four `podcast_*` operations, podcast snapshot sections, [`PlaybackSource`] on queue items, reserved `playback_rate` | +//! +//! The ALPN stays `furumi/sync/2` for v3 and v4. [`SyncOpPayload`] and +//! [`PlaybackSource`] are closed serde enums: a peer that does not know a +//! `kind` fails to deserialize the **whole** message that contains it, not +//! just one operation. Senders must therefore filter by the peer's version +//! with [`op_allowed_for_peer`] and [`playback_allowed_for_peer`]. See the +//! workspace `PODCASTS.md` for the podcast contract. use std::collections::BTreeMap; use std::str::FromStr; @@ -12,6 +28,12 @@ use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncReadExt}; use crate::error::{MusicDhtError, Result}; +use crate::podcast::{ + EpisodeId, FeedUrl, ListenSession, MAX_PODCAST_SNAPSHOT_BYTES, MAX_SNAPSHOT_LISTEN_SESSIONS, + MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, MAX_SNAPSHOT_SUBSCRIPTIONS, PodcastEpisodeRef, + PodcastError, PodcastId, PodcastMeta, PodcastResult, ProgressRecord, ProgressStatus, + ShareRecord, SubscriptionRecord, +}; /// Legacy ALPN used by personal-device sync v1. pub const SYNC_ALPN_V1: &[u8] = b"furumi/sync/1"; @@ -27,7 +49,22 @@ pub const SYNC_ALPN_V2: &[u8] = b"furumi/sync/2"; pub const SYNC_ALPN: &[u8] = SYNC_ALPN_V2; /// Version of the personal-device sync wire protocol. -pub const DEVICE_SYNC_PROTOCOL_VERSION: u16 = 3; +/// +/// Version 4 adds podcasts. Always advertise the constant of the running +/// client for **this** device only; for other devices keep the version they +/// reported themselves (see [`profile_protocol_version`]). +pub const DEVICE_SYNC_PROTOCOL_VERSION: u16 = 4; + +/// First protocol version that understands podcast operations, podcast +/// snapshot sections and [`PlaybackSource`]. +pub const PODCAST_PROTOCOL_VERSION: u16 = 4; + +/// Version assumed for a device whose profile never reported one. +/// +/// Profiles written by clients before v4 carry no version in +/// [`SyncOpPayload::DeviceProfileSet`]. Treat such a device as the oldest +/// protocol: it will be upgraded as soon as it reports its own version. +pub const UNREPORTED_PROTOCOL_VERSION: u16 = 1; /// Minimum actual listening time retained for an interrupted listen. pub const MIN_RECORDED_LISTEN_MS: i64 = 5_000; @@ -40,6 +77,25 @@ pub const fn supports_listen_history(protocol_version: u16) -> bool { protocol_version >= 2 } +/// Returns whether a peer may receive podcast operations, podcast snapshot +/// sections and playback state that contains podcast episodes. +pub const fn supports_podcasts(protocol_version: u16) -> bool { + protocol_version >= PODCAST_PROTOCOL_VERSION +} + +/// Protocol version to store for a device after reading one of its profiles. +/// +/// `reported` is the version the device itself put into its +/// [`DeviceProfileWire`] (Hello/pairing) or [`SyncOpPayload::DeviceProfileSet`]; +/// `previously_known` is the stored value. Receivers and relays must never +/// substitute their own [`DEVICE_SYNC_PROTOCOL_VERSION`] for another device: +/// doing so would make version gating send podcast state to old clients. +pub fn profile_protocol_version(reported: Option, previously_known: Option) -> u16 { + reported + .or(previously_known) + .unwrap_or(UNREPORTED_PROTOCOL_VERSION) +} + /// Default invite lifetime used by applications unless they need a custom TTL. pub const DEFAULT_INVITE_TTL_MS: i64 = 10 * 60 * 1000; @@ -77,7 +133,10 @@ pub struct DeviceProfileWire { pub name: String, /// Application version. pub client_version: String, - /// Wire protocol version. + /// Wire protocol version reported by **this** device. + /// + /// When relaying other devices' profiles, send the version stored for + /// them, never the relay's own constant. pub protocol_version: u16, /// Transport endpoint id, as string. pub endpoint_id: String, @@ -239,7 +298,31 @@ impl ListenEvent { } } +/// Explicit source of a queue item (protocol v4). +/// +/// Absent on music items from older clients, which are music. Podcast +/// episodes are identified only by this field: never by a negative id, a +/// placeholder release or a URL in `file_path`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PlaybackSource { + /// Music track resolved by `content_id` / `fed`. + Music, + /// Podcast episode streamed from the publisher (boxed: music items stay small). + PodcastEpisode(Box), + /// Opaque placeholder that replaces a private item in Jam snapshots + /// (see [`redact_for_jam`]). It is never playable and carries no title, + /// URL or id. + Private, +} + +/// Display title of [`PlaybackTrack::private_placeholder`]. +pub const PRIVATE_ITEM_TITLE: &str = "Private item"; + /// Portable playback queue track. +/// +/// For podcast items [`PlaybackTrack::source`] is authoritative; the legacy +/// display fields mirror it (see [`PlaybackTrack::podcast_episode`]). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PlaybackTrack { /// Local or placeholder id on the sending device. @@ -285,6 +368,89 @@ pub struct PlaybackTrack { /// Federation metadata for unresolved tracks. #[serde(default)] pub fed: Option, + /// Explicit item source (protocol v4). `None` means music. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +impl PlaybackTrack { + /// Queue item for a podcast episode. + /// + /// Legacy display fields are filled for presentation only: `title` is the + /// episode title, `artist_names` and `release_title` the podcast title, + /// `id`/`release_id` are `0`, `file_path` is empty and `content_id` is + /// `None`. + pub fn podcast_episode(episode: PodcastEpisodeRef) -> Self { + Self { + id: 0, + title: episode.episode_title.clone(), + track_number: None, + disc_number: None, + duration_seconds: episode.duration_ms.unwrap_or(0) as f64 / 1_000.0, + artist_names: vec![episode.podcast_title.clone()], + featured_artist_names: Vec::new(), + release_id: 0, + release_title: episode.podcast_title.clone(), + release_year: None, + file_path: String::new(), + content_id: None, + audio_format: episode.mime.clone(), + audio_bitrate: None, + audio_sample_rate: None, + audio_bit_depth: None, + file_size_bytes: None, + play_count: 0, + fed: None, + source: Some(PlaybackSource::PodcastEpisode(Box::new(episode))), + } + } + + /// Opaque, unplayable placeholder for a private item. + pub fn private_placeholder() -> Self { + Self { + id: 0, + title: PRIVATE_ITEM_TITLE.to_string(), + track_number: None, + disc_number: None, + duration_seconds: 0.0, + artist_names: Vec::new(), + featured_artist_names: Vec::new(), + release_id: 0, + release_title: String::new(), + release_year: None, + file_path: String::new(), + content_id: None, + audio_format: None, + audio_bitrate: None, + audio_sample_rate: None, + audio_bit_depth: None, + file_size_bytes: None, + play_count: 0, + fed: None, + source: Some(PlaybackSource::Private), + } + } + + /// The podcast episode of this item, if it is one. + pub fn podcast_episode_ref(&self) -> Option<&PodcastEpisodeRef> { + match &self.source { + Some(PlaybackSource::PodcastEpisode(episode)) => Some(episode), + _ => None, + } + } + + /// Whether this item is a Jam private placeholder. + pub fn is_private_placeholder(&self) -> bool { + matches!(self.source, Some(PlaybackSource::Private)) + } + + /// Whether this item needs protocol v4 (podcast episode or placeholder). + pub fn requires_podcast_support(&self) -> bool { + matches!( + self.source, + Some(PlaybackSource::PodcastEpisode(_) | PlaybackSource::Private) + ) + } } /// Playback repeat mode. @@ -325,6 +491,12 @@ pub struct PlaybackStateWire { pub shuffle: bool, /// Repeat mode. pub repeat: PlaybackRepeat, + /// Reserved playback rate (1.0 = normal speed); no client uses it yet. + /// + /// When implemented, position extrapolation and handoff must account for + /// it. Receivers that do not support rates treat it as 1.0. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub playback_rate: Option, } /// Playback snapshot with the common coordination envelope. @@ -409,6 +581,13 @@ pub enum SyncOpPayload { endpoint_ticket: String, /// Endpoint id. endpoint_id: String, + /// Protocol version of the origin device (v4+). + /// + /// `None` from older clients. Receivers store + /// [`profile_protocol_version`]`(protocol_version, stored)` and must + /// never substitute their own constant. + #[serde(default, skip_serializing_if = "Option::is_none")] + protocol_version: Option, }, /// Trust a device. DeviceTrusted { @@ -440,10 +619,74 @@ pub enum SyncOpPayload { /// Portable listening event. event: ListenEvent, }, + /// Subscribe to or unsubscribe from a podcast (protocol v4). + /// + /// Last-writer-wins by `(hlc_ms, op_id)` of the operation. Unsubscribing + /// is state, not a tombstone. Only explicit user actions emit it; RSS + /// refreshes never do. Materialize with [`SyncOpWire::podcast_subscription`]. + PodcastSubscriptionSet { + /// Podcast. + podcast_id: PodcastId, + /// Desired subscription state. + subscribed: bool, + /// Channel metadata, so other devices subscribe without the URL. + podcast: PodcastMeta, + /// Known feed URLs (grow-only, bounded). + #[serde(default)] + feed_aliases: Vec, + }, + /// Share or stop sharing a podcast card with the federation (protocol v4). + /// + /// Last-writer-wins; `shared = false` is a tombstone. Unsubscribing also + /// emits `shared = false`. + PodcastShareSet { + /// Podcast. + podcast_id: PodcastId, + /// Desired share state. + shared: bool, + }, + /// Write the durable resume register of an episode (protocol v4). + /// + /// Ordered by `(generation, hlc_ms, origin_device_id, op_id)`; see + /// [`ProgressRecord`]. Clients coalesce periodic checkpoints in their + /// outgoing log: only the latest per episode needs to stay pending. + PodcastProgressSet { + /// Episode. + episode_id: EpisodeId, + /// Owning podcast. + podcast_id: PodcastId, + /// Resume position in milliseconds. + position_ms: i64, + /// Listening status. + status: ProgressStatus, + /// Lamport generation (see [`ProgressRecord`]). + generation: u64, + /// Session or explicit action id. + session_id: String, + /// Display reference, so a device without the feed can show and + /// resume the episode. + #[serde(default)] + episode: Option, + }, + /// Record or update one listening session (protocol v4). + /// + /// Grow-only set keyed by `session.session_id`; see [`ListenSession`]. + /// `session.device_id` must equal the operation origin. + PodcastListenRecorded { + /// Listening session. + session: ListenSession, + /// Display reference for history on devices without the feed. + #[serde(default)] + episode: Option, + }, } impl SyncOpPayload { /// Returns true for operations that remove state and may eventually compact. + /// + /// `podcast_share_set { shared: false }` is a tombstone. Unsubscribing + /// (`podcast_subscription_set { subscribed: false }`) is **not**: it is + /// last-writer-wins state that must keep beating stale subscriptions. pub fn is_tombstone(&self) -> bool { matches!( self, @@ -451,16 +694,274 @@ impl SyncOpPayload { | SyncOpPayload::PlaylistDeleted { .. } | SyncOpPayload::PlaylistTrackRemoved { .. } | SyncOpPayload::DeviceRevoked { .. } + | SyncOpPayload::PodcastShareSet { shared: false, .. } + ) + } + + /// Returns true for the protocol v4 podcast operation kinds. + pub fn is_podcast(&self) -> bool { + matches!( + self, + SyncOpPayload::PodcastSubscriptionSet { .. } + | SyncOpPayload::PodcastShareSet { .. } + | SyncOpPayload::PodcastProgressSet { .. } + | SyncOpPayload::PodcastListenRecorded { .. } ) } } +impl SyncOpWire { + /// Subscription record written by a `podcast_subscription_set` operation. + /// + /// The register stamp is the operation's `(hlc_ms, op_id)`. + pub fn podcast_subscription(&self) -> Option { + let SyncOpPayload::PodcastSubscriptionSet { + podcast_id, + subscribed, + podcast, + feed_aliases, + } = &self.payload + else { + return None; + }; + let mut record = SubscriptionRecord { + podcast_id: podcast_id.clone(), + subscribed: *subscribed, + podcast: podcast.clone(), + feed_aliases: feed_aliases.clone(), + hlc_ms: self.hlc_ms, + op_id: self.op_id.clone(), + }; + record.normalize(); + Some(record) + } + + /// Share record written by a `podcast_share_set` operation. + pub fn podcast_share(&self) -> Option { + let SyncOpPayload::PodcastShareSet { podcast_id, shared } = &self.payload else { + return None; + }; + Some(ShareRecord { + podcast_id: podcast_id.clone(), + shared: *shared, + hlc_ms: self.hlc_ms, + op_id: self.op_id.clone(), + }) + } + + /// Progress register written by a `podcast_progress_set` operation. + /// + /// `device_id` is the operation origin. + pub fn podcast_progress(&self) -> Option { + let SyncOpPayload::PodcastProgressSet { + episode_id, + podcast_id, + position_ms, + status, + generation, + session_id, + .. + } = &self.payload + else { + return None; + }; + Some(ProgressRecord { + episode_id: episode_id.clone(), + podcast_id: podcast_id.clone(), + position_ms: *position_ms, + status: *status, + generation: *generation, + session_id: session_id.clone(), + hlc_ms: self.hlc_ms, + device_id: self.origin_device_id.clone(), + op_id: self.op_id.clone(), + }) + } + + /// Listening session carried by a `podcast_listen_recorded` operation. + pub fn podcast_listen_session(&self) -> Option<&ListenSession> { + match &self.payload { + SyncOpPayload::PodcastListenRecorded { session, .. } => Some(session), + _ => None, + } + } + + /// Validates a podcast operation against the contract. + /// + /// Non-podcast operations are accepted unchanged. Receivers must skip + /// (but still acknowledge) invalid podcast operations. + pub fn validate_podcast(&self) -> PodcastResult<()> { + let check_ref = |episode: &Option, + episode_id: &EpisodeId, + podcast_id: &PodcastId| { + match episode { + Some(episode) + if &episode.episode_id != episode_id || &episode.podcast_id != podcast_id => + { + Err(PodcastError::Mismatch("episode reference")) + } + Some(episode) => episode.validate(), + None => Ok(()), + } + }; + match &self.payload { + SyncOpPayload::PodcastSubscriptionSet { .. } => self + .podcast_subscription() + .expect("subscription payload") + .validate(), + SyncOpPayload::PodcastShareSet { .. } => { + self.podcast_share().expect("share payload").validate() + } + SyncOpPayload::PodcastProgressSet { + episode_id, + podcast_id, + episode, + .. + } => { + self.podcast_progress() + .expect("progress payload") + .validate()?; + check_ref(episode, episode_id, podcast_id) + } + SyncOpPayload::PodcastListenRecorded { session, episode } => { + session.validate()?; + if session.device_id != self.origin_device_id { + return Err(PodcastError::Mismatch( + "session device and operation origin", + )); + } + check_ref(episode, &session.episode_id, &session.podcast_id) + } + _ => Ok(()), + } + } +} + +/// Returns whether `payload` may be sent to a peer with `peer_version`. +/// +/// * `listen_recorded` needs v2; +/// * every `podcast_*` operation needs v4; +/// * a `playback_command` whose state contains podcast items needs v4; +/// * everything else is accepted by every version. +/// +/// Gating is mandatory: an older peer cannot deserialize an unknown `kind` +/// and would reject the entire message, stalling all synchronization. +pub fn op_allowed_for_peer(payload: &SyncOpPayload, peer_version: u16) -> bool { + match payload { + SyncOpPayload::ListenRecorded { .. } => supports_listen_history(peer_version), + SyncOpPayload::PlaybackCommand { command, .. } => { + !playback_command_has_podcasts(command) || supports_podcasts(peer_version) + } + payload if payload.is_podcast() => supports_podcasts(peer_version), + _ => true, + } +} + +/// Returns whether the queue contains podcast episodes or Jam placeholders. +pub fn playback_state_has_podcasts(state: &PlaybackStateWire) -> bool { + state + .queue + .iter() + .any(PlaybackTrack::requires_podcast_support) +} + +/// Returns whether a playback command carries podcast items. +pub fn playback_command_has_podcasts(command: &PlaybackCommand) -> bool { + match command { + crate::playback::Command::SetState { state, .. } + | crate::playback::Command::ActiveChanged { state, .. } => { + playback_state_has_podcasts(state) + } + } +} + +/// Returns whether `state` may be sent to (or handed off to) a peer. +/// +/// For a peer below v4 with podcast items in the queue, do not send the +/// playback snapshot and refuse the handoff with an "update required" message. +/// Never strip the episodes to make the state fit: an old client would then +/// forward a queue without them and lose them for every device. +pub fn playback_allowed_for_peer(state: &PlaybackStateWire, peer_version: u16) -> bool { + !playback_state_has_podcasts(state) || supports_podcasts(peer_version) +} + +/// Returns whether a `SetState`/`ActiveChanged` command from a peer may be +/// applied to the local playback state. +/// +/// While the local queue contains podcast items, commands from peers below +/// v4 are rejected: such a peer does not know the items and its state would +/// silently drop them. +pub fn playback_command_allowed_from_peer(local: &PlaybackStateWire, peer_version: u16) -> bool { + supports_podcasts(peer_version) || !playback_state_has_podcasts(local) +} + +/// Returns a copy of `state` safe to show to Jam participants. +/// +/// Every podcast episode (and any existing placeholder) is replaced by +/// [`PlaybackTrack::private_placeholder`]: no title, URL, feed or id leaves +/// the host. Queue order, position and transport state are kept. +pub fn redact_for_jam(mut state: PlaybackStateWire) -> PlaybackStateWire { + for track in &mut state.queue { + if track.requires_podcast_support() { + *track = PlaybackTrack::private_placeholder(); + } + } + state +} + +/// Returns whether a Jam participant command may be considered at all. +/// +/// Commands that contain real podcast episodes are rejected: participants +/// cannot add episodes to the host queue. Placeholders are allowed here and +/// must be mapped back with [`restore_jam_private_items`]. +pub fn jam_command_allowed(command: &PlaybackCommand) -> bool { + match command { + crate::playback::Command::SetState { state, .. } + | crate::playback::Command::ActiveChanged { state, .. } => !state + .queue + .iter() + .any(|track| track.podcast_episode_ref().is_some()), + } +} + +/// Maps the placeholders of a participant's state back to the host's items. +/// +/// The k-th placeholder of `participant` becomes the k-th private item of +/// `host`. Returns `None` (reject the command) when the participant state +/// contains real podcast episodes or a different number of placeholders, so a +/// participant can reorder music and move placeholders but cannot drop, +/// duplicate or inject private items. +pub fn restore_jam_private_items( + host: &PlaybackStateWire, + mut participant: PlaybackStateWire, +) -> Option { + let mut private = host + .queue + .iter() + .filter(|track| track.requires_podcast_support()); + for track in &mut participant.queue { + if track.podcast_episode_ref().is_some() { + return None; + } + if track.is_private_placeholder() { + *track = private.next()?.clone(); + } + } + private.next().is_none().then_some(participant) +} + /// Materialized sync snapshot sent with every handshake. /// -/// Listening history is intentionally absent: it is an unbounded append-only -/// log, not last-write-wins state. New devices catch it up from missing -/// [`SyncOpPayload::ListenRecorded`] operations using the existing per-origin -/// vector and bounded operation batches. +/// Music listening history is intentionally absent: it is an unbounded +/// append-only log, not last-write-wins state. New devices catch it up from +/// missing [`SyncOpPayload::ListenRecorded`] operations using the existing +/// per-origin vector and bounded operation batches. +/// +/// Podcast state (protocol v4) **is** included, bounded by +/// [`SyncSnapshot::bound_podcast_state`]. Operations hidden from a peer while +/// it ran an older version are never re-requested after it upgrades, because +/// sync vectors only grow, so snapshots are the recovery path. Send podcast +/// sections only to v4 peers ([`SyncSnapshot::for_peer`]). #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct SyncSnapshot { /// Current likes. @@ -478,6 +979,108 @@ pub struct SyncSnapshot { /// Removed playlist items. #[serde(default)] pub removed_playlist_items: Vec, + /// Podcast subscription registers, including unsubscribed ones (v4). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub podcast_subscriptions: Vec, + /// Podcast share registers, including `shared = false` tombstones (v4). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub podcast_shares: Vec, + /// Episode progress registers (v4). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub podcast_progress: Vec, + /// Most recent podcast listening sessions (v4). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub podcast_listen_sessions: Vec, +} + +impl SyncSnapshot { + /// Whether any podcast section is non-empty. + pub fn has_podcast_state(&self) -> bool { + !self.podcast_subscriptions.is_empty() + || !self.podcast_shares.is_empty() + || !self.podcast_progress.is_empty() + || !self.podcast_listen_sessions.is_empty() + } + + /// Removes all podcast sections. + pub fn strip_podcast_state(&mut self) { + self.podcast_subscriptions.clear(); + self.podcast_shares.clear(); + self.podcast_progress.clear(); + self.podcast_listen_sessions.clear(); + } + + /// Copy of the snapshot for a peer: podcast sections only for v4+. + pub fn for_peer(&self, peer_version: u16) -> Self { + let mut snapshot = self.clone(); + if !supports_podcasts(peer_version) { + snapshot.strip_podcast_state(); + } + snapshot + } + + /// Sorts and truncates the podcast sections to the snapshot limits. + /// + /// Each section is ordered newest first (subscriptions, shares and + /// progress by `hlc_ms`, sessions by `ended_at_ms`, ties by id) and capped + /// by its `MAX_SNAPSHOT_*` count. Sections then share + /// [`MAX_PODCAST_SNAPSHOT_BYTES`] of serialized JSON in priority order: + /// subscriptions, shares, progress, sessions. Older registers that do not + /// fit still propagate through operations. + pub fn bound_podcast_state(&mut self) { + let mut budget = MAX_PODCAST_SNAPSHOT_BYTES; + self.podcast_subscriptions.sort_by(|a, b| { + (b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&(a.hlc_ms, &a.op_id, &a.podcast_id)) + }); + bound_section( + &mut self.podcast_subscriptions, + MAX_SNAPSHOT_SUBSCRIPTIONS, + &mut budget, + ); + self.podcast_shares.sort_by(|a, b| { + (b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&(a.hlc_ms, &a.op_id, &a.podcast_id)) + }); + bound_section(&mut self.podcast_shares, MAX_SNAPSHOT_SHARES, &mut budget); + self.podcast_progress.sort_by(|a, b| { + (b.hlc_ms, &b.op_id, &b.episode_id).cmp(&(a.hlc_ms, &a.op_id, &a.episode_id)) + }); + bound_section( + &mut self.podcast_progress, + MAX_SNAPSHOT_PROGRESS, + &mut budget, + ); + self.podcast_listen_sessions + .sort_by(|a, b| (b.ended_at_ms, &b.session_id).cmp(&(a.ended_at_ms, &a.session_id))); + bound_section( + &mut self.podcast_listen_sessions, + MAX_SNAPSHOT_LISTEN_SESSIONS, + &mut budget, + ); + } + + /// Whether the podcast sections respect the per-section count limits. + /// + /// Receivers should ignore the podcast sections of a snapshot that + /// violates them and validate every record before merging it. + pub fn podcast_state_within_limits(&self) -> bool { + self.podcast_subscriptions.len() <= MAX_SNAPSHOT_SUBSCRIPTIONS + && self.podcast_shares.len() <= MAX_SNAPSHOT_SHARES + && self.podcast_progress.len() <= MAX_SNAPSHOT_PROGRESS + && self.podcast_listen_sessions.len() <= MAX_SNAPSHOT_LISTEN_SESSIONS + } +} + +fn bound_section(items: &mut Vec, max_items: usize, budget: &mut usize) { + let mut keep = 0; + for item in items.iter().take(max_items) { + let size = serde_json::to_vec(item).map_or(usize::MAX, |bytes| bytes.len() + 1); + if size > *budget { + break; + } + *budget -= size; + keep += 1; + } + items.truncate(keep); } /// Snapshot like row. @@ -924,4 +1527,676 @@ mod tests { assert!(!supports_listen_history(1)); assert!(supports_listen_history(2)); } + + mod podcasts { + use super::*; + use crate::playback::Command; + use crate::podcast::{FeedUrl, MAX_SNAPSHOT_SHARES}; + + fn feed() -> FeedUrl { + FeedUrl::parse("https://private.example.test/feed?token=s3cr3t").unwrap() + } + + fn podcast_id() -> PodcastId { + PodcastId::from_feed_url(&feed()) + } + + fn episode_id() -> EpisodeId { + EpisodeId::from_guid(&podcast_id(), "ep-1").unwrap() + } + + fn episode() -> PodcastEpisodeRef { + PodcastEpisodeRef { + podcast_id: podcast_id(), + episode_id: episode_id(), + feed_url: feed(), + podcast_title: "Secret Show".to_string(), + episode_title: "Private Episode".to_string(), + audio_url: "https://cdn.example.test/private.mp3?sig=xyz".to_string(), + mime: Some("audio/mpeg".to_string()), + duration_ms: Some(3_600_000), + published_at_ms: Some(1_700_000_000_000), + season: Some(1), + episode_number: Some(3), + image_url: None, + } + } + + fn music(id: i64) -> PlaybackTrack { + PlaybackTrack { + id, + title: format!("Song {id}"), + track_number: Some(1), + disc_number: None, + duration_seconds: 200.0, + artist_names: vec!["Artist".to_string()], + featured_artist_names: Vec::new(), + release_id: 7, + release_title: "Album".to_string(), + release_year: Some(2020), + file_path: String::new(), + content_id: Some(format!("b3:{}", "a".repeat(64))), + audio_format: None, + audio_bitrate: None, + audio_sample_rate: None, + audio_bit_depth: None, + file_size_bytes: None, + play_count: 0, + fed: None, + source: None, + } + } + + fn state(queue: Vec) -> PlaybackStateWire { + PlaybackStateWire { + queue, + queue_pos: 1, + playing: true, + paused: false, + idle_since_ms: None, + position_secs: 42.5, + volume: 80, + shuffle: false, + repeat: PlaybackRepeat::Off, + playback_rate: None, + } + } + + fn mixed() -> PlaybackStateWire { + state(vec![ + music(1), + PlaybackTrack::podcast_episode(episode()), + music(2), + ]) + } + + fn op(seq: i64, payload: SyncOpPayload) -> SyncOpWire { + SyncOpWire { + op_id: format!("dev-a:{seq}"), + origin_device_id: "dev-a".to_string(), + seq, + hlc_ms: 1_000 + seq, + payload, + } + } + + fn session() -> ListenSession { + ListenSession { + session_id: "s-1".to_string(), + episode_id: episode_id(), + podcast_id: podcast_id(), + device_id: "dev-a".to_string(), + started_at_ms: 10, + ended_at_ms: 20, + start_position_ms: 0, + end_position_ms: 60_000, + listened_ms: 60_000, + completed: false, + } + } + + fn podcast_ops() -> Vec { + vec![ + SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id(), + subscribed: true, + podcast: PodcastMeta { + feed_url: feed(), + title: "Secret Show".to_string(), + author: None, + description: Some("About things".to_string()), + link: None, + image_url: None, + }, + feed_aliases: vec![feed()], + }, + SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id(), + shared: false, + }, + SyncOpPayload::PodcastProgressSet { + episode_id: episode_id(), + podcast_id: podcast_id(), + position_ms: 120_000, + status: ProgressStatus::InProgress, + generation: 3, + session_id: "s-1".to_string(), + episode: Some(episode()), + }, + SyncOpPayload::PodcastListenRecorded { + session: session(), + episode: None, + }, + ] + } + + #[test] + fn podcast_ops_round_trip_and_materialize() { + let kinds = [ + "podcast_subscription_set", + "podcast_share_set", + "podcast_progress_set", + "podcast_listen_recorded", + ]; + for (seq, (payload, kind)) in podcast_ops().into_iter().zip(kinds).enumerate() { + let op = op(seq as i64 + 1, payload); + let json = serde_json::to_string(&op).unwrap(); + assert!(json.contains(&format!(r#""kind":"{kind}""#)), "{json}"); + let back: SyncOpWire = serde_json::from_str(&json).unwrap(); + assert_eq!(back, op); + assert!(op.payload.is_podcast()); + op.validate_podcast().unwrap(); + } + let ops: Vec = podcast_ops() + .into_iter() + .enumerate() + .map(|(i, payload)| op(i as i64 + 1, payload)) + .collect(); + let subscription = ops[0].podcast_subscription().unwrap(); + assert_eq!( + (subscription.hlc_ms, subscription.op_id.as_str()), + (1_001, "dev-a:1") + ); + assert!(subscription.subscribed); + let share = ops[1].podcast_share().unwrap(); + assert!(!share.shared); + let progress = ops[2].podcast_progress().unwrap(); + assert_eq!(progress.device_id, "dev-a"); + assert_eq!(progress.op_id, "dev-a:3"); + assert_eq!(progress.generation, 3); + assert_eq!(ops[3].podcast_listen_session(), Some(&session())); + assert!(ops[0].podcast_share().is_none()); + } + + #[test] + fn tombstone_rules() { + let ops = podcast_ops(); + let unsubscribe = SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id(), + subscribed: false, + podcast: match &ops[0] { + SyncOpPayload::PodcastSubscriptionSet { podcast, .. } => podcast.clone(), + _ => unreachable!(), + }, + feed_aliases: Vec::new(), + }; + assert!(!unsubscribe.is_tombstone(), "unsubscribe is LWW state"); + assert!(ops[1].is_tombstone(), "share=false is a tombstone"); + let share = SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id(), + shared: true, + }; + assert!(!share.is_tombstone()); + assert!(!ops[2].is_tombstone()); + assert!(!ops[3].is_tombstone()); + } + + #[test] + fn invalid_podcast_ops_are_detected() { + let mut foreign = session(); + foreign.device_id = "dev-b".to_string(); + let op1 = op( + 1, + SyncOpPayload::PodcastListenRecorded { + session: foreign, + episode: None, + }, + ); + assert!(op1.validate_podcast().is_err()); + let mut wrong = episode(); + wrong.episode_id = EpisodeId::from_guid(&podcast_id(), "other").unwrap(); + let op2 = op( + 2, + SyncOpPayload::PodcastProgressSet { + episode_id: episode_id(), + podcast_id: podcast_id(), + position_ms: 1, + status: ProgressStatus::InProgress, + generation: 1, + session_id: "s".to_string(), + episode: Some(wrong), + }, + ); + assert!(op2.validate_podcast().is_err()); + let op3 = op( + 3, + SyncOpPayload::PodcastProgressSet { + episode_id: episode_id(), + podcast_id: podcast_id(), + position_ms: -5, + status: ProgressStatus::InProgress, + generation: 1, + session_id: "s".to_string(), + episode: None, + }, + ); + assert!(op3.validate_podcast().is_err()); + // Music operations are not affected. + let like = op( + 4, + SyncOpPayload::TrackLikeSet { + content_id: "b3:x".to_string(), + liked: true, + fed: None, + }, + ); + like.validate_podcast().unwrap(); + } + + #[test] + fn legacy_v3_json_still_parses() { + let track = r#"{"id":5,"title":"T","track_number":null,"disc_number":null, + "duration_seconds":1.0,"release_id":1,"release_title":"R","release_year":null, + "content_id":null,"audio_format":null,"audio_bitrate":null, + "audio_sample_rate":null,"audio_bit_depth":null,"file_size_bytes":null}"#; + let track: PlaybackTrack = serde_json::from_str(track).unwrap(); + assert_eq!(track.source, None); + let state = r#"{"queue":[],"playing":false,"paused":true,"position_secs":0.0, + "shuffle":false,"repeat":"off"}"#; + let state: PlaybackStateWire = serde_json::from_str(state).unwrap(); + assert_eq!(state.playback_rate, None); + let profile = r#"{"kind":"device_profile_set","name":"n","client_version":"1", + "endpoint_ticket":"t","endpoint_id":"e"}"#; + match serde_json::from_str::(profile).unwrap() { + SyncOpPayload::DeviceProfileSet { + protocol_version, .. + } => { + assert_eq!(protocol_version, None) + } + other => panic!("{other:?}"), + } + let snapshot: SyncSnapshot = + serde_json::from_str(r#"{"likes":[],"playlists":[]}"#).unwrap(); + assert!(!snapshot.has_podcast_state()); + } + + #[test] + fn music_only_messages_keep_the_v3_shape() { + let json = serde_json::to_value(state(vec![music(1)])).unwrap(); + assert!(json.get("playback_rate").is_none()); + assert!(json["queue"][0].get("source").is_none()); + let snapshot = serde_json::to_value(SyncSnapshot::default()).unwrap(); + assert!(snapshot.get("podcast_progress").is_none()); + let profile = serde_json::to_value(SyncOpPayload::DeviceProfileSet { + name: "n".to_string(), + client_version: "1".to_string(), + endpoint_ticket: "t".to_string(), + endpoint_id: "e".to_string(), + protocol_version: None, + }) + .unwrap(); + assert!(profile.get("protocol_version").is_none()); + } + + #[test] + fn podcast_queue_items_round_trip() { + let mut state = mixed(); + state.playback_rate = Some(1.5); + let json = serde_json::to_string(&state).unwrap(); + assert!(json.contains(r#""kind":"podcast_episode""#)); + let back: PlaybackStateWire = serde_json::from_str(&json).unwrap(); + assert_eq!(back, state); + let item = &back.queue[1]; + assert_eq!(item.podcast_episode_ref(), Some(&episode())); + assert_eq!(item.file_path, ""); + assert_eq!(item.id, 0); + assert_eq!(item.content_id, None); + assert_eq!(item.duration_seconds, 3_600.0); + let explicit_music = PlaybackTrack { + source: Some(PlaybackSource::Music), + ..music(3) + }; + let json = serde_json::to_string(&explicit_music).unwrap(); + assert!(json.contains(r#""source":{"kind":"music"}"#), "{json}"); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + explicit_music + ); + } + + /// The v3 operation enum, as compiled into deployed v3 clients. + mod v3 { + use serde::Deserialize; + + #[derive(Debug, Deserialize)] + #[serde(tag = "kind", rename_all = "snake_case")] + #[allow(dead_code)] + pub enum SyncOpPayload { + TrackLikeSet { + content_id: String, + liked: bool, + }, + PlaylistCreated { + playlist_id: String, + title: String, + }, + PlaylistRenamed { + playlist_id: String, + title: String, + }, + PlaylistDeleted { + playlist_id: String, + }, + PlaylistTrackAdded { + playlist_id: String, + content_id: String, + position: i64, + }, + PlaylistTrackRemoved { + playlist_id: String, + content_id: String, + }, + DeviceProfileSet { + name: String, + client_version: String, + endpoint_ticket: String, + endpoint_id: String, + }, + DeviceTrusted { + target_device_id: String, + }, + DeviceRevoked { + target_device_id: String, + target_max_seq_seen: i64, + }, + PlaybackCommand { + target_device_id: String, + }, + ListenRecorded { + event: serde_json::Value, + }, + } + + #[derive(Debug, Deserialize)] + #[allow(dead_code)] + pub struct SyncOpWire { + pub op_id: String, + pub origin_device_id: String, + pub seq: i64, + pub hlc_ms: i64, + pub payload: SyncOpPayload, + } + } + + #[test] + fn v3_peers_cannot_parse_new_kinds_so_gating_is_mandatory() { + let like = op( + 1, + SyncOpPayload::TrackLikeSet { + content_id: "b3:x".to_string(), + liked: true, + fed: None, + }, + ); + let profile = op( + 2, + SyncOpPayload::DeviceProfileSet { + name: "n".to_string(), + client_version: "2".to_string(), + endpoint_ticket: "t".to_string(), + endpoint_id: "e".to_string(), + protocol_version: Some(DEVICE_SYNC_PROTOCOL_VERSION), + }, + ); + // Additive fields are fine for v3. + for compatible in [&like, &profile] { + let json = serde_json::to_string(compatible).unwrap(); + serde_json::from_str::(&json).unwrap(); + } + for (seq, payload) in podcast_ops().into_iter().enumerate() { + let podcast = op(seq as i64 + 10, payload); + let json = serde_json::to_string(&podcast).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + // One unknown op poisons the whole batch of a Hello message. + let batch = serde_json::to_string(&vec![like.clone(), podcast.clone()]).unwrap(); + assert!(serde_json::from_str::>(&batch).is_err()); + assert!(!op_allowed_for_peer(&podcast.payload, 3)); + assert!(op_allowed_for_peer(&podcast.payload, 4)); + } + } + + #[test] + fn gating_boundaries() { + assert!(!supports_podcasts(3)); + assert!(supports_podcasts(4)); + assert!(supports_podcasts(5)); + let listen = SyncOpPayload::ListenRecorded { + event: listen(150_000, Some(300_000), ListenEndReason::Finished), + }; + assert!(!op_allowed_for_peer(&listen, 1)); + assert!(op_allowed_for_peer(&listen, 2)); + for payload in podcast_ops() { + for version in 0..=3 { + assert!(!op_allowed_for_peer(&payload, version)); + } + assert!(op_allowed_for_peer(&payload, 4)); + } + let like = SyncOpPayload::TrackLikeSet { + content_id: "b3:x".to_string(), + liked: false, + fed: None, + }; + assert!(op_allowed_for_peer(&like, 1)); + + let command = |state| SyncOpPayload::PlaybackCommand { + target_device_id: "dev-b".to_string(), + command: Command::ActiveChanged { + active_device_id: "dev-b".to_string(), + active_device_name: "B".to_string(), + state, + }, + authority: None, + }; + assert!(op_allowed_for_peer(&command(state(vec![music(1)])), 3)); + assert!(!op_allowed_for_peer(&command(mixed()), 3)); + assert!(op_allowed_for_peer(&command(mixed()), 4)); + + assert!(playback_allowed_for_peer(&state(vec![music(1)]), 1)); + assert!(!playback_allowed_for_peer(&mixed(), 3)); + assert!(playback_allowed_for_peer(&mixed(), 4)); + let redacted = redact_for_jam(mixed()); + assert!(!playback_allowed_for_peer(&redacted, 3)); + + assert!(playback_command_allowed_from_peer( + &state(vec![music(1)]), + 3 + )); + assert!(!playback_command_allowed_from_peer(&mixed(), 3)); + assert!(playback_command_allowed_from_peer(&mixed(), 4)); + } + + #[test] + fn profile_versions_are_never_substituted() { + assert_eq!(profile_protocol_version(Some(4), Some(3)), 4); + assert_eq!(profile_protocol_version(None, Some(3)), 3); + assert_eq!( + profile_protocol_version(None, None), + UNREPORTED_PROTOCOL_VERSION + ); + assert!(!supports_podcasts(profile_protocol_version(None, None))); + } + + #[test] + fn jam_redaction_hides_private_items() { + let original = mixed(); + let redacted = redact_for_jam(original.clone()); + assert_eq!(redacted.queue.len(), 3); + assert_eq!(redacted.queue[0], original.queue[0]); + assert_eq!(redacted.queue[2], original.queue[2]); + assert!(redacted.queue[1].is_private_placeholder()); + assert_eq!(redacted.queue[1].title, PRIVATE_ITEM_TITLE); + assert_eq!(redacted.queue_pos, original.queue_pos); + assert_eq!(redacted.position_secs, original.position_secs); + let json = serde_json::to_string(&redacted).unwrap(); + for secret in [ + "Secret Show", + "Private Episode", + "private.mp3", + "s3cr3t", + "private.example.test", + podcast_id().as_str(), + episode_id().as_str(), + ] { + assert!(!json.contains(secret), "{secret} leaked: {json}"); + } + assert!(json.contains(r#""source":{"kind":"private"}"#)); + // Redaction is idempotent. + assert_eq!(redact_for_jam(redacted.clone()), redacted); + } + + #[test] + fn jam_commands_cannot_inject_or_drop_private_items() { + let host = mixed(); + let set = |state| Command::SetState { state, seek: false }; + // Participants see placeholders; echoing them back is allowed + // and maps to the host's episodes. + let mut seen = redact_for_jam(host.clone()); + seen.paused = true; + assert!(jam_command_allowed(&set(seen.clone()))); + let restored = restore_jam_private_items(&host, seen.clone()).unwrap(); + assert_eq!(restored.queue, host.queue); + assert!(restored.paused); + // Reordering music around the placeholder keeps the episode. + let mut reordered = seen.clone(); + reordered.queue.swap(0, 2); + let restored = restore_jam_private_items(&host, reordered).unwrap(); + assert_eq!(restored.queue[1].podcast_episode_ref(), Some(&episode())); + // Injecting a real episode is rejected. + let injected = state(vec![music(1), PlaybackTrack::podcast_episode(episode())]); + assert!(!jam_command_allowed(&set(injected.clone()))); + assert!(restore_jam_private_items(&host, injected).is_none()); + // Dropping or duplicating placeholders is rejected. + let mut dropped = seen.clone(); + dropped.queue.remove(1); + assert!(restore_jam_private_items(&host, dropped).is_none()); + let mut duplicated = seen; + duplicated.queue.push(PlaybackTrack::private_placeholder()); + assert!(restore_jam_private_items(&host, duplicated).is_none()); + // Music-only Jam is unaffected. + let music_only = state(vec![music(1)]); + assert!(jam_command_allowed(&set(music_only.clone()))); + assert_eq!( + restore_jam_private_items(&music_only, music_only.clone()), + Some(music_only) + ); + } + + #[test] + fn snapshot_sections_round_trip_and_are_gated() { + let ops: Vec = podcast_ops() + .into_iter() + .enumerate() + .map(|(i, payload)| op(i as i64 + 1, payload)) + .collect(); + let snapshot = SyncSnapshot { + podcast_subscriptions: vec![ops[0].podcast_subscription().unwrap()], + podcast_shares: vec![ops[1].podcast_share().unwrap()], + podcast_progress: vec![ops[2].podcast_progress().unwrap()], + podcast_listen_sessions: vec![session()], + ..SyncSnapshot::default() + }; + let message = WireMessage::Hello { + group_id: "g".to_string(), + profile: DeviceProfileWire { + device_id: "dev-a".to_string(), + name: "A".to_string(), + client_version: "1".to_string(), + protocol_version: DEVICE_SYNC_PROTOCOL_VERSION, + endpoint_id: "e".to_string(), + endpoint_ticket: "t".to_string(), + revoked: false, + revoke_cutoff_seq: None, + updated_at_ms: 0, + }, + devices: Vec::new(), + vector: BTreeMap::new(), + ops, + snapshot: snapshot.clone(), + playback: None, + }; + let json = serde_json::to_string(&message).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), message); + assert!(snapshot.has_podcast_state()); + assert!(!snapshot.for_peer(3).has_podcast_state()); + assert_eq!(snapshot.for_peer(4), snapshot); + } + + #[test] + fn snapshot_bounds_accept_the_limit_and_cut_the_next_record() { + let shares = |count: usize| -> Vec { + (0..count) + .map(|i| ShareRecord { + podcast_id: PodcastId::from_feed_url( + &FeedUrl::parse(&format!("https://a.test/{i}")).unwrap(), + ), + shared: i % 2 == 0, + hlc_ms: i as i64, + op_id: format!("dev:{i}"), + }) + .collect() + }; + let mut at = SyncSnapshot { + podcast_shares: shares(MAX_SNAPSHOT_SHARES), + ..SyncSnapshot::default() + }; + assert!(at.podcast_state_within_limits()); + at.bound_podcast_state(); + assert_eq!(at.podcast_shares.len(), MAX_SNAPSHOT_SHARES); + let mut over = SyncSnapshot { + podcast_shares: shares(MAX_SNAPSHOT_SHARES + 1), + ..SyncSnapshot::default() + }; + assert!(!over.podcast_state_within_limits()); + over.bound_podcast_state(); + assert_eq!(over.podcast_shares.len(), MAX_SNAPSHOT_SHARES); + // Newest first: the oldest record is the one cut. + assert_eq!(over.podcast_shares[0].hlc_ms, MAX_SNAPSHOT_SHARES as i64); + assert!(over.podcast_shares.iter().all(|share| share.hlc_ms > 0)); + } + + #[test] + fn snapshot_byte_budget_is_enforced() { + let big = "x".repeat(crate::podcast::MAX_DESCRIPTION_CHARS); + let subscriptions: Vec = (0..MAX_SNAPSHOT_SUBSCRIPTIONS) + .map(|i| { + let url = FeedUrl::parse(&format!("https://a.test/{i}")).unwrap(); + SubscriptionRecord { + podcast_id: PodcastId::from_feed_url(&url), + subscribed: true, + podcast: PodcastMeta { + feed_url: url, + title: big.chars().take(1_000).collect(), + author: None, + description: Some(big.clone()), + link: None, + image_url: None, + }, + feed_aliases: Vec::new(), + hlc_ms: i as i64, + op_id: format!("dev:{i}"), + } + }) + .collect(); + let mut snapshot = SyncSnapshot { + podcast_subscriptions: subscriptions, + podcast_listen_sessions: vec![session()], + ..SyncSnapshot::default() + }; + snapshot.bound_podcast_state(); + let bytes: usize = snapshot + .podcast_subscriptions + .iter() + .map(|record| serde_json::to_vec(record).unwrap().len() + 1) + .sum::() + + snapshot + .podcast_listen_sessions + .iter() + .map(|record| serde_json::to_vec(record).unwrap().len() + 1) + .sum::(); + assert!(bytes <= MAX_PODCAST_SNAPSHOT_BYTES); + assert!(snapshot.podcast_subscriptions.len() < MAX_SNAPSHOT_SUBSCRIPTIONS); + assert!(!snapshot.podcast_subscriptions.is_empty()); + assert!(serde_json::to_vec(&snapshot).unwrap().len() < MAX_SYNC_LINE); + } + } } diff --git a/crates/music-dht/src/jam.rs b/crates/music-dht/src/jam.rs index 346716e..b208b42 100644 --- a/crates/music-dht/src/jam.rs +++ b/crates/music-dht/src/jam.rs @@ -3,6 +3,12 @@ //! Jam is separate from personal-device sync. Possession of a [`JamInvite`] //! capability authorizes playback control for one host process, but never //! grants likes, playlists, history, or trusted-device membership. +//! +//! Podcast episodes are private. Hosts send [`redact_jam_snapshot`] to +//! participants, reject participant commands for which +//! [`crate::device_sync::jam_command_allowed`] is false, and map placeholders +//! back with [`crate::device_sync::restore_jam_private_items`] before applying +//! a participant's state. use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -140,6 +146,17 @@ pub enum JamWireMessage { }, } +/// Returns a playback snapshot safe to send to Jam participants. +/// +/// Podcast episodes become opaque placeholders (see +/// [`crate::device_sync::redact_for_jam`]) and the personal coordination +/// envelope is removed, since it belongs to the trusted-device scope. +pub fn redact_jam_snapshot(mut snapshot: PlaybackSnapshot) -> PlaybackSnapshot { + snapshot.state = crate::device_sync::redact_for_jam(snapshot.state); + snapshot.coordination = None; + snapshot +} + /// Reads a bounded Jam message from an async reader. pub async fn read_message_from(reader: &mut R) -> Result { let mut line = Vec::new(); @@ -277,6 +294,53 @@ mod tests { assert!(JamInvite::from_uri("frid://i/abcd").is_err()); } + #[test] + fn jam_snapshots_hide_podcast_episodes() { + use crate::device_sync::{PlaybackRepeat, PlaybackStateWire, PlaybackTrack}; + use crate::podcast::{EpisodeId, FeedUrl, PodcastEpisodeRef, PodcastId}; + + let feed = FeedUrl::parse("https://private.test/feed").unwrap(); + let podcast_id = PodcastId::from_feed_url(&feed); + let episode = PodcastEpisodeRef { + episode_id: EpisodeId::from_guid(&podcast_id, "e").unwrap(), + podcast_id, + feed_url: feed, + podcast_title: "Hidden show".to_string(), + episode_title: "Hidden episode".to_string(), + audio_url: "https://private.test/e.mp3".to_string(), + mime: None, + duration_ms: None, + published_at_ms: None, + season: None, + episode_number: None, + image_url: None, + }; + let snapshot = PlaybackSnapshot { + device_id: "host".to_string(), + device_name: "Host".to_string(), + active: true, + updated_at_ms: 1, + state: PlaybackStateWire { + queue: vec![PlaybackTrack::podcast_episode(episode)], + queue_pos: 0, + playing: true, + paused: false, + idle_since_ms: None, + position_secs: 1.0, + volume: 50, + shuffle: false, + repeat: PlaybackRepeat::Off, + playback_rate: None, + }, + coordination: None, + }; + let redacted = redact_jam_snapshot(snapshot); + let json = serde_json::to_string(&redacted).unwrap(); + assert!(!json.contains("Hidden")); + assert!(!json.contains("private.test")); + assert!(redacted.state.queue[0].is_private_placeholder()); + } + #[test] fn weak_capability_is_rejected() { let mut invite = invite(); -- 2.54.0 From 4f7e9992802692be5ebec0c8a0006b0a06922362 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 15:40:55 +0200 Subject: [PATCH 4/7] Document podcast contract and bump crate versions Add PODCASTS.md as the normative podcast contract: identity and URL canonicalization, the no-GUID rule, synced versus local versus public data, operation kinds and merge rules with examples, snapshot recovery, version gating for peers below v4, Jam privacy and the reserved playback_rate. Update PLAYBACK_PROTOCOL.md, ARCHITECTURE.md, the crate READMEs and CHANGELOG.md. Bump music-dht to 0.6.0 and furumi-library to 0.3.0: new public enum variants and struct fields are breaking changes for 0.x, and furumi-library re-exposes music-dht types. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 33 ++- CHANGELOG.md | 18 ++ Cargo.lock | 4 +- Cargo.toml | 2 +- PLAYBACK_PROTOCOL.md | 46 +++- PODCASTS.md | 400 +++++++++++++++++++++++++++++++ README.md | 2 +- crates/furumi-library/Cargo.toml | 2 +- crates/furumi-library/README.md | 4 + crates/music-dht/Cargo.toml | 2 +- crates/music-dht/README.md | 56 ++++- crates/music-dht/src/lib.rs | 4 + 12 files changed, 563 insertions(+), 10 deletions(-) create mode 100644 PODCASTS.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f226926..3129f45 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -177,13 +177,39 @@ their local output is playing. The web gateway uses a passive server profile; browser startup and explicit selection are separate inputs. This register provides eventual exclusivity after a partition heals, not a -quorum lock during a partition. Device-sync profiles now advertise version 3; +quorum lock during a partition. Device-sync profiles now advertise version 4 +(version 3 introduced coordination, version 4 adds podcast queue items); the additive coordination envelope is independently versioned and transported over the existing JSON `furumi/sync/2` stream. Legacy library records remain readable, but legacy unfenced personal playback messages cannot control an upgraded output. See [PLAYBACK_PROTOCOL.md](PLAYBACK_PROTOCOL.md) for the contract, adapter obligations, configuration and rollout rules. +Podcasts are a separate content type with a pure shared contract in +`music_dht::podcast` (normative description: [PODCASTS.md](PODCASTS.md)). It +owns canonical feed URLs, domain-separated BLAKE3 podcast and episode ids, +feed aliases, the no-GUID episode rule, bounded metadata DTOs, the +self-contained `PodcastEpisodeRef` queue item, and convergent merge functions +for subscriptions, share decisions, progress registers and listening +sessions. The tolerant RSS/Atom parser is shared behind the `podcast-feed` +feature because episode identity depends on it. HTTP fetching, scheduling, +buffering and decoding remain client runtime work, and persistence lives in +client storage that implements the same rules. Podcast audio is fetched from +the publisher: it is never imported, content-addressed or offered for P2P +transfer. + +Device-sync protocol v4 carries podcast state between trusted devices only: +four `podcast_*` operations, bounded snapshot sections (the recovery path, +because sync vectors never re-request operations hidden from an older peer), +and an explicit `PlaybackTrack.source`. Operation and source enums are closed, +so senders gate on the version each peer reported itself; podcast state is +never sent to peers below v4, handoff to them is refused, and their playback +commands are ignored while the queue contains episodes. Jam snapshots replace +episodes with opaque placeholders. Progress ordering uses a Lamport +generation rather than wall-clock `hlc_ms`, so a device with a fast clock +cannot undo an observed seek or restart. Public podcast discovery is a future +separate overlay; the music DHT schema and `federation-net` are unchanged. + Music discovery identifies an owner and content; it does not force all domain traffic through DHT messages. `music-dht` exposes the underlying authenticated stream capability and defines shared wire models for Furumi catalog and device @@ -283,3 +309,8 @@ architectural constraint: - Local durable state survives ordinary peer and network failure. - Additional stream protocols reuse authenticated connectivity without coupling bulk data to the typed control channel. +- Private personal state (podcast subscriptions, progress, history) stays in + the trusted-device group; Jam and public overlays only see explicit public + projections or opaque placeholders. +- A device's protocol version is what that device reported; relays never + substitute their own. diff --git a/CHANGELOG.md b/CHANGELOG.md index e734e7c..09475a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased — podcast contract + +- `music-dht 0.6.0` (breaking): new `music_dht::podcast` module with the shared + podcast contract: canonical `FeedUrl`, `PodcastId`/`EpisodeId` derivation + (`furumi-podcast:*:v1`), feed aliases, the no-GUID episode identity rule, + `PodcastMeta`/`EpisodeMeta`/`PodcastEpisodeRef`, and convergent merge rules + for subscriptions, shares, progress registers and listening sessions. +- `music-dht 0.6.0`: optional `podcast-feed` feature with the shared tolerant + RSS/Atom parser (`quick-xml`, `encoding_rs`, `url`). +- `music-dht 0.6.0`: device-sync protocol version 4 (ALPN unchanged): four + `podcast_*` operation kinds, bounded podcast snapshot sections, + `PlaybackTrack.source`, reserved `PlaybackStateWire.playback_rate`, + `DeviceProfileSet.protocol_version`, version-gating helpers and Jam + redaction. New struct fields and enum variants break exhaustive + construction and matching in clients. +- `furumi-library 0.3.0`: follows `music-dht 0.6.0`, whose types it re-exposes. +- See `PODCASTS.md` for the normative contract. + ## v0.4.1 — 2026-09-02 - `federation-net 0.3.1`: recover timed-out or failed Mainline-DHT rendezvous clients and expose network health to applications. diff --git a/Cargo.lock b/Cargo.lock index 9c9ccc9..f22946e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -932,7 +932,7 @@ dependencies = [ [[package]] name = "furumi-library" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", "blake3", @@ -2099,7 +2099,7 @@ checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" [[package]] name = "music-dht" -version = "0.5.0" +version = "0.6.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 86aa1c5..4533dd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ repository = "https://gt.hexor.cy/ab/frid" [workspace.dependencies] federation-net = { path = "crates/federation-net", version = "0.3.1" } -music-dht = { path = "crates/music-dht", version = "0.5.0" } +music-dht = { path = "crates/music-dht", version = "0.6.0" } iroh = "1" iroh-base = "1" iroh-tickets = "1" diff --git a/PLAYBACK_PROTOCOL.md b/PLAYBACK_PROTOCOL.md index 3f71d8a..eaaaa7f 100644 --- a/PLAYBACK_PROTOCOL.md +++ b/PLAYBACK_PROTOCOL.md @@ -97,8 +97,9 @@ is a runtime input, not a persisted setting. ## Versioning and rollout The library-sync JSON ALPN remains `furumi/sync/2`; catalog, pairing and library -data do not require a new transport. Device profiles advertise protocol version -3. Coordination envelopes independently carry version 1. Snapshot coordination +data do not require a new transport. Device profiles advertised protocol +version 3 when coordination was introduced (version 4 adds podcast queue items, +see below). Coordination envelopes independently carry version 1. Snapshot coordination and operation authority are additive optional JSON fields so library-sync records from older clients still deserialize. Upgraded clients ignore legacy personal playback snapshots/commands without the coordination envelope. @@ -109,6 +110,47 @@ applications while preventing divergent control message definitions. These envelopes use JSON, not a backwards-compatible promise for postcard layouts. Do not change Jam's independent authority by interpreting personal stamps there. +## Protocol v4: podcast queue items + +Device profiles advertise protocol version 4. The JSON ALPN stays +`furumi/sync/2` and the coordination envelope stays version 1. Version 4 adds +to the portable queue DTO: + +- `PlaybackTrack.source`: `music`, `podcast_episode` with a self-contained + `PodcastEpisodeRef`, or `private` (Jam placeholder). It is omitted for + music, so music-only state serializes exactly as in v3. Podcast episodes are + never encoded as negative ids or URLs in `file_path`. +- `PlaybackStateWire.playback_rate`: reserved, omitted when unset. Treat a + missing or unsupported value as 1.0. + +`PlaybackSource` is a closed enum: a v3 client fails to parse any message +with a podcast item. Adapters therefore apply the shared helpers from +`music_dht::device_sync`: + +1. Send a playback snapshot to a peer only when + `playback_allowed_for_peer(state, peer_version)` holds; otherwise send + `None`. +2. Refuse an explicit transfer to a device below v4 while the queue contains + podcast items and show that the target needs an update. The decision uses + the version that device reported itself (`profile_protocol_version`), + never the local constant. Do not strip episodes from the queue. +3. Ignore `SetState`/`ActiveChanged` from peers below v4 while the local + queue contains podcast items (`playback_command_allowed_from_peer`). +4. A stamped `playback_command` operation with podcast items is itself gated + by `op_allowed_for_peer`. + +A handoff of a podcast episode records the position and the play/pause intent +first, transfers the queue with its sources, and lets the receiver open the +publisher's HTTP source and seek. Ownership still changes only through the +claim, so a late response cannot make the previous owner resume audio. The +durable episode progress is a separate register; see `PODCASTS.md`. + +Jam hosts send `music_dht::jam::redact_jam_snapshot`, which replaces podcast +items with opaque `private` placeholders and drops the personal coordination +envelope. They reject participant commands with real episodes +(`jam_command_allowed`) and map placeholders back with +`restore_jam_private_items` before applying a participant's state. + For local integration checks, patch the three frid crates with Cargo's external `--config` mechanism. Keep released dependency declarations and lockfiles in the applications; publish frid and then bump client dependencies for rollout. diff --git a/PODCASTS.md b/PODCASTS.md new file mode 100644 index 0000000..71d5de1 --- /dev/null +++ b/PODCASTS.md @@ -0,0 +1,400 @@ +# Podcast contract + +Status: normative for `music-dht` 0.6 and device-sync protocol v4. + +Podcasts are a separate content type shared by every Furumi client. This +document defines what all clients must agree on: identity, normalization, +private state and its merge rules, trusted-device sync operations, snapshot +recovery, queue items, version gating and Jam privacy. The Rust definitions +live in `music_dht::podcast` (pure types and functions) and +`music_dht::device_sync` (wire DTOs and gating helpers). Clients must use +them instead of local copies. + +Public discovery (signed podcast cards in a separate DHT overlay) is not part +of this version of the contract; it will be specified with the overlay. + +## Responsibilities + +| Layer | Owns | +| --- | --- | +| `music_dht::podcast` | ids, URL canonicalization, no-GUID rule, metadata DTOs and bounds, merge functions, RSS/Atom parser (feature `podcast-feed`) | +| `music_dht::device_sync` | sync operation kinds, snapshot sections, `PlaybackSource`, version gating, Jam redaction | +| Client storage (`furumi-library`, web PostgreSQL) | persistence and migrations implementing the rules below | +| Client runtime | HTTP fetching, conditional requests, scheduling, audio buffering and decoding, UI | + +Audio is always fetched from the publisher. It is never imported into the +music library, never content-addressed with a BLAKE3 content id and never +offered for P2P transfer. RSS refreshes fetch metadata only. + +## Data classes + +| Data | Trusted devices | Public federation | Local only | +| --- | --- | --- | --- | +| Subscription state, channel metadata, feed aliases | `podcast_subscription_set` + snapshot | never | | +| Share decision | `podcast_share_set` + snapshot | only the published card (future overlay) | | +| Episode progress | `podcast_progress_set` + snapshot | never | | +| Listening sessions, play counts | `podcast_listen_recorded` + snapshot | never | | +| Episode catalog | each device parses RSS itself; `PodcastEpisodeRef` travels with progress and queue items | never | | +| ETag/Last-Modified, last check time, fetch errors, artwork cache, audio buffer | | | yes | + +Synchronizing to a trusted device is not permission to publish. Private +podcasts, their URLs, descriptions and episodes must not reach DHT records, +public catalogs, public search results or Jam participants. + +## Identity + +### Feed URL canonicalization + +`FeedUrl::parse` produces the canonical form. Every client must use it (or +an implementation that passes the same tests) before deriving ids or +comparing URLs. + +1. Only `http://` and `https://` are accepted (case-insensitive) and the + scheme is lowercased. +2. Userinfo (`user@`, `user:pass@`), whitespace, control characters and `\` + are rejected. +3. The host is lowercased. Non-ASCII hosts are rejected; pass the IDNA + (`xn--`) form as produced by WHATWG URL parsers. Bracketed IPv6 literals + are accepted. +4. An empty port and the default port (80 for http, 443 for https) are + removed; other ports lose leading zeros. +5. The fragment is removed. +6. An empty path becomes `/`. The rest of the path and the **entire** query + string are kept byte-for-byte: parameter order, case, percent-encoding and + dot segments are preserved. Query parameters can be part of the source + identity (for example a per-user token) and must never be dropped. Only + non-ASCII characters are percent-encoded (UTF-8, uppercase hex). +7. The result is at most 2048 bytes. Canonicalization is idempotent; + deserialization accepts only canonical strings. + +| Input | Canonical | +| --- | --- | +| `HTTPS://Example.COM:443/Feed.xml?b=2&a=1#top` | `https://example.com/Feed.xml?b=2&a=1` | +| `http://example.com:80` | `http://example.com/` | +| `https://example.com/rss?token=XyZ&x=%2f` | unchanged | +| `https://user@example.com/rss` | rejected | + +`FeedUrl::from_user_input` is for typed or pasted input only: it trims, maps +`feed:`, `podcast:`, `pcast:`, `itpc:` to `https://` and adds `https://` when +the scheme is missing, then canonicalizes. + +### Podcast id + +```text +podcast_id = "pod1:" + base32_lower_nopad(BLAKE3("furumi-podcast:podcast:v1" || 0x00 || canonical_feed_url)) +``` + +The id is derived **once**, from the canonical URL of the first subscription. +It never changes afterwards. + +### Feed aliases + +Redirect targets (permanent redirects), `itunes:new-feed-url` and URLs used to +subscribe again are feed aliases of an existing podcast: + +* the subscription record carries `feed_aliases`, a grow-only set merged + independently of the last-writer-wins register (`merge_feed_aliases`). The + URL that derives the podcast id is always kept, plus the 8 smallest other + URLs; this bounded union is commutative, associative and idempotent; +* `podcast.feed_url` is the URL currently fetched and is always added to the + aliases; +* before deriving a new id for a URL, clients call `resolve_podcast_id` with + every known `(podcast_id, url)` pair. A match reuses the existing podcast, + its progress and its history. With several matches the smallest id wins. + +Limitation: two devices that subscribe offline, through different URLs of +the same feed, before either learns the alias, create two podcast ids. The +contract does not merge podcast ids automatically in this version. + +### Episode id + +```text +episode_id = "ep1:" + base32_lower_nopad(BLAKE3("furumi-podcast:episode:v1" || 0x00 || podcast_id || key)) +``` + +| Key | Material | +| --- | --- | +| GUID | `guid:` + RSS `` / Atom `` trimmed, case-sensitive | +| link | `link:` + item `` trimmed | +| date + title | `date-title:` + publication unix seconds + `\0` + normalized title | +| enclosure | `enclosure:` + enclosure URL trimmed | + +The normalized title is `music_dht::normalize_name` (NFKC, lowercase, +non-alphanumeric runs collapsed). GUIDs longer than 2048 bytes are treated as +absent. Changing the title, artwork, description or audio URL of an episode +never changes its id and never resets its progress. + +### No-GUID rule + +`resolve_episode_ids(podcast_id, channel_link, items, known)` is normative. +For one parsed feed document: + +1. Items with a GUID get the GUID id. A GUID never matches another known + episode, so the result does not depend on local history. +2. An item without GUID first searches the known episodes of the podcast, + stage by stage: same enclosure URL, then same ``, then same + publication second and normalized title. The first stage with candidates + wins and the smallest candidate id is reused. A known episode can be + claimed by only one item of the document. +3. Otherwise a new id is derived from the first available key: ``, + then date + title, then enclosure URL. + +A `` is identifying only if it is unique in the document and differs +from the channel `` (many feeds repeat the homepage in every item). +Enclosure URLs and date + title pairs shared by several GUID-less items are +not identifying either. Items without any usable key, and duplicates of an +earlier item of the same document, get no id and are skipped. + +"Known episodes" include episodes from earlier refreshes **and** episodes +learned through sync (`PodcastEpisodeRef::identity_hints`). Feed GUID-less +episodes into it so a device that sees an episode first through sync keeps +the id used by the other devices. + +Limitation: for GUID-less episodes the id can depend on what a device saw +first. If a publisher changes several identifying fields of a GUID-less +episode at once, a device that never saw the old version can derive a +different id. + +## Metadata and bounds + +| Field | Bound | +| --- | --- | +| titles, author | 1000 characters, non-empty titles | +| descriptions | 4000 characters (plain text) | +| URLs | 2048 bytes, audio and artwork URLs absolute `http(s)` | +| GUID | 2048 bytes | +| MIME | 128 printable ASCII bytes | +| device, session, operation ids | 1..256 bytes, no control characters | +| feed aliases | 8 plus the origin URL | + +`PodcastMeta` (channel), `EpisodeMeta` (episode, including season, episode +number, `itunes:episodeType`, publication time and duration in milliseconds, +audio URL, MIME, byte length, artwork and link) and `PodcastEpisodeRef` +(queue item) validate these bounds. `PodcastMeta::bounded` and +`EpisodeMeta::to_ref` truncate publisher data to fit. Receivers drop records +that fail validation. + +Episodes without a season are shown in their own group. Unknown +`itunes:episodeType` values are `full`. + +## Personal state and merge rules + +All merge functions are commutative, associative and idempotent per entity; +ties are broken by a total order over all fields. The same set of operations +therefore yields the same state in any delivery order, with duplicates. +`hlc_ms` is the sync operation timestamp; it is not a true hybrid logical +clock, so ordering of user intent on progress uses a Lamport generation. + +### Subscription + +* Register: `subscribed` + `podcast` metadata, last-writer-wins by + `(hlc_ms, op_id)`. +* Unsubscribing writes `subscribed = false`. It is state, not a tombstone, + and is never compacted, so it beats a stale subscription from a device that + returns after being offline. +* Unsubscribing stops monitoring but keeps progress and history. + Subscribing again continues where the user stopped. Erasing history is a + separate explicit action (not part of this version). +* Only explicit subscribe and unsubscribe actions write the register. RSS + refreshes keep their metadata locally; a background refresh that wrote the + register could resurrect an unsubscribed podcast. +* Subscribing to a podcast already subscribed locally must not emit a new + operation. + +### Share + +* `ShareRecord`, last-writer-wins by `(hlc_ms, op_id)`. `shared = false` is a + tombstone that revokes the publication. +* Unsubscribing also writes `shared = false`. +* `effective_share(subscription, share)` decides whether to publish: the user + is subscribed, chose to share, **and** the share decision is newer than the + latest subscription write. Every new subscription, including subscribing + again or subscribing from someone else's public card, is private until the + user shares it again, and a stale share decision cannot republish. + +### Progress + +`ProgressRecord { episode_id, podcast_id, position_ms, status, generation, +session_id, hlc_ms, device_id, op_id }`, with `status` one of `not_started`, +`in_progress`, `completed`. The winner is the greatest +`(generation, hlc_ms, device_id, op_id)`. Progress is never the maximum +position: seeking back and "start over" are real changes. + +* Explicit actions use `generation = max_seen + 1` + (`ProgressRecord::explicit_generation`): starting a listening session, + seeking, "start over", "mark played", "mark unplayed", handing playback to + another device. `max_seen` is the generation of the current register, + including received records. +* Periodic checkpoints and pause/stop saves within a session keep the + generation of that session's latest explicit action. +* Write progress periodically (every 15–30 s) and on pause, seek, stop, + episode change and handoff, in the background. Coalesce pending checkpoints + so the outgoing log keeps only the latest per episode. + +| Events | Result | +| --- | --- | +| A checkpoints session s1 at g3; B seeks back at g4 with a slower clock; A's late g3 checkpoint arrives | B's seek (g4) wins | +| A restarts (g3, position 0); a queued g2 checkpoint of the old session arrives later | restart wins | +| A and B both saw g5 and listen offline; both start sessions at g6 | the later `hlc_ms` (then `device_id`, `op_id`) wins on every device; both sessions are kept | + +Ordinary Play resumes from `resume_position_ms()` (the position when in +progress, otherwise 0). "Start over" is a separate explicit action. Publisher +metadata changes never touch progress; if the duration or audio changed, the +client keeps the stored position and lets the user decide. + +### Listening sessions and counts + +`ListenSession { session_id, episode_id, podcast_id, device_id, +started_at_ms, ended_at_ms, start_position_ms, end_position_ms, listened_ms, +completed }` forms a grow-only set keyed by `session_id` (generated at +playback start). A device may resend its session with more listening time; +versions merge to the greatest `(completed, listened_ms, ended_at_ms, ...)`. + +* A session is recorded in history when `completed` or + `listened_ms >= 30 000` (`should_record`). +* `completed` is set when the session reaches the end of the stream or + `reaches_completion(position_ms, duration_ms)`: within the smaller of 30 s + and 5 % of the duration from the end. Completing also writes progress with + status `completed` (explicit generation). +* The play count of an episode is the number of distinct completed sessions + (`completed_listen_count`). Each session counts at most once, so + re-delivery never double counts, and listening again is a new session. +* "Mark played" only writes progress; it does not create a session or invent + listening time. +* Concurrent offline sessions are all kept; there is no promise of a single + global session during a partition. + +## Trusted-device sync (protocol v4) + +The ALPN remains `furumi/sync/2`; device profiles advertise +`protocol_version = 4`. New operation kinds: + +| Kind | Payload | Materialization | Tombstone | +| --- | --- | --- | --- | +| `podcast_subscription_set` | `podcast_id`, `subscribed`, `podcast`, `feed_aliases` | `SyncOpWire::podcast_subscription` | no | +| `podcast_share_set` | `podcast_id`, `shared` | `SyncOpWire::podcast_share` | `shared = false` | +| `podcast_progress_set` | `episode_id`, `podcast_id`, `position_ms`, `status`, `generation`, `session_id`, optional `episode` ref | `SyncOpWire::podcast_progress` | no | +| `podcast_listen_recorded` | `session`, optional `episode` ref | `SyncOpWire::podcast_listen_session` | no | + +Register stamps come from the operation: `hlc_ms`, `op_id`, and for progress +`device_id = origin_device_id`. A session's `device_id` must equal the +operation origin. `SyncOpWire::validate_podcast` checks this and the bounds; +receivers skip invalid podcast operations but still acknowledge them, so the +log keeps moving. + +`DeviceProfileSet` gains an optional `protocol_version`. Each device reports +its own version. Receivers store +`profile_protocol_version(reported, previously_known)` and relays forward the +stored version of other devices in `DeviceProfileWire.protocol_version`. They +must never substitute their own constant: version gating would then send +podcast state to an old client. A device that never reported a version is +treated as version 1. + +### Snapshot recovery + +Sync vectors only grow, so operations hidden from a peer while it ran an +older version are never re-requested after it upgrades. Podcast state is +therefore part of `SyncSnapshot`: + +| Section | Content | Limit | +| --- | --- | --- | +| `podcast_subscriptions` | all subscription registers, including unsubscribed | 1000 | +| `podcast_shares` | all share registers, including `shared = false` | 1000 | +| `podcast_progress` | progress registers | 5000 | +| `podcast_listen_sessions` | most recent sessions | 2000 | + +`SyncSnapshot::bound_podcast_state` orders each section newest first, caps it +and fits all sections into a 4 MiB JSON budget in the priority order above, +leaving room for music state inside the 8 MiB line limit. Records that do not +fit still propagate through operations. Receivers ignore podcast sections +that exceed the counts (`podcast_state_within_limits`), validate each record +and merge it with the same functions as operations. A new or long-absent +device thus converges from one snapshot. + +Empty sections are not serialized, so snapshots without podcast state are +byte-identical to v3. + +## Queue and playback + +`PlaybackTrack.source: Option` makes the item source +explicit: absent or `music` for music, `podcast_episode` with a +`PodcastEpisodeRef`, or `private` for a Jam placeholder. Negative ids, +placeholder releases and URLs in `file_path` are not part of the contract. +`PlaybackTrack::podcast_episode` fills the legacy display fields (episode +title, podcast title as artist and release, `id = 0`, empty `file_path`, no +`content_id`); receivers read `source`. + +`PodcastEpisodeRef` is self-contained: podcast and episode ids, canonical +feed URL, titles, audio URL, MIME, duration, publication time, season, +episode number, artwork. A device without the subscription can display, +resume and play the item; if the audio URL is stale it refreshes the feed and +looks the episode up by id. Mixed queues keep order, position and identity of +every item. + +Handoff follows `PLAYBACK_PROTOCOL.md`: record position and play/pause +intent, transfer the queue with its sources, the receiver opens the HTTP +source and seeks, and ownership moves only through the coordination claim. + +`PlaybackStateWire.playback_rate: Option` is reserved. No client +implements variable speed yet; receivers treat a missing or unsupported value +as 1.0. When implemented, position extrapolation and handoff must use it. + +## Version gating for peers below v4 + +`SyncOpPayload` and `PlaybackSource` are closed serde enums. A v3 client that +receives an unknown `kind` fails to parse the **whole** message, not one +operation, which would stall synchronization. Additive `serde(default)` +fields alone do not make this compatible. Therefore: + +1. Never send podcast operations to a peer below v4 + (`op_allowed_for_peer`). +2. Never send podcast snapshot sections (`SyncSnapshot::for_peer`) or a + playback snapshot whose queue contains podcast items + (`playback_allowed_for_peer`) to a peer below v4. +3. Refuse a handoff to a device below v4 while the queue contains podcast + items and tell the user that the target must be updated. Never strip the + episodes to make the queue fit. +4. Reject `SetState`/`ActiveChanged` from a peer below v4 while the local + queue contains podcast items (`playback_command_allowed_from_peer`): its + state would silently drop them. +5. Music operations and music-only playback keep working with v1–v3 peers. + +## Jam privacy + +Jam participants are outside the trusted-device group. + +* Hosts send `jam::redact_jam_snapshot` (or `redact_for_jam` on the state): + every podcast item becomes a `private` placeholder titled "Private item" + without URL, feed, ids, titles or duration; music items, order and position + are kept. +* Commands containing real podcast episodes are rejected + (`jam_command_allowed`): participants cannot add episodes. +* Before applying a participant's state, hosts call + `restore_jam_private_items`: the k-th placeholder maps to the host's k-th + private item. States that drop, duplicate or inject private items are + rejected. Participants may still pause, seek and reorder music. + +## Feed parser + +With the `podcast-feed` feature, `music_dht::podcast::feed` parses RSS 2.0, +RSS 1.0 and Atom tolerantly: BOM and declared single-byte encodings, HTML +entities, CDATA, mismatched end tags, undeclared prefixes, depth and size +limits, lenient dates and durations, relative media URLs resolved against +the fetch URL. Items without an absolute `http(s)` audio URL are dropped. It +extracts `itunes:season`, `itunes:episode`, `itunes:episodeType`, +`guid isPermaLink` and `itunes:new-feed-url`, and produces identity hints for +`resolve_episode_ids`. It performs no I/O. + +## Versioning + +* `music-dht` 0.6.0 and `furumi-library` 0.3.0: new public types, enum + variants and struct fields are breaking for 0.x. +* Device-sync protocol 4, ALPN unchanged. Podcast payloads are JSON; no + postcard layout changes. +* Id domains `furumi-podcast:podcast:v1` and `furumi-podcast:episode:v1` and + the `pod1:`/`ep1:` prefixes are frozen. A different derivation needs new + domains, new prefixes, a migration and a new protocol version. +* Adding an operation kind or a `PlaybackSource` variant requires a new + protocol version and gating as above. +* Public podcast cards will use a separate signed overlay with its own ALPN + and version; the music DHT schema and `federation-net` do not change. diff --git a/README.md b/README.md index abb2d88..f443ffa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The workspace has three layers: - [`music-dht`](crates/music-dht) builds a Kademlia-style distributed music directory on top of that transport, including catalog discovery, content-id lookup, signed similarity-LSH routing, direct byte streams, and shared Furumi - wire types. + wire types, including the shared podcast contract ([PODCASTS.md](PODCASTS.md)). - [`furumi-library`](crates/furumi-library) owns the shared local `library.db` contract, migrations, catalog operations, metadata import, artwork, playlists, likes, history, and similarity persistence used by Furumi clients. diff --git a/crates/furumi-library/Cargo.toml b/crates/furumi-library/Cargo.toml index 44fbcf2..2c44a11 100644 --- a/crates/furumi-library/Cargo.toml +++ b/crates/furumi-library/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "furumi-library" -version = "0.2.0" +version = "0.3.0" description = "Client-independent SQLite music library for Furumi players" readme = "README.md" documentation = "https://docs.rs/furumi-library" diff --git a/crates/furumi-library/README.md b/crates/furumi-library/README.md index 4fa5454..f2aac84 100644 --- a/crates/furumi-library/README.md +++ b/crates/furumi-library/README.md @@ -19,3 +19,7 @@ publication, clients can depend on the released crate version instead. `Library::open(default_db_path()?)` opens the same platform-native Furumi library used by every client. Connections use WAL, foreign keys, and a busy timeout so independently running clients can safely share the database. + +Version 0.3 follows `music-dht` 0.6, whose device-sync and podcast contract +types it re-exposes. Podcast persistence is not part of this release; it will +implement the rules of the workspace `PODCASTS.md`. diff --git a/crates/music-dht/Cargo.toml b/crates/music-dht/Cargo.toml index cec4116..86becc2 100644 --- a/crates/music-dht/Cargo.toml +++ b/crates/music-dht/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "music-dht" -version = "0.5.0" +version = "0.6.0" description = "Distributed music library search: a Kademlia-style DHT on top of federation-net" readme = "README.md" documentation = "https://docs.rs/music-dht" diff --git a/crates/music-dht/README.md b/crates/music-dht/README.md index f72586c..d807ffa 100644 --- a/crates/music-dht/README.md +++ b/crates/music-dht/README.md @@ -185,7 +185,9 @@ new clients should accept both ALPNs and only send `ListenRecorded` when `supports_listen_history(peer.protocol_version)` returns true. Likes, playlists and membership remain compatible with v1. Profile version 3 adds versioned personal playback coordination over the existing JSON sync stream; -legacy unfenced playback messages are ignored by upgraded clients. +legacy unfenced playback messages are ignored by upgraded clients. Profile +version 4 adds podcasts (see below); `op_allowed_for_peer` applies all of +these rules to one operation. `music_dht::playback` provides the shared pure ownership engine and generic snapshot/command envelopes. Configure startup and failover with `Config`, feed @@ -193,12 +195,64 @@ actual output reports, and persist `Checkpoint` before applying decisions. `Config::passive()` is intended for always-running gateways. See the workspace `PLAYBACK_PROTOCOL.md` for invariants, configuration and migration details. +## Podcasts + +`music_dht::podcast` is the shared, pure podcast contract (no SQLite, HTTP or +clock). Clients use it for identity, merge rules and wire DTOs; see the +workspace `PODCASTS.md` for the normative rules. + +```rust +use music_dht::podcast::{ + EpisodeIdentityHints, FeedUrl, PodcastId, ProgressRecord, resolve_episode_ids, +}; + +let url = FeedUrl::from_user_input("feed://example.com/rss?token=abc")?; +let podcast = PodcastId::from_feed_url(&url); // "pod1:..." +let hints = [EpisodeIdentityHints { + guid: Some("episode-1".into()), + ..Default::default() +}]; +let ids = resolve_episode_ids(&podcast, None, &hints, &[]); // "ep1:..." + +// Explicit actions (seek, start over, new session) bump the generation. +let generation = ProgressRecord::explicit_generation(Some(4)); // 5 +``` + +* `FeedUrl` canonicalizes feed URLs and keeps every query parameter. +* `PodcastId`, `EpisodeId`, `EpisodeKey`, `resolve_podcast_id`, + `merge_feed_aliases`, `resolve_episode_ids`: identity, aliases and the + no-GUID rule. +* `PodcastMeta`, `EpisodeMeta`, `PodcastEpisodeRef`: bounded metadata and the + self-contained queue item. +* `SubscriptionRecord`, `ShareRecord`, `ProgressRecord`, `ListenSession` with + `merge_subscription`, `merge_share`, `merge_progress`, + `merge_listen_session`, `effective_share`, `reaches_completion` and + `completed_listen_count`. + +Enable the `podcast-feed` feature for `music_dht::podcast::feed`, the shared +tolerant RSS 2.0 / RSS 1.0 / Atom parser: + +```toml +music-dht = { version = "0.6", features = ["podcast-feed"] } +``` + +Device-sync protocol v4 adds `podcast_subscription_set`, `podcast_share_set`, +`podcast_progress_set` and `podcast_listen_recorded`, podcast snapshot +sections, `PlaybackTrack.source` and a reserved `playback_rate`. Old clients +cannot parse these, so always filter with `op_allowed_for_peer`, +`playback_allowed_for_peer`, `SyncSnapshot::for_peer` and +`playback_command_allowed_from_peer`, using the protocol version each device +reported itself (`profile_protocol_version`). Jam hosts use +`jam::redact_jam_snapshot`, `jam_command_allowed` and +`restore_jam_private_items`. + ## Verification ```bash cargo fmt --check cargo clippy --workspace --all-targets --all-features -- -D warnings cargo test -p music-dht +cargo test -p music-dht --features podcast-feed ``` The integration test starts three real nodes in one process (they use Iroh's diff --git a/crates/music-dht/src/lib.rs b/crates/music-dht/src/lib.rs index 669e751..30447d8 100644 --- a/crates/music-dht/src/lib.rs +++ b/crates/music-dht/src/lib.rs @@ -23,6 +23,10 @@ //! * [`similarity_lsh`] turns those embeddings into deterministic compact //! routing summaries, while [`similarity_dht`] stores and discovers signed //! peer summaries without owning model inference or an embedding database. +//! * [`podcast`] is the pure, shared podcast contract (canonical feed URLs, +//! podcast/episode ids, metadata DTOs and convergent merge rules), with an +//! optional RSS/Atom parser behind the `podcast-feed` feature; +//! [`device_sync`] protocol v4 carries it between trusted devices. //! * Records are replicated to the `K` nodes whose ids are XOR-closest to //! each key. Publishers pick the targets from their routing table and send //! batched store requests (one pipeline per peer), so even a large library -- 2.54.0 From 629857dbfe8ff3d06c903490ff46c2b58dfe47bb Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:10:33 +0200 Subject: [PATCH 5/7] Make podcast episode identity order-independent and linear Run the no-GUID matching stages across the whole document so a weak date+title match of an earlier item cannot take a known episode that a later item matches by enclosure. Keep feed links verbatim (trimmed only) because they are identity material, and stop html_to_text from scanning to the end of the text at every '<' when no '>' follows. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/podcast/feed.rs | 42 ++++-- crates/music-dht/src/podcast/feed_tests.rs | 66 +++++++++ crates/music-dht/src/podcast/identity.rs | 160 ++++++++++++++++----- 3 files changed, 219 insertions(+), 49 deletions(-) diff --git a/crates/music-dht/src/podcast/feed.rs b/crates/music-dht/src/podcast/feed.rs index 8da47b8..f53665f 100644 --- a/crates/music-dht/src/podcast/feed.rs +++ b/crates/music-dht/src/podcast/feed.rs @@ -61,7 +61,8 @@ pub struct ParsedFeed { pub author: Option, /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. pub description: Option, - /// Publisher web page. + /// Publisher web page: `` text or Atom `href`, trimmed, verbatim + /// (identity material, see [`resolve_episode_ids`]). pub link: Option, /// Absolute http(s) artwork URL. pub image_url: Option, @@ -101,7 +102,8 @@ pub struct ParsedEpisode { pub mime_type: Option, /// Enclosure length in bytes. pub byte_len: Option, - /// Item web page. + /// Item web page: `` text or Atom `href`, trimmed, verbatim + /// (identity material, see [`resolve_episode_ids`]). pub link: Option, /// Absolute http(s) artwork URL. pub image_url: Option, @@ -662,7 +664,7 @@ impl Parser { let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); if rel.is_empty() || rel == "alternate" { let href = attrs.get("href").unwrap_or_default(); - set_once(&mut self.feed.link, clean_short(&href)); + set_once(&mut self.feed.link, href.trim().to_string()); } None } @@ -770,7 +772,7 @@ impl Parser { }); } } else if rel.is_empty() || rel == "alternate" { - set_once(&mut item.link, clean_short(&href)); + set_once(&mut item.link, href.trim().to_string()); } None } @@ -970,11 +972,7 @@ impl Parser { audio_url: media.url, mime_type: media.mime, byte_len: media.len, - link: item - .link - .as_deref() - .map(clean_short) - .filter(|s| !s.is_empty()), + link: item.link.as_deref().and_then(raw_link), image_url: [item.itunes_image, item.thumbnail] .into_iter() .flatten() @@ -1016,11 +1014,7 @@ impl Parser { title, author, description, - link: feed - .link - .as_deref() - .map(clean_short) - .filter(|s| !s.is_empty()), + link: feed.link.as_deref().and_then(raw_link), image_url, new_feed_url: feed .new_feed_url @@ -1199,6 +1193,17 @@ fn parse_number(text: &str) -> Option { // --------------------------------------------------------------------------- // Text cleanup +/// `` text (already XML-unescaped) or Atom `href`, trimmed. +/// +/// Links are identity material for GUID-less episodes (`EpisodeKey::Link`), +/// so they are deliberately **not** passed through the display text cleaner: +/// no tag stripping, second entity decoding, whitespace collapsing or +/// truncation. Links that are empty or longer than `MAX_URL_BYTES` are absent. +fn raw_link(text: &str) -> Option { + let link = text.trim(); + (!link.is_empty() && link.len() <= super::MAX_URL_BYTES).then(|| link.to_string()) +} + fn clean_short(text: &str) -> String { html_to_text(text, MAX_TITLE_CHARS) } @@ -1248,10 +1253,17 @@ pub(crate) fn html_to_text(input: &str, max_chars: usize) -> String { .chars() .next() .is_some_and(|c| c.is_ascii_alphabetic() || matches!(c, '/' | '!' | '?')); - let Some(gt) = after.find('>').filter(|_| looks_like_tag) else { + if !looks_like_tag { stripped.push('<'); rest = after; continue; + } + let Some(gt) = after.find('>') else { + // No '>' anywhere after this point: nothing else can be a tag. + // Stopping here keeps the scan linear on text full of '<'. + stripped.push('<'); + rest = after; + break; }; let tag = &after[..gt]; rest = &after[gt + 1..]; diff --git a/crates/music-dht/src/podcast/feed_tests.rs b/crates/music-dht/src/podcast/feed_tests.rs index 80d027b..b9dfe30 100644 --- a/crates/music-dht/src/podcast/feed_tests.rs +++ b/crates/music-dht/src/podcast/feed_tests.rs @@ -663,3 +663,69 @@ fn hostile_sizes_stay_bounded() { None ); } + +#[test] +fn text_full_of_unclosed_angle_brackets_is_linear() { + // '' anywhere: the old + // scanner searched for '>' to the end of the text at every '<'. + let body = "<a".repeat(MAX_CAPTURE_BYTES / 2); + let xml = rss( + "", + &format!( + r#"lt{body}{body}"# + ), + ); + let started = std::time::Instant::now(); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "took {:?}", + started.elapsed() + ); + let episode = &feed.episodes[0]; + assert!(episode.title.starts_with("bold A https://e.test/a?x=1&amp;y=<b>
+Bhttps://e.test/{long_path}"# + ); + let feed = parse_feed(rss("", &items).as_bytes()).unwrap(); + assert_eq!( + feed.episodes[0].link.as_deref(), + Some("https://e.test/a?x=1&y=") + ); + let long = format!("https://e.test/{long_path}"); + assert_eq!(feed.episodes[1].link.as_deref(), Some(long.as_str())); + let podcast_id = + PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed").unwrap()); + let ids = feed.resolve_episode_ids(&podcast_id, &[]); + assert_eq!( + ids[1], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::link(&long).unwrap() + )) + ); + let atom = r#"T + +E +"#; + let feed = parse_feed(atom.as_bytes()).unwrap(); + assert_eq!(feed.link.as_deref(), Some("https://e.test/home?a=1&b=2")); + assert_eq!( + feed.episodes[0].link.as_deref(), + Some("https://e.test/e?q=") + ); +} diff --git a/crates/music-dht/src/podcast/identity.rs b/crates/music-dht/src/podcast/identity.rs index af91112..94e83e3 100644 --- a/crates/music-dht/src/podcast/identity.rs +++ b/crates/music-dht/src/podcast/identity.rs @@ -57,6 +57,9 @@ impl EpisodeIdentityHints { } } +/// One matching stage of the no-GUID rule: does `known` match `item`? +type StageMatcher<'f> = &'f dyn Fn(&Usable<'_>, &EpisodeIdentityHints) -> bool; + struct Usable<'a> { link: Option<&'a str>, enclosure: Option<&'a str>, @@ -103,11 +106,13 @@ pub fn resolve_episode_id( /// /// 1. An item with a GUID gets `EpisodeId::from_guid`. A GUID never matches /// a different known episode, so the id is independent of local history. -/// 2. An item without a GUID first looks for a known episode, in this order: -/// same enclosure URL, then same ``, then same publication second and -/// [`normalize_episode_title`]. The first stage with candidates wins; among -/// candidates the smallest id is reused. A known episode already claimed by -/// another item of this document is not a candidate. +/// 2. Items without a GUID then look for known episodes in three stages run +/// over the whole document: same enclosure URL, then same ``, then +/// same publication second and [`normalize_episode_title`]. A stage +/// assigns matches for all remaining items before the next stage starts; +/// among candidates the smallest id is reused. A known episode already +/// claimed (by a GUID item or an earlier stage) is not a candidate, so the +/// result does not depend on item order. /// 3. Otherwise a new id is derived from the first available key: ``, /// then `(date, title)`, then enclosure URL. /// @@ -166,40 +171,47 @@ pub fn resolve_episode_ids( } } - // Pass 2: GUID-less items reuse a matching known episode. - let mut pending = Vec::new(); - for (index, item) in items.iter().enumerate() { - if item.guid_key().is_some() { - continue; - } - let own = &usable[index]; - let stages: [&dyn Fn(&EpisodeIdentityHints) -> bool; 3] = [ - &|candidate| { - own.enclosure - .is_some_and(|url| candidate.enclosure() == Some(url)) - }, - &|candidate| own.link.is_some_and(|link| candidate.link() == Some(link)), - &|candidate| { - own.date_title - .as_ref() - .is_some_and(|pair| candidate.date_title().as_ref() == Some(pair)) - }, - ]; - let reused = stages.iter().find_map(|matches| { - known + // Pass 2: GUID-less items reuse a matching known episode. Stages run + // across the whole document: every enclosure match is assigned before any + // link match, and every link match before any (date, title) match, so a + // weak match of an earlier item cannot steal the known episode that a + // later item matches strongly. Within one stage an item's key is unique in + // the document, so two items never compete for the same known episode + // and the result does not depend on item order. + let mut pending: Vec = (0..items.len()) + .filter(|index| items[*index].guid_key().is_none()) + .collect(); + let stages: [StageMatcher<'_>; 3] = [ + &|own, candidate| { + own.enclosure + .is_some_and(|url| candidate.enclosure() == Some(url)) + }, + &|own, candidate| own.link.is_some_and(|link| candidate.link() == Some(link)), + &|own, candidate| { + own.date_title + .as_ref() + .is_some_and(|pair| candidate.date_title().as_ref() == Some(pair)) + }, + ]; + for matches in stages { + let mut unmatched = Vec::with_capacity(pending.len()); + for index in pending { + let own = &usable[index]; + let reused = known .iter() - .filter(|known| !claimed.contains(&known.episode_id) && matches(&known.hints)) + .filter(|known| !claimed.contains(&known.episode_id) && matches(own, &known.hints)) .map(|known| &known.episode_id) .min() - .cloned() - }); - match reused { - Some(id) => { - claimed.insert(id.clone()); - out[index] = Some(id); + .cloned(); + match reused { + Some(id) => { + claimed.insert(id.clone()); + out[index] = Some(id); + } + None => unmatched.push(index), } - None => pending.push(index), } + pending = unmatched; } // Pass 3: derive new ids from the first available fallback key. @@ -509,4 +521,84 @@ mod tests { assert_eq!(resolve_episode_ids(&p, None, &items, &known), forward); assert!(forward.iter().all(Option::is_some)); } + + #[test] + fn strong_matches_beat_earlier_weak_matches_in_any_item_order() { + let p = podcast(); + let k1 = EpisodeId::from_guid(&p, "legacy-k1").unwrap(); + let known = [KnownEpisode { + episode_id: k1.clone(), + hints: hints( + None, + Some("https://cdn/e1.mp3"), + None, + Some(5_000), + Some("Same"), + ), + }]; + // A new item that only matches K1 by (date, title), and the real K1 + // (same enclosure) renamed and re-dated by the publisher. + let weak = hints( + None, + Some("https://cdn/new.mp3"), + None, + Some(5_000), + Some("Same"), + ); + let strong = hints( + None, + Some("https://cdn/e1.mp3"), + None, + Some(9_000), + Some("Renamed"), + ); + let forward = resolve_episode_ids(&p, None, &[weak.clone(), strong.clone()], &known); + assert_eq!(forward[1].as_ref(), Some(&k1), "enclosure match keeps K1"); + assert_ne!(forward[0].as_ref(), Some(&k1)); + let reverse = resolve_episode_ids(&p, None, &[strong, weak], &known); + assert_eq!(reverse, vec![forward[1].clone(), forward[0].clone()]); + } + + #[test] + fn result_is_independent_of_item_order() { + let p = podcast(); + let items: Vec = (0..5) + .map(|i| { + hints( + None, + Some(&format!("https://cdn/{i}.mp3")), + (i % 2 == 0).then(|| format!("https://e/{i}")).as_deref(), + Some((i % 3) * 1_000), + Some(&format!("Ep {}", i % 2)), + ) + }) + .collect(); + // Known episodes whose hints overlap different items at different + // stages. + let known: Vec = (0..5) + .map(|i| KnownEpisode { + episode_id: EpisodeId::from_guid(&p, &format!("legacy-{i}")).unwrap(), + hints: hints( + None, + Some(&format!("https://cdn/{}.mp3", (i + 2) % 7)), + Some(&format!("https://e/{}", (i + 1) % 5)), + Some((i % 3) * 1_000), + Some(&format!("Ep {}", i % 2)), + ), + }) + .collect(); + let reference = resolve_episode_ids(&p, None, &items, &known); + let mut order: Vec = (0..items.len()).collect(); + for _ in 0..20 { + order.rotate_left(1); + let middle = order.len() / 2; + order.swap(0, middle); + let shuffled: Vec = + order.iter().map(|i| items[*i].clone()).collect(); + let result = resolve_episode_ids(&p, None, &shuffled, &known); + for (position, index) in order.iter().enumerate() { + assert_eq!(result[position], reference[*index], "order {order:?}"); + } + } + } } -- 2.54.0 From 085bff6191b7eae96eacfa2d39fa775451c02711 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:10:33 +0200 Subject: [PATCH 6/7] Harden podcast sync registers, versions and snapshots - Order subscription and share registers by a per-podcast Lamport generation before hlc_ms (next_podcast_generation), so clock skew cannot undo an observed unsubscribe or share decision. - share=false is register state, not a compactable tombstone. - Add podcast_feed_aliases_added so refreshes can sync aliases without touching the registers; SubscriptionRecord::feed_urls keeps the current feed URL resolvable beyond the alias bound. - Carry progress_generation on queue items for handoff. - Distinguish self-reported and relayed peer versions, and gate PairRequest by an optional inviter version in InviteWire. - Give each podcast snapshot section its own byte budget, drop channel descriptions from snapshots and allow a caller-provided budget. - Reject one-slash http(s) typos in FeedUrl::from_user_input. - Pin that malformed ids and feed URLs reject the whole message. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/device_sync.rs | 469 +++++++++++++++++--- crates/music-dht/src/jam.rs | 1 + crates/music-dht/src/podcast/feed_url.rs | 29 +- crates/music-dht/src/podcast/meta.rs | 17 +- crates/music-dht/src/podcast/mod.rs | 15 +- crates/music-dht/src/podcast/state.rs | 133 ++++-- crates/music-dht/src/podcast/state_tests.rs | 133 +++++- 7 files changed, 699 insertions(+), 98 deletions(-) diff --git a/crates/music-dht/src/device_sync.rs b/crates/music-dht/src/device_sync.rs index 679abff..12769d4 100644 --- a/crates/music-dht/src/device_sync.rs +++ b/crates/music-dht/src/device_sync.rs @@ -11,13 +11,16 @@ //! | 1 | likes, playlists, membership (`furumi/sync/1`) | //! | 2 | [`SyncOpPayload::ListenRecorded`] (`furumi/sync/2`) | //! | 3 | fenced personal playback coordination | -//! | 4 | podcasts: four `podcast_*` operations, podcast snapshot sections, [`PlaybackSource`] on queue items, reserved `playback_rate` | +//! | 4 | podcasts: five `podcast_*` operations, podcast snapshot sections, [`PlaybackSource`] on queue items, reserved `playback_rate` | //! -//! The ALPN stays `furumi/sync/2` for v3 and v4. [`SyncOpPayload`] and -//! [`PlaybackSource`] are closed serde enums: a peer that does not know a -//! `kind` fails to deserialize the **whole** message that contains it, not -//! just one operation. Senders must therefore filter by the peer's version -//! with [`op_allowed_for_peer`] and [`playback_allowed_for_peer`]. See the +//! The ALPN stays `furumi/sync/2` for v3 and v4. [`SyncOpPayload`] is a +//! closed serde enum: a peer that does not know an operation `kind` fails to +//! deserialize the **whole** message that contains it, not just one +//! operation. Peers before v4 ignore [`PlaybackTrack::source`] and would treat +//! episodes as broken music items. Senders must therefore filter by the +//! peer's version ([`profile_protocol_version`], [`relayed_protocol_version`], +//! [`InviteWire::inviter_protocol_version`]) with [`op_allowed_for_peer`], +//! [`playback_allowed_for_peer`] and [`SyncSnapshot::for_peer`]. See the //! workspace `PODCASTS.md` for the podcast contract. use std::collections::BTreeMap; @@ -29,10 +32,10 @@ use tokio::io::{AsyncRead, AsyncReadExt}; use crate::error::{MusicDhtError, Result}; use crate::podcast::{ - EpisodeId, FeedUrl, ListenSession, MAX_PODCAST_SNAPSHOT_BYTES, MAX_SNAPSHOT_LISTEN_SESSIONS, - MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, MAX_SNAPSHOT_SUBSCRIPTIONS, PodcastEpisodeRef, - PodcastError, PodcastId, PodcastMeta, PodcastResult, ProgressRecord, ProgressStatus, - ShareRecord, SubscriptionRecord, + EpisodeId, FeedUrl, ListenSession, MAX_FEED_ALIASES, MAX_PODCAST_SNAPSHOT_BYTES, + MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, + MAX_SNAPSHOT_SUBSCRIPTIONS, PodcastEpisodeRef, PodcastError, PodcastId, PodcastMeta, + PodcastResult, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, }; /// Legacy ALPN used by personal-device sync v1. @@ -83,11 +86,20 @@ pub const fn supports_podcasts(protocol_version: u16) -> bool { protocol_version >= PODCAST_PROTOCOL_VERSION } -/// Protocol version to store for a device after reading one of its profiles. +/// Protocol version to store for a device after reading **its own** report. /// -/// `reported` is the version the device itself put into its -/// [`DeviceProfileWire`] (Hello/pairing) or [`SyncOpPayload::DeviceProfileSet`]; -/// `previously_known` is the stored value. Receivers and relays must never +/// Self-reports are: the `profile` of a `Hello`, `PairRequest` or +/// `PairResponse` sent by that device; the entry of a `devices` list whose +/// `device_id` is the authenticated sender of the message (a `SyncResponse` +/// has no `profile` field, so this is how an initiator learns the responder's +/// version); and a [`SyncOpPayload::DeviceProfileSet`] whose +/// `origin_device_id` is that device. `reported` is the version in the report +/// (`None` when an older client left it out) and `previously_known` the +/// stored value. A self-report replaces the stored value, so a downgrade is +/// noticed. +/// +/// Profiles of **other** devices relayed by a peer are not self-reports: use +/// [`relayed_protocol_version`] for them. Receivers and relays must never /// substitute their own [`DEVICE_SYNC_PROTOCOL_VERSION`] for another device: /// doing so would make version gating send podcast state to old clients. pub fn profile_protocol_version(reported: Option, previously_known: Option) -> u16 { @@ -96,6 +108,18 @@ pub fn profile_protocol_version(reported: Option, previously_known: Option< .unwrap_or(UNREPORTED_PROTOCOL_VERSION) } +/// Protocol version to store for a device after reading a profile of it +/// relayed by **another** device (an entry of a `devices` list). +/// +/// Deployed v1–v3 clients relay every device with their own protocol +/// constant, so a relayed value can understate a device's version. A relayed +/// value therefore only raises the stored version (`max(stored, relayed)`) +/// and is taken as is only when nothing is stored. Lowering requires a +/// self-report ([`profile_protocol_version`]). +pub fn relayed_protocol_version(relayed: u16, previously_known: Option) -> u16 { + previously_known.map_or(relayed, |stored| stored.max(relayed)) +} + /// Default invite lifetime used by applications unless they need a custom TTL. pub const DEFAULT_INVITE_TTL_MS: i64 = 10 * 60 * 1000; @@ -122,6 +146,25 @@ pub struct InviteWire { /// Expiration timestamp in unix milliseconds. #[serde(rename = "e")] pub expires_at_ms: i64, + /// Device-sync protocol version of the inviter (v4+). + /// + /// `None` in invites created by older clients. The invite consumer sends + /// its `PairRequest` before it hears from the inviter, so it gates the + /// operations, snapshot sections and playback state of that request by + /// [`InviteWire::inviter_protocol_version`]. + #[serde(rename = "p", default, skip_serializing_if = "Option::is_none")] + pub protocol_version: Option, +} + +impl InviteWire { + /// Protocol version to assume for the inviter while building a + /// `PairRequest`: the version in the invite, or + /// [`UNREPORTED_PROTOCOL_VERSION`] for invites from older clients, which + /// strips all podcast state from the request. Podcast state then catches + /// up through the snapshot of the next regular sync. + pub fn inviter_protocol_version(&self) -> u16 { + self.protocol_version.unwrap_or(UNREPORTED_PROTOCOL_VERSION) + } } /// Public device profile exchanged inside a trusted sync group. @@ -303,6 +346,13 @@ impl ListenEvent { /// Absent on music items from older clients, which are music. Podcast /// episodes are identified only by this field: never by a negative id, a /// placeholder release or a URL in `file_path`. +/// +/// Clients before v4 do not know [`PlaybackTrack::source`] at all and ignore +/// it: they would take an episode for a broken music item (id 0, no content +/// id) and forward it that way, losing the episode for every device. That is +/// why podcast-bearing playback state is gated by version. This enum is +/// closed for v4 clients: a new variant needs a protocol bump, because a v4 +/// client fails to parse the whole message on an unknown `kind`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlaybackSource { @@ -621,9 +671,10 @@ pub enum SyncOpPayload { }, /// Subscribe to or unsubscribe from a podcast (protocol v4). /// - /// Last-writer-wins by `(hlc_ms, op_id)` of the operation. Unsubscribing - /// is state, not a tombstone. Only explicit user actions emit it; RSS - /// refreshes never do. Materialize with [`SyncOpWire::podcast_subscription`]. + /// Last-writer-wins by `(generation, hlc_ms, op_id)`; see + /// [`SubscriptionRecord`]. Unsubscribing is state, not a tombstone. Only + /// explicit user actions emit it; RSS refreshes never do. Materialize with + /// [`SyncOpWire::podcast_subscription`]. PodcastSubscriptionSet { /// Podcast. podcast_id: PodcastId, @@ -634,16 +685,37 @@ pub enum SyncOpPayload { /// Known feed URLs (grow-only, bounded). #[serde(default)] feed_aliases: Vec, + /// Per-podcast Lamport generation + /// ([`crate::podcast::next_podcast_generation`]). + #[serde(default)] + generation: u64, }, /// Share or stop sharing a podcast card with the federation (protocol v4). /// - /// Last-writer-wins; `shared = false` is a tombstone. Unsubscribing also - /// emits `shared = false`. + /// Last-writer-wins by `(generation, hlc_ms, op_id)`; see [`ShareRecord`]. + /// `shared = false` is privacy-protecting state and **not** a tombstone: + /// it is never compacted. Unsubscribing also emits `shared = false`. PodcastShareSet { /// Podcast. podcast_id: PodcastId, /// Desired share state. shared: bool, + /// Per-podcast Lamport generation, shared with the subscription. + #[serde(default)] + generation: u64, + }, + /// Record feed URLs of a podcast learned by any device (protocol v4). + /// + /// Grow-only: merged into the bounded alias set with + /// [`crate::podcast::merge_feed_aliases`]; it never touches the + /// subscription or share register, so RSS refreshes may emit it (after a + /// permanent redirect or an `itunes:new-feed-url`), but only when merging + /// the new URLs changes the stored set. + PodcastFeedAliasesAdded { + /// Podcast. + podcast_id: PodcastId, + /// Canonical feed URLs to add (at most `MAX_FEED_ALIASES + 1`). + feed_aliases: Vec, }, /// Write the durable resume register of an episode (protocol v4). /// @@ -684,9 +756,12 @@ pub enum SyncOpPayload { impl SyncOpPayload { /// Returns true for operations that remove state and may eventually compact. /// - /// `podcast_share_set { shared: false }` is a tombstone. Unsubscribing - /// (`podcast_subscription_set { subscribed: false }`) is **not**: it is - /// last-writer-wins state that must keep beating stale subscriptions. + /// No podcast operation is a tombstone. Unsubscribing + /// (`podcast_subscription_set { subscribed: false }`) and revoking a share + /// (`podcast_share_set { shared: false }`) are last-writer-wins state that + /// must keep beating stale `true` writes delivered again later; compacting + /// them (and deleting the state row, as like tombstones do) would let an + /// older operation still in some log resubscribe or republish the podcast. pub fn is_tombstone(&self) -> bool { matches!( self, @@ -694,7 +769,6 @@ impl SyncOpPayload { | SyncOpPayload::PlaylistDeleted { .. } | SyncOpPayload::PlaylistTrackRemoved { .. } | SyncOpPayload::DeviceRevoked { .. } - | SyncOpPayload::PodcastShareSet { shared: false, .. } ) } @@ -704,6 +778,7 @@ impl SyncOpPayload { self, SyncOpPayload::PodcastSubscriptionSet { .. } | SyncOpPayload::PodcastShareSet { .. } + | SyncOpPayload::PodcastFeedAliasesAdded { .. } | SyncOpPayload::PodcastProgressSet { .. } | SyncOpPayload::PodcastListenRecorded { .. } ) @@ -713,13 +788,15 @@ impl SyncOpPayload { impl SyncOpWire { /// Subscription record written by a `podcast_subscription_set` operation. /// - /// The register stamp is the operation's `(hlc_ms, op_id)`. + /// The register stamp is the payload `generation` and the operation's + /// `(hlc_ms, op_id)`. pub fn podcast_subscription(&self) -> Option { let SyncOpPayload::PodcastSubscriptionSet { podcast_id, subscribed, podcast, feed_aliases, + generation, } = &self.payload else { return None; @@ -729,6 +806,7 @@ impl SyncOpWire { subscribed: *subscribed, podcast: podcast.clone(), feed_aliases: feed_aliases.clone(), + generation: *generation, hlc_ms: self.hlc_ms, op_id: self.op_id.clone(), }; @@ -738,17 +816,37 @@ impl SyncOpWire { /// Share record written by a `podcast_share_set` operation. pub fn podcast_share(&self) -> Option { - let SyncOpPayload::PodcastShareSet { podcast_id, shared } = &self.payload else { + let SyncOpPayload::PodcastShareSet { + podcast_id, + shared, + generation, + } = &self.payload + else { return None; }; Some(ShareRecord { podcast_id: podcast_id.clone(), shared: *shared, + generation: *generation, hlc_ms: self.hlc_ms, op_id: self.op_id.clone(), }) } + /// Podcast and feed URLs of a `podcast_feed_aliases_added` operation. + /// + /// Merge them with [`crate::podcast::merge_feed_aliases`] (or + /// [`SubscriptionRecord::add_feed_aliases`]); the registers are untouched. + pub fn podcast_feed_aliases(&self) -> Option<(&PodcastId, &[FeedUrl])> { + match &self.payload { + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id, + feed_aliases, + } => Some((podcast_id, feed_aliases)), + _ => None, + } + } + /// Progress register written by a `podcast_progress_set` operation. /// /// `device_id` is the operation origin. @@ -789,7 +887,16 @@ impl SyncOpWire { /// Validates a podcast operation against the contract. /// /// Non-podcast operations are accepted unchanged. Receivers must skip - /// (but still acknowledge) invalid podcast operations. + /// (but still acknowledge) podcast operations that fail this check + /// (bounds, stamps, mismatched references). + /// + /// Ids ([`PodcastId`], [`EpisodeId`]) and feed URLs ([`FeedUrl`]) are + /// checked earlier, while the message is deserialized: a non-canonical id + /// or URL is a schema violation like a wrong JSON type, and the whole + /// message fails to parse. Conforming clients cannot produce one, because + /// these types only serialize canonical values and their rules are frozen + /// with the `v1` derivation domains; changing them requires a new protocol + /// version. pub fn validate_podcast(&self) -> PodcastResult<()> { let check_ref = |episode: &Option, episode_id: &EpisodeId, @@ -812,6 +919,15 @@ impl SyncOpWire { SyncOpPayload::PodcastShareSet { .. } => { self.podcast_share().expect("share payload").validate() } + SyncOpPayload::PodcastFeedAliasesAdded { feed_aliases, .. } => { + if feed_aliases.is_empty() || feed_aliases.len() > MAX_FEED_ALIASES + 1 { + return Err(PodcastError::InvalidField { + field: "feed_aliases", + reason: "must hold 1 to MAX_FEED_ALIASES + 1 URLs", + }); + } + Ok(()) + } SyncOpPayload::PodcastProgressSet { episode_id, podcast_id, @@ -960,7 +1076,8 @@ pub fn restore_jam_private_items( /// Podcast state (protocol v4) **is** included, bounded by /// [`SyncSnapshot::bound_podcast_state`]. Operations hidden from a peer while /// it ran an older version are never re-requested after it upgrades, because -/// sync vectors only grow, so snapshots are the recovery path. Send podcast +/// sync vectors only grow, so snapshots are the only recovery path for such a +/// peer, and only for the records that fit the snapshot bounds. Send podcast /// sections only to v4 peers ([`SyncSnapshot::for_peer`]). #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct SyncSnapshot { @@ -1019,31 +1136,75 @@ impl SyncSnapshot { snapshot } - /// Sorts and truncates the podcast sections to the snapshot limits. - /// - /// Each section is ordered newest first (subscriptions, shares and - /// progress by `hlc_ms`, sessions by `ended_at_ms`, ties by id) and capped - /// by its `MAX_SNAPSHOT_*` count. Sections then share - /// [`MAX_PODCAST_SNAPSHOT_BYTES`] of serialized JSON in priority order: - /// subscriptions, shares, progress, sessions. Older registers that do not - /// fit still propagate through operations. + /// Bounds the podcast sections with the default budget + /// [`MAX_PODCAST_SNAPSHOT_BYTES`]; see + /// [`SyncSnapshot::bound_podcast_state_to`]. pub fn bound_podcast_state(&mut self) { - let mut budget = MAX_PODCAST_SNAPSHOT_BYTES; + self.bound_podcast_state_to(MAX_PODCAST_SNAPSHOT_BYTES); + } + + /// Sorts and truncates the podcast sections to the snapshot limits and to + /// `budget_bytes` of serialized JSON. + /// + /// * Subscription records lose `podcast.description`: it is the largest + /// field and every device refreshes channel metadata from the feed. The + /// register (`subscribed`, stamps) and feed aliases are kept whole. + /// * Each section is ordered newest first (subscriptions and shares by + /// `(generation, hlc_ms)`, progress by `(generation, hlc_ms)`, sessions + /// by `ended_at_ms`; ties by id) and capped by its `MAX_SNAPSHOT_*` + /// count. + /// * Each section has its own share of the budget (subscriptions 25 %, + /// shares 5 %, progress 45 %, sessions 25 %), so large subscription + /// records cannot crowd out progress. Unused budget passes on to the + /// following sections in that order. + /// + /// Callers pass [`MAX_PODCAST_SNAPSHOT_BYTES`], or less when the rest of + /// the message (music snapshot, operations, playback) leaves less room + /// under [`MAX_SYNC_LINE`]; see [`podcast_snapshot_budget`]. + /// + /// Records cut here reach a peer only through operations it has not + /// acknowledged yet. A peer that skipped podcast operations while it ran + /// an older version never receives the cut records: it recovers the newest + /// records only, and its play counts can differ from other devices. + pub fn bound_podcast_state_to(&mut self, budget_bytes: usize) { + let share = |percent: usize| budget_bytes / 100 * percent; + let mut budget = 0usize; + for record in &mut self.podcast_subscriptions { + record.podcast.description = None; + } self.podcast_subscriptions.sort_by(|a, b| { - (b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&(a.hlc_ms, &a.op_id, &a.podcast_id)) + (b.generation, b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&( + a.generation, + a.hlc_ms, + &a.op_id, + &a.podcast_id, + )) }); + budget += share(25); bound_section( &mut self.podcast_subscriptions, MAX_SNAPSHOT_SUBSCRIPTIONS, &mut budget, ); self.podcast_shares.sort_by(|a, b| { - (b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&(a.hlc_ms, &a.op_id, &a.podcast_id)) + (b.generation, b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&( + a.generation, + a.hlc_ms, + &a.op_id, + &a.podcast_id, + )) }); + budget += share(5); bound_section(&mut self.podcast_shares, MAX_SNAPSHOT_SHARES, &mut budget); self.podcast_progress.sort_by(|a, b| { - (b.hlc_ms, &b.op_id, &b.episode_id).cmp(&(a.hlc_ms, &a.op_id, &a.episode_id)) + (b.generation, b.hlc_ms, &b.op_id, &b.episode_id).cmp(&( + a.generation, + a.hlc_ms, + &a.op_id, + &a.episode_id, + )) }); + budget += share(45); bound_section( &mut self.podcast_progress, MAX_SNAPSHOT_PROGRESS, @@ -1051,6 +1212,7 @@ impl SyncSnapshot { ); self.podcast_listen_sessions .sort_by(|a, b| (b.ended_at_ms, &b.session_id).cmp(&(a.ended_at_ms, &a.session_id))); + budget += budget_bytes - share(25) - share(5) - share(45); bound_section( &mut self.podcast_listen_sessions, MAX_SNAPSHOT_LISTEN_SESSIONS, @@ -1070,6 +1232,17 @@ impl SyncSnapshot { } } +/// Byte budget for the podcast snapshot sections of one message. +/// +/// `other_bytes` is the serialized size of the message without podcast +/// sections. The result keeps the whole line under [`MAX_SYNC_LINE`] with a +/// 64 KiB margin and never exceeds [`MAX_PODCAST_SNAPSHOT_BYTES`]. +pub fn podcast_snapshot_budget(other_bytes: usize) -> usize { + MAX_SYNC_LINE + .saturating_sub(other_bytes.saturating_add(64 * 1024)) + .min(MAX_PODCAST_SNAPSHOT_BYTES) +} + fn bound_section(items: &mut Vec, max_items: usize, budget: &mut usize) { let mut keep = 0; for item in items.iter().take(max_items) { @@ -1559,6 +1732,7 @@ mod tests { season: Some(1), episode_number: Some(3), image_url: None, + progress_generation: Some(4), } } @@ -1649,10 +1823,16 @@ mod tests { image_url: None, }, feed_aliases: vec![feed()], + generation: 1, }, SyncOpPayload::PodcastShareSet { podcast_id: podcast_id(), shared: false, + generation: 2, + }, + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id: podcast_id(), + feed_aliases: vec![FeedUrl::parse("https://moved.example.test/feed").unwrap()], }, SyncOpPayload::PodcastProgressSet { episode_id: episode_id(), @@ -1675,6 +1855,7 @@ mod tests { let kinds = [ "podcast_subscription_set", "podcast_share_set", + "podcast_feed_aliases_added", "podcast_progress_set", "podcast_listen_recorded", ]; @@ -1698,14 +1879,25 @@ mod tests { (1_001, "dev-a:1") ); assert!(subscription.subscribed); + assert_eq!(subscription.generation, 1); let share = ops[1].podcast_share().unwrap(); assert!(!share.shared); - let progress = ops[2].podcast_progress().unwrap(); + assert_eq!(share.generation, 2); + let (alias_podcast, aliases) = ops[2].podcast_feed_aliases().unwrap(); + assert_eq!(alias_podcast, &podcast_id()); + let mut with_alias = subscription.clone(); + with_alias.add_feed_aliases(aliases); + assert!(with_alias.feed_aliases.contains(&aliases[0])); + // The alias operation never touches the register. + assert_eq!(with_alias.order_key(), subscription.order_key()); + assert_eq!(with_alias.subscribed, subscription.subscribed); + let progress = ops[3].podcast_progress().unwrap(); assert_eq!(progress.device_id, "dev-a"); - assert_eq!(progress.op_id, "dev-a:3"); + assert_eq!(progress.op_id, "dev-a:4"); assert_eq!(progress.generation, 3); - assert_eq!(ops[3].podcast_listen_session(), Some(&session())); + assert_eq!(ops[4].podcast_listen_session(), Some(&session())); assert!(ops[0].podcast_share().is_none()); + assert!(ops[0].podcast_feed_aliases().is_none()); } #[test] @@ -1719,16 +1911,21 @@ mod tests { _ => unreachable!(), }, feed_aliases: Vec::new(), + generation: 3, }; assert!(!unsubscribe.is_tombstone(), "unsubscribe is LWW state"); - assert!(ops[1].is_tombstone(), "share=false is a tombstone"); + // share=false protects privacy: compacting it would let a stale + // share=true delivered again republish the podcast. + assert!(!ops[1].is_tombstone(), "share=false is LWW state"); let share = SyncOpPayload::PodcastShareSet { podcast_id: podcast_id(), shared: true, + generation: 4, }; assert!(!share.is_tombstone()); - assert!(!ops[2].is_tombstone()); - assert!(!ops[3].is_tombstone()); + for payload in &ops { + assert!(!payload.is_tombstone(), "{payload:?}"); + } } #[test] @@ -1781,6 +1978,49 @@ mod tests { }, ); like.validate_podcast().unwrap(); + let empty_aliases = op( + 5, + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id: podcast_id(), + feed_aliases: Vec::new(), + }, + ); + assert!(empty_aliases.validate_podcast().is_err()); + } + + #[test] + fn malformed_ids_and_feed_urls_reject_the_whole_message() { + // Pins the documented behaviour: ids and feed URLs are validated + // while deserializing, so a non-canonical value is a schema + // violation of the whole message, not a skippable operation. + let like = op( + 1, + SyncOpPayload::TrackLikeSet { + content_id: "b3:x".to_string(), + liked: true, + fed: None, + }, + ); + let share = op( + 2, + SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id(), + shared: true, + generation: 1, + }, + ); + let batch = serde_json::to_string(&vec![like, share]).unwrap(); + serde_json::from_str::>(&batch).unwrap(); + let upper = batch.replace(podcast_id().as_str(), &podcast_id().as_str().to_uppercase()); + assert!(serde_json::from_str::>(&upper).is_err()); + let subscription = op(3, podcast_ops().remove(0)); + let json = serde_json::to_string(&subscription).unwrap(); + let non_canonical = json.replace( + "https://private.example.test/feed", + "HTTPS://private.example.test:443/feed", + ); + assert_ne!(json, non_canonical); + assert!(serde_json::from_str::(&non_canonical).is_err()); } #[test] @@ -2014,6 +2254,67 @@ mod tests { UNREPORTED_PROTOCOL_VERSION ); assert!(!supports_podcasts(profile_protocol_version(None, None))); + // A self-report may lower the version (downgrade). + assert_eq!(profile_protocol_version(Some(3), Some(4)), 3); + } + + #[test] + fn relayed_versions_only_raise_the_stored_version() { + // A deployed v3 relay forwards every device with its own constant. + assert_eq!(relayed_protocol_version(3, Some(4)), 4); + assert!(supports_podcasts(relayed_protocol_version(3, Some(4)))); + assert_eq!(relayed_protocol_version(4, Some(3)), 4); + assert_eq!(relayed_protocol_version(3, None), 3); + assert_eq!(relayed_protocol_version(1, Some(2)), 2); + } + + #[test] + fn pair_requests_gate_by_the_invite_version() { + let invite = InviteWire { + v: 1, + ticket: "t".to_string(), + device_id: "inviter".to_string(), + invite_id: "i".to_string(), + secret: "s".to_string(), + expires_at_ms: 1, + protocol_version: None, + }; + // Invites of older clients carry no version: assume the oldest. + let legacy = r#"{"v":1,"t":"t","d":"inviter","i":"i","s":"s","e":1}"#; + let parsed: InviteWire = serde_json::from_str(legacy).unwrap(); + assert_eq!(parsed, invite); + assert_eq!(serde_json::to_string(&invite).unwrap(), legacy); + assert_eq!( + parsed.inviter_protocol_version(), + UNREPORTED_PROTOCOL_VERSION + ); + let snapshot = SyncSnapshot { + podcast_listen_sessions: vec![session()], + ..SyncSnapshot::default() + }; + assert!( + !snapshot + .for_peer(parsed.inviter_protocol_version()) + .has_podcast_state() + ); + for payload in podcast_ops() { + assert!(!op_allowed_for_peer( + &payload, + parsed.inviter_protocol_version() + )); + } + assert!(!playback_allowed_for_peer( + &mixed(), + parsed.inviter_protocol_version() + )); + let current = InviteWire { + protocol_version: Some(DEVICE_SYNC_PROTOCOL_VERSION), + ..invite + }; + let link = encode_invite(¤t).unwrap(); + let back = parse_invite(&link).unwrap(); + assert_eq!(back, current); + assert!(supports_podcasts(back.inviter_protocol_version())); } #[test] @@ -2091,7 +2392,7 @@ mod tests { let snapshot = SyncSnapshot { podcast_subscriptions: vec![ops[0].podcast_subscription().unwrap()], podcast_shares: vec![ops[1].podcast_share().unwrap()], - podcast_progress: vec![ops[2].podcast_progress().unwrap()], + podcast_progress: vec![ops[3].podcast_progress().unwrap()], podcast_listen_sessions: vec![session()], ..SyncSnapshot::default() }; @@ -2130,6 +2431,7 @@ mod tests { &FeedUrl::parse(&format!("https://a.test/{i}")).unwrap(), ), shared: i % 2 == 0, + generation: 1, hlc_ms: i as i64, op_id: format!("dev:{i}"), }) @@ -2156,7 +2458,10 @@ mod tests { #[test] fn snapshot_byte_budget_is_enforced() { - let big = "x".repeat(crate::podcast::MAX_DESCRIPTION_CHARS); + // Worst-case subscription records: long Cyrillic title, author and + // description, plus a few progress registers. + let big: String = "я".repeat(crate::podcast::MAX_DESCRIPTION_CHARS); + let title: String = big.chars().take(crate::podcast::MAX_TITLE_CHARS).collect(); let subscriptions: Vec = (0..MAX_SNAPSHOT_SUBSCRIPTIONS) .map(|i| { let url = FeedUrl::parse(&format!("https://a.test/{i}")).unwrap(); @@ -2165,38 +2470,82 @@ mod tests { subscribed: true, podcast: PodcastMeta { feed_url: url, - title: big.chars().take(1_000).collect(), - author: None, + title: title.clone(), + author: Some(title.clone()), description: Some(big.clone()), link: None, image_url: None, }, feed_aliases: Vec::new(), + generation: 1, hlc_ms: i as i64, op_id: format!("dev:{i}"), } }) .collect(); - let mut snapshot = SyncSnapshot { + let progress: Vec = (0..10) + .map(|i| { + op( + i + 1, + SyncOpPayload::PodcastProgressSet { + episode_id: EpisodeId::from_guid(&podcast_id(), &format!("e{i}")) + .unwrap(), + podcast_id: podcast_id(), + position_ms: 1_000, + status: ProgressStatus::InProgress, + generation: 1, + session_id: "s".to_string(), + episode: None, + }, + ) + .podcast_progress() + .unwrap() + }) + .collect(); + let original = SyncSnapshot { podcast_subscriptions: subscriptions, + podcast_progress: progress, podcast_listen_sessions: vec![session()], ..SyncSnapshot::default() }; + let section_bytes = |snapshot: &SyncSnapshot| -> usize { + fn sum(items: &[T]) -> usize { + items + .iter() + .map(|item| serde_json::to_vec(item).unwrap().len() + 1) + .sum() + } + sum(&snapshot.podcast_subscriptions) + + sum(&snapshot.podcast_shares) + + sum(&snapshot.podcast_progress) + + sum(&snapshot.podcast_listen_sessions) + }; + let mut snapshot = original.clone(); snapshot.bound_podcast_state(); - let bytes: usize = snapshot - .podcast_subscriptions - .iter() - .map(|record| serde_json::to_vec(record).unwrap().len() + 1) - .sum::() - + snapshot - .podcast_listen_sessions + assert!(section_bytes(&snapshot) <= MAX_PODCAST_SNAPSHOT_BYTES); + assert!( + snapshot + .podcast_subscriptions .iter() - .map(|record| serde_json::to_vec(record).unwrap().len() + 1) - .sum::(); - assert!(bytes <= MAX_PODCAST_SNAPSHOT_BYTES); + .all(|record| record.podcast.description.is_none()), + "descriptions are refreshed from RSS, not snapshotted" + ); assert!(snapshot.podcast_subscriptions.len() < MAX_SNAPSHOT_SUBSCRIPTIONS); assert!(!snapshot.podcast_subscriptions.is_empty()); + // Subscriptions cannot crowd out progress and sessions. + assert_eq!(snapshot.podcast_progress.len(), 10); + assert_eq!(snapshot.podcast_listen_sessions.len(), 1); assert!(serde_json::to_vec(&snapshot).unwrap().len() < MAX_SYNC_LINE); + + // A smaller budget, for a message whose other parts are large. + let budget = podcast_snapshot_budget(6 * 1024 * 1024); + assert!(budget < MAX_PODCAST_SNAPSHOT_BYTES); + let mut small = original; + small.bound_podcast_state_to(budget); + assert!(section_bytes(&small) <= budget); + assert_eq!(small.podcast_progress.len(), 10); + assert_eq!(podcast_snapshot_budget(0), MAX_PODCAST_SNAPSHOT_BYTES); + assert_eq!(podcast_snapshot_budget(MAX_SYNC_LINE), 0); } } } diff --git a/crates/music-dht/src/jam.rs b/crates/music-dht/src/jam.rs index b208b42..f530948 100644 --- a/crates/music-dht/src/jam.rs +++ b/crates/music-dht/src/jam.rs @@ -314,6 +314,7 @@ mod tests { season: None, episode_number: None, image_url: None, + progress_generation: Some(7), }; let snapshot = PlaybackSnapshot { device_id: "host".to_string(), diff --git a/crates/music-dht/src/podcast/feed_url.rs b/crates/music-dht/src/podcast/feed_url.rs index ea13774..d553577 100644 --- a/crates/music-dht/src/podcast/feed_url.rs +++ b/crates/music-dht/src/podcast/feed_url.rs @@ -43,18 +43,25 @@ impl FeedUrl { /// /// Surrounding whitespace is trimmed, `feed:`, `podcast:`, `pcast:` and /// `itpc:` prefixes map to `https://` (or to the embedded `http(s)` URL), - /// and input without a scheme defaults to `https://`. + /// and input without a scheme defaults to `https://`. `http:` or `https:` + /// not followed by `//` (for example `https:/host/feed`) is rejected. pub fn from_user_input(input: &str) -> PodcastResult { let trimmed = input.trim(); if trimmed.is_empty() { return Err(invalid("feed URL is empty")); } let lower = trimmed.to_ascii_lowercase(); + if is_http_scheme_typo(&lower) { + return Err(invalid("feed URL must start with http:// or https://")); + } let mut candidate = trimmed.to_string(); for scheme in ["feed:", "podcast:", "pcast:", "itpc:"] { if lower.starts_with(scheme) { let rest = &trimmed[scheme.len()..]; let rest_lower = rest.to_ascii_lowercase(); + if is_http_scheme_typo(&rest_lower) { + return Err(invalid("feed URL must start with http:// or https://")); + } candidate = if rest_lower.starts_with("http://") || rest_lower.starts_with("https://") { rest.to_string() @@ -110,6 +117,13 @@ fn invalid(reason: &str) -> PodcastError { PodcastError::InvalidFeedUrl(reason.to_string()) } +/// `http:` or `https:` not followed by `//`, such as `https:/host/feed`. +fn is_http_scheme_typo(lower: &str) -> bool { + ["http:", "https:"] + .iter() + .any(|scheme| lower.starts_with(scheme) && !lower[scheme.len()..].starts_with("//")) +} + fn has_scheme(text: &str) -> bool { let Some((scheme, rest)) = text.split_once(':') else { return false; @@ -252,7 +266,8 @@ fn push_ascii(out: &mut String, value: &str) { /// /// `known` yields `(podcast_id, url)` pairs of every URL already associated /// with a podcast: the URL of its first subscription, its current fetch URL -/// and its feed aliases. When `url` matches, the existing id is returned (the +/// and its feed aliases ([`super::SubscriptionRecord::feed_urls`]; the bounded +/// alias set alone can miss the current fetch URL). When `url` matches, the existing id is returned (the /// smallest id if, after a concurrent offline subscription, several podcasts /// share the URL). Otherwise a new id is derived from `url`. /// @@ -277,7 +292,9 @@ pub fn resolve_podcast_id<'a>( /// derived is always kept; of the other URLs the [`MAX_FEED_ALIASES`] /// lexicographically smallest are kept. Taking the smallest `k` elements of a /// union is commutative, associative and idempotent, so devices converge -/// regardless of delivery order. +/// regardless of delivery order. Other URLs, including a podcast's current +/// fetch URL, can fall outside the bound; the materialized subscription keeps +/// its `podcast.feed_url` separately. pub fn merge_feed_aliases(podcast: &PodcastId, a: &[FeedUrl], b: &[FeedUrl]) -> Vec { let mut origin = None; let mut rest = BTreeSet::new(); @@ -437,6 +454,12 @@ mod tests { "https://", "file:///etc/passwd", "javascript:alert(1)", + // One-slash typos must not become a host named "https". + "https:/a.test/feed", + "http:/a.test/feed", + "HTTPS:a.test/feed", + "https:///a.test/feed", + "feed:https:/a.test/feed", ] { assert!(FeedUrl::from_user_input(bad).is_err(), "{bad:?}"); } diff --git a/crates/music-dht/src/podcast/meta.rs b/crates/music-dht/src/podcast/meta.rs index 1f36ff7..aa036bf 100644 --- a/crates/music-dht/src/podcast/meta.rs +++ b/crates/music-dht/src/podcast/meta.rs @@ -227,6 +227,7 @@ impl EpisodeMeta { episode_number: self.episode_number, image_url: valid_url(self.image_url.as_deref()) .or_else(|| valid_url(podcast.image_url.as_deref())), + progress_generation: None, } } } @@ -236,8 +237,9 @@ impl EpisodeMeta { /// Carried by playback queues (`PlaybackSource::PodcastEpisode`) and by /// progress operations, so a receiving trusted device can display, resume and /// play the episode without being subscribed and without its own copy of the -/// feed. It contains no personal state. It is private: Jam snapshots replace it -/// with an opaque placeholder. +/// feed. Apart from the optional handoff hint +/// [`progress_generation`](Self::progress_generation) it contains no personal +/// state. It is private: Jam snapshots replace it with an opaque placeholder. /// /// If the audio URL went stale, refresh the feed at `feed_url` and look the /// episode up by `episode_id`. @@ -273,6 +275,17 @@ pub struct PodcastEpisodeRef { /// Episode or podcast artwork URL. #[serde(default)] pub image_url: Option, + /// Greatest progress generation the sender has seen for this episode. + /// + /// Set on queue items of a playback state (in particular on handoff), + /// because the playback state can arrive before the sender's progress + /// operations. A device that starts playing the item writes its explicit + /// action with + /// [`ProgressRecord::explicit_generation`](super::ProgressRecord::explicit_generation)`(max(local, progress_generation))`. + /// Ignored in progress and listening operations, whose own generation is + /// authoritative. Not part of the episode identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub progress_generation: Option, } impl PodcastEpisodeRef { diff --git a/crates/music-dht/src/podcast/mod.rs b/crates/music-dht/src/podcast/mod.rs index d4b2bb6..4daec10 100644 --- a/crates/music-dht/src/podcast/mod.rs +++ b/crates/music-dht/src/podcast/mod.rs @@ -28,11 +28,13 @@ //! //! # Personal state //! -//! * [`SubscriptionRecord`]: last-writer-wins by `(hlc_ms, op_id)`; -//! unsubscribing is state, not a tombstone. Feed aliases merge as a bounded -//! grow-only set. -//! * [`ShareRecord`]: last-writer-wins; `shared = false` is a tombstone. See -//! [`effective_share`] for the rule that a (re)subscription is private. +//! * [`SubscriptionRecord`]: last-writer-wins by `(generation, hlc_ms, op_id)` +//! where `generation` is a per-podcast Lamport counter +//! ([`next_podcast_generation`]); unsubscribing is state, not a tombstone. +//! Feed aliases merge as a bounded grow-only set. +//! * [`ShareRecord`]: last-writer-wins with the same ordering; `shared = false` +//! is register state that is never compacted. See [`effective_share`] for +//! the rule that a (re)subscription is private. //! * [`ProgressRecord`]: register ordered by //! `(generation, hlc_ms, device_id, op_id)`. Explicit user actions bump the //! generation to `max_seen + 1`; periodic checkpoints keep the generation of @@ -85,7 +87,8 @@ pub use meta::{EpisodeMeta, EpisodeType, PodcastEpisodeRef, PodcastMeta}; pub use state::{ COMPLETION_TAIL_MAX_MS, ListenSession, MIN_RECORDED_EPISODE_LISTEN_MS, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, completed_listen_count, effective_share, - merge_listen_session, merge_progress, merge_share, merge_subscription, reaches_completion, + merge_listen_session, merge_progress, merge_share, merge_subscription, next_podcast_generation, + reaches_completion, }; /// Maximum characters of a title (podcast, episode, author). diff --git a/crates/music-dht/src/podcast/state.rs b/crates/music-dht/src/podcast/state.rs index c258eb7..f238995 100644 --- a/crates/music-dht/src/podcast/state.rs +++ b/crates/music-dht/src/podcast/state.rs @@ -57,15 +57,29 @@ pub enum ProgressStatus { /// seeking backwards and "start over" are real changes. /// /// * Explicit user actions (start of a new listening session, seek, "start -/// over", "mark played", "mark unplayed", handoff to another device) use -/// [`ProgressRecord::explicit_generation`]: one more than the greatest -/// generation the device has seen for the episode. This is a Lamport clock, -/// so wall-clock skew cannot undo an explicit action that was observed. +/// over", "mark played", "mark unplayed", taking over playback through a +/// handoff) use [`ProgressRecord::explicit_generation`]: one more than the +/// greatest generation the device has seen for the episode. This is a +/// Lamport clock, so wall-clock skew cannot undo an explicit action that was +/// observed. An explicit action during a session becomes the generation of +/// that session. /// * Periodic checkpoints keep the generation of the latest explicit action of /// their session. A late checkpoint of an older session therefore loses /// against any newer explicit action, whatever its `hlc_ms`. /// * Concurrent explicit actions on offline devices receive the same /// generation and are ordered by `hlc_ms`, then `device_id`, then `op_id`. +/// * Handoff: only the receiving device writes the explicit action, with +/// `explicit_generation(max(local, carried))` where `carried` is +/// [`PodcastEpisodeRef::progress_generation`](super::PodcastEpisodeRef::progress_generation) +/// of the queue item. The sender writes a final checkpoint at its session +/// generation. +/// * A device that observes a greater generation for the episode it is +/// playing stops writing at its old generation: either playback moved away, +/// or the user keeps listening locally and the device writes its next +/// record as an explicit action. +/// * Outgoing-log coalescing may drop a pending progress operation only when +/// another pending operation of the same episode [supersedes](Self::supersedes) +/// it; dropping by sequence order could discard an explicit action. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ProgressRecord { /// Episode the register belongs to. @@ -162,18 +176,25 @@ pub fn merge_progress( /// Personal subscription state of one podcast. /// /// `subscribed` and `podcast` form one last-writer-wins register ordered by -/// `(hlc_ms, op_id)`. Unsubscribing writes `subscribed = false`: it is state, -/// not a tombstone, and never deletes progress or history, so subscribing -/// again continues where the user stopped. +/// `(generation, hlc_ms, op_id)`. `generation` is a Lamport counter shared by +/// the subscription and share registers of the podcast (see +/// [`next_podcast_generation`]), so an action taken after observing another +/// device's write wins whatever the device clocks say. Unsubscribing writes +/// `subscribed = false`: it is state, not a tombstone, is never compacted, and +/// never deletes progress or history, so subscribing again continues where +/// the user stopped. /// /// Only explicit user actions (subscribe, unsubscribe) write this register. /// Background RSS refreshes keep their metadata locally and must not emit a /// subscription operation; otherwise a stale device could resurrect an -/// unsubscribed podcast. +/// unsubscribed podcast. Feed aliases learned by refreshes travel in their +/// own grow-only operation. /// /// `feed_aliases` is a bounded grow-only set merged independently of the -/// register (see [`merge_feed_aliases`]); it always contains every -/// `podcast.feed_url` seen. +/// register (see [`merge_feed_aliases`]): the URL the id was derived from +/// plus the [`MAX_FEED_ALIASES`] smallest other URLs. The current +/// `podcast.feed_url` may fall outside that bound, so look podcasts up by +/// [`SubscriptionRecord::feed_urls`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SubscriptionRecord { /// Podcast. @@ -185,6 +206,9 @@ pub struct SubscriptionRecord { /// Known feed URLs of the podcast. #[serde(default)] pub feed_aliases: Vec, + /// Lamport generation shared with the share register (see type docs). + #[serde(default)] + pub generation: u64, /// Logical write time. pub hlc_ms: i64, /// Writing operation id. @@ -192,9 +216,14 @@ pub struct SubscriptionRecord { } impl SubscriptionRecord { + /// Register ordering key: `(generation, hlc_ms, op_id)`. + pub fn order_key(&self) -> (u64, i64, &str) { + (self.generation, self.hlc_ms, self.op_id.as_str()) + } + fn total_cmp(&self, other: &Self) -> Ordering { - (self.hlc_ms, self.op_id.as_str()) - .cmp(&(other.hlc_ms, other.op_id.as_str())) + self.order_key() + .cmp(&other.order_key()) .then_with(|| self.subscribed.cmp(&other.subscribed)) .then_with(|| self.podcast.cmp(&other.podcast)) } @@ -204,7 +233,10 @@ impl SubscriptionRecord { self.total_cmp(other) == Ordering::Greater } - /// Ensures `feed_aliases` is sorted, bounded and contains `podcast.feed_url`. + /// Merges `podcast.feed_url` into `feed_aliases` and bounds the set. + /// + /// The result is sorted and bounded by [`merge_feed_aliases`]; it may not + /// contain `podcast.feed_url` when many smaller aliases are known. pub fn normalize(&mut self) { self.feed_aliases = merge_feed_aliases( &self.podcast_id, @@ -213,6 +245,23 @@ impl SubscriptionRecord { ); } + /// Adds aliases learned through a `podcast_feed_aliases_added` operation. + /// + /// Only the grow-only alias set changes; the register is untouched. + pub fn add_feed_aliases(&mut self, aliases: &[FeedUrl]) { + self.feed_aliases = merge_feed_aliases(&self.podcast_id, &self.feed_aliases, aliases); + } + + /// Every URL that identifies this podcast: the aliases and the current + /// `podcast.feed_url`, sorted and without duplicates. + /// + /// Stores pass these pairs to [`super::resolve_podcast_id`]. + pub fn feed_urls(&self) -> Vec { + let mut urls: BTreeSet = self.feed_aliases.iter().cloned().collect(); + urls.insert(self.podcast.feed_url.clone()); + urls.into_iter().collect() + } + /// Validates contract bounds. pub fn validate(&self) -> PodcastResult<()> { self.podcast.validate()?; @@ -251,8 +300,12 @@ pub fn merge_subscription( /// Decision to publish a podcast card to the federation. /// -/// Last-writer-wins by `(hlc_ms, op_id)`; `shared = false` is the tombstone -/// that revokes the publication. Unsubscribing must also write +/// Last-writer-wins by `(generation, hlc_ms, op_id)`, with `generation` the +/// per-podcast Lamport counter shared with [`SubscriptionRecord`] (see +/// [`next_podcast_generation`]). `shared = false` revokes the publication. +/// It is privacy-protecting register state, **not** a tombstone: like an +/// unsubscribe it is never compacted, otherwise a stale `shared = true` +/// delivered again would republish the podcast. Unsubscribing must also write /// `shared = false`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ShareRecord { @@ -260,6 +313,9 @@ pub struct ShareRecord { pub podcast_id: PodcastId, /// Whether the user chose to share the podcast. pub shared: bool, + /// Lamport generation shared with the subscription register. + #[serde(default)] + pub generation: u64, /// Logical write time. pub hlc_ms: i64, /// Writing operation id. @@ -267,12 +323,15 @@ pub struct ShareRecord { } impl ShareRecord { + /// Register ordering key: `(generation, hlc_ms, op_id)`. + pub fn order_key(&self) -> (u64, i64, &str) { + (self.generation, self.hlc_ms, self.op_id.as_str()) + } + fn total_cmp(&self, other: &Self) -> Ordering { - (self.hlc_ms, self.op_id.as_str(), self.shared).cmp(&( - other.hlc_ms, - other.op_id.as_str(), - other.shared, - )) + self.order_key() + .cmp(&other.order_key()) + .then_with(|| self.shared.cmp(&other.shared)) } /// Returns whether this record wins over `other`. @@ -298,13 +357,36 @@ pub fn merge_share(current: &ShareRecord, incoming: &ShareRecord) -> PodcastResu }) } +/// Generation for a new write of the subscription or share register. +/// +/// One more than the greatest generation of either register of the podcast +/// (`None` when it has none), so a write made after observing another +/// device's subscription, unsubscription or share decision orders after it +/// even when that device's clock runs ahead. Concurrent writes on devices +/// that did not see each other receive the same generation and are ordered +/// by `hlc_ms`, then `op_id`. +pub fn next_podcast_generation( + subscription: Option<&SubscriptionRecord>, + share: Option<&ShareRecord>, +) -> u64 { + subscription + .map(|record| record.generation) + .into_iter() + .chain(share.map(|record| record.generation)) + .max() + .map_or(1, |generation| generation.saturating_add(1)) +} + /// Whether a podcast must currently be published. /// /// True only when the user is subscribed, chose to share, and the share -/// decision is newer than the latest subscription write. A new subscription -/// (including re-subscribing, or subscribing from someone else's public card) -/// is therefore always private until the user shares it again, and a stale -/// device cannot republish a podcast through an old share decision. +/// decision orders after the latest subscription write (by +/// `(generation, hlc_ms, op_id)`). A new subscription (including +/// re-subscribing, or subscribing from someone else's public card) is +/// therefore always private until the user shares it again, and a stale +/// device cannot republish a podcast through an old share decision. A share +/// decision taken after observing the subscription always applies, because it +/// carries a greater generation. pub fn effective_share( subscription: Option<&SubscriptionRecord>, share: Option<&ShareRecord>, @@ -314,8 +396,7 @@ pub fn effective_share( subscription.subscribed && share.shared && subscription.podcast_id == share.podcast_id - && (share.hlc_ms, share.op_id.as_str()) - > (subscription.hlc_ms, subscription.op_id.as_str()) + && share.order_key() > subscription.order_key() } _ => false, } diff --git a/crates/music-dht/src/podcast/state_tests.rs b/crates/music-dht/src/podcast/state_tests.rs index 2bbf60a..8110268 100644 --- a/crates/music-dht/src/podcast/state_tests.rs +++ b/crates/music-dht/src/podcast/state_tests.rs @@ -282,6 +282,7 @@ fn subscription( subscribed, podcast: podcast_meta(title, url), feed_aliases: Vec::new(), + generation: 1, hlc_ms, op_id: op.to_string(), }; @@ -289,6 +290,12 @@ fn subscription( record } +fn with_generation(record: &T, set: impl Fn(&mut T)) -> T { + let mut record = record.clone(); + set(&mut record); + record +} + #[test] fn subscriptions_converge_for_every_permutation() { let ops = [ @@ -330,13 +337,14 @@ fn share(shared: bool, hlc_ms: i64, op: &str) -> ShareRecord { ShareRecord { podcast_id: podcast_id(), shared, + generation: 1, hlc_ms, op_id: op.to_string(), } } #[test] -fn shares_converge_and_false_is_a_tombstone() { +fn shares_converge_and_false_is_never_beaten_by_a_stale_true() { let ops = [ share(true, 100, "dev-a:1"), share(false, 200, "dev-a:2"), @@ -364,6 +372,129 @@ fn effective_share_rules() { assert!(!effective_share(Some(&unsubscribed), Some(&stale))); } +#[test] +fn clock_skew_cannot_undo_an_observed_unsubscribe() { + // Device A's clock runs ten minutes ahead. B sees A's subscription, then + // the user unsubscribes on B. + let sub_a = subscription(true, 1_000_600_000, "dev-a:1", "Show", "feed"); + assert_eq!(next_podcast_generation(None, None), 1); + let generation = next_podcast_generation(Some(&sub_a), None); + assert_eq!(generation, 2); + let unsub_b = with_generation( + &subscription(false, 1_000_060_000, "dev-b:1", "Show", "feed"), + |r| r.generation = generation, + ); + let merged = assert_converges(&[sub_a.clone(), unsub_b.clone()], merge_subscription); + assert!(!merged.subscribed, "the later unsubscribe wins"); + // Concurrent writes (same generation) still fall back to the clock. + let concurrent = with_generation(&unsub_b, |r| r.generation = 1); + assert!(merge_subscription(&sub_a, &concurrent).unwrap().subscribed); +} + +#[test] +fn share_after_subscribe_applies_despite_clock_skew() { + let sub_a = subscription(true, 1_000_600_000, "dev-a:1", "Show", "feed"); + let share_b = with_generation(&share(true, 1_000_060_000, "dev-b:1"), |r| { + r.generation = next_podcast_generation(Some(&sub_a), None) + }); + assert!(effective_share(Some(&sub_a), Some(&share_b))); + // Unsubscribe written after the share (higher generation) revokes it, + // and so does a later re-subscription until the user shares again. + let unsub = with_generation(&subscription(false, 1, "dev-a:2", "Show", "feed"), |r| { + r.generation = next_podcast_generation(Some(&sub_a), Some(&share_b)) + }); + assert!(!effective_share(Some(&unsub), Some(&share_b))); + let resub = with_generation(&subscription(true, 2, "dev-a:3", "Show", "feed"), |r| { + r.generation = next_podcast_generation(Some(&unsub), Some(&share_b)) + }); + assert!(!effective_share(Some(&resub), Some(&share_b))); + let reshare = with_generation(&share(true, 3, "dev-a:4"), |r| { + r.generation = next_podcast_generation(Some(&resub), Some(&share_b)) + }); + assert!(effective_share(Some(&resub), Some(&reshare))); + // Revoking (shared = false) with a higher generation wins over every + // stale true, in any order. + let revoke = with_generation(&share(false, 0, "dev-b:2"), |r| { + r.generation = next_podcast_generation(Some(&resub), Some(&reshare)) + }); + let merged = assert_converges(&[share_b, reshare, revoke.clone()], merge_share); + assert_eq!(merged, revoke); +} + +#[test] +fn current_feed_url_stays_resolvable_beyond_the_alias_bound() { + let mut record = subscription(true, 1, "dev-a:1", "Show", "feed"); + record.add_feed_aliases( + &(0..=crate::podcast::MAX_FEED_ALIASES) + .map(|i| feed_url(&format!("a{i}"))) + .collect::>(), + ); + record.podcast.feed_url = FeedUrl::parse("https://z.test/current").unwrap(); + record.normalize(); + assert!(!record.feed_aliases.contains(&record.podcast.feed_url)); + let urls = record.feed_urls(); + assert!(urls.contains(&record.podcast.feed_url)); + assert!(urls.contains(&feed_url("feed")), "origin is kept"); + let known: Vec<(&PodcastId, &FeedUrl)> = + urls.iter().map(|url| (&record.podcast_id, url)).collect(); + assert_eq!( + crate::podcast::resolve_podcast_id(&record.podcast.feed_url, known), + podcast_id() + ); +} + +#[test] +fn handoff_receiver_continues_with_the_carried_generation() { + use ProgressStatus::*; + // Sender A plays session s1 at generation 5 with a fast clock; its + // progress operations have not reached B when the handoff arrives. + let a_checkpoint = progress(5, 9_000_000, "dev-a", "dev-a:9", 600_000, InProgress, "s1"); + let carried = Some(a_checkpoint.generation); + let b_local = Some(2); + let generation = ProgressRecord::explicit_generation(b_local.max(carried)); + assert_eq!(generation, 6); + let b_start = progress( + generation, 1_000, "dev-b", "dev-b:1", 600_000, InProgress, "s2", + ); + let b_pause = progress( + generation, 2_000, "dev-b", "dev-b:2", 660_000, InProgress, "s2", + ); + // Without the carried generation B would write generation 3 and lose + // every write to A's older checkpoint. + let uninformed = progress( + ProgressRecord::explicit_generation(b_local), + 2_000, + "dev-b", + "dev-b:2", + 660_000, + InProgress, + "s2", + ); + assert!(a_checkpoint.supersedes(&uninformed)); + let merged = assert_converges(&[a_checkpoint, b_start, b_pause.clone()], merge_progress); + assert_eq!(merged, b_pause); + assert_eq!(merged.resume_position_ms(), 660_000); +} + +#[test] +fn coalescing_keeps_the_superseding_record_not_the_last_queued() { + use ProgressStatus::*; + // "Mark played" (explicit, generation 5) during a session, followed in + // the outgoing log by a checkpoint of the old session (generation 3). + let mark_played = progress(5, 100, "dev-a", "dev-a:7", 0, Completed, "mark"); + let late_checkpoint = progress(3, 101, "dev-a", "dev-a:8", 30_000, InProgress, "s1"); + let pending = [mark_played.clone(), late_checkpoint.clone()]; + // Keep only records that no other pending record supersedes. + let kept: Vec<&ProgressRecord> = pending + .iter() + .filter(|record| !pending.iter().any(|other| other.supersedes(record))) + .collect(); + assert_eq!(kept, vec![&mark_played]); + // Dropping by sequence order would have published the wrong state. + assert_ne!(pending.last(), Some(&mark_played)); + assert_eq!(fold(&pending, merge_progress), mark_played); +} + /// Minimal client store used to show that whole operation sets converge and /// that publisher metadata never reaches personal progress. #[derive(Debug, Default, Clone, PartialEq)] -- 2.54.0 From 65d6ca2f14a1324dc129b90694da83ceb55c60dc Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:10:33 +0200 Subject: [PATCH 7/7] Document podcast register ordering, versions and snapshot limits Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 21 +-- CHANGELOG.md | 10 +- PLAYBACK_PROTOCOL.md | 23 +++- PODCASTS.md | 267 +++++++++++++++++++++++++++---------- crates/music-dht/README.md | 16 ++- 5 files changed, 243 insertions(+), 94 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3129f45..71fc8d5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -199,15 +199,18 @@ the publisher: it is never imported, content-addressed or offered for P2P transfer. Device-sync protocol v4 carries podcast state between trusted devices only: -four `podcast_*` operations, bounded snapshot sections (the recovery path, -because sync vectors never re-request operations hidden from an older peer), -and an explicit `PlaybackTrack.source`. Operation and source enums are closed, -so senders gate on the version each peer reported itself; podcast state is -never sent to peers below v4, handoff to them is refused, and their playback -commands are ignored while the queue contains episodes. Jam snapshots replace -episodes with opaque placeholders. Progress ordering uses a Lamport -generation rather than wall-clock `hlc_ms`, so a device with a fast clock -cannot undo an observed seek or restart. Public podcast discovery is a future +five `podcast_*` operations, bounded snapshot sections (the recovery path, +because sync vectors never re-request operations hidden from an older peer; +past the bounds an upgraded peer recovers only the newest records), and an +explicit `PlaybackTrack.source`. The operation enum is closed and v3 clients +ignore `source`, so senders gate on the version each peer reported itself +(relayed profiles only raise it; pairing requests use the version in the +invite); podcast state is never sent to peers below v4, handoff to them is +refused, and their playback commands are ignored while the queue contains +episodes. Jam snapshots replace episodes with opaque placeholders. Progress, +subscription and share registers order by a Lamport generation before +wall-clock `hlc_ms`, so a device with a fast clock cannot undo an observed +seek, restart, unsubscribe or share decision. Public podcast discovery is a future separate overlay; the music DHT schema and `federation-net` are unchanged. Music discovery identifies an owner and content; it does not force all domain diff --git a/CHANGELOG.md b/CHANGELOG.md index 09475a8..91d2b29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,13 @@ for subscriptions, shares, progress registers and listening sessions. - `music-dht 0.6.0`: optional `podcast-feed` feature with the shared tolerant RSS/Atom parser (`quick-xml`, `encoding_rs`, `url`). -- `music-dht 0.6.0`: device-sync protocol version 4 (ALPN unchanged): four - `podcast_*` operation kinds, bounded podcast snapshot sections, +- `music-dht 0.6.0`: device-sync protocol version 4 (ALPN unchanged): five + `podcast_*` operation kinds (subscription, share, feed aliases, progress, + listening sessions), bounded podcast snapshot sections, `PlaybackTrack.source`, reserved `PlaybackStateWire.playback_rate`, - `DeviceProfileSet.protocol_version`, version-gating helpers and Jam - redaction. New struct fields and enum variants break exhaustive + `DeviceProfileSet.protocol_version`, optional `InviteWire` protocol version + (`p`), self-reported vs relayed version helpers, version-gating helpers and + Jam redaction. New struct fields and enum variants break exhaustive construction and matching in clients. - `furumi-library 0.3.0`: follows `music-dht 0.6.0`, whose types it re-exposes. - See `PODCASTS.md` for the normative contract. diff --git a/PLAYBACK_PROTOCOL.md b/PLAYBACK_PROTOCOL.md index eaaaa7f..3b86d26 100644 --- a/PLAYBACK_PROTOCOL.md +++ b/PLAYBACK_PROTOCOL.md @@ -119,31 +119,42 @@ to the portable queue DTO: - `PlaybackTrack.source`: `music`, `podcast_episode` with a self-contained `PodcastEpisodeRef`, or `private` (Jam placeholder). It is omitted for music, so music-only state serializes exactly as in v3. Podcast episodes are - never encoded as negative ids or URLs in `file_path`. + never encoded as negative ids or URLs in `file_path`. Queue items carry + `PodcastEpisodeRef.progress_generation`, the greatest progress generation + the sender has seen for the episode. - `PlaybackStateWire.playback_rate`: reserved, omitted when unset. Treat a missing or unsupported value as 1.0. -`PlaybackSource` is a closed enum: a v3 client fails to parse any message -with a podcast item. Adapters therefore apply the shared helpers from -`music_dht::device_sync`: +A v3 client does not know `source` and ignores it: it would treat an episode +as a broken music item (id 0, no content id) and forward the queue that way, +losing the episode for every device. `PlaybackSource` is also a closed enum +for v4 clients, so a future variant needs a new protocol version. Adapters +therefore apply the shared helpers from `music_dht::device_sync`: 1. Send a playback snapshot to a peer only when `playback_allowed_for_peer(state, peer_version)` holds; otherwise send `None`. 2. Refuse an explicit transfer to a device below v4 while the queue contains podcast items and show that the target needs an update. The decision uses - the version that device reported itself (`profile_protocol_version`), + the version that device reported itself (`profile_protocol_version`; + profiles relayed by other devices only raise it, `relayed_protocol_version`), never the local constant. Do not strip episodes from the queue. 3. Ignore `SetState`/`ActiveChanged` from peers below v4 while the local queue contains podcast items (`playback_command_allowed_from_peer`). 4. A stamped `playback_command` operation with podcast items is itself gated by `op_allowed_for_peer`. +5. A `PairRequest` is sent before the invite consumer hears from the inviter: + gate its playback snapshot by `InviteWire::inviter_protocol_version()` + (version 1 for invites without one). A handoff of a podcast episode records the position and the play/pause intent first, transfers the queue with its sources, and lets the receiver open the publisher's HTTP source and seek. Ownership still changes only through the claim, so a late response cannot make the previous owner resume audio. The -durable episode progress is a separate register; see `PODCASTS.md`. +durable episode progress is a separate register; see `PODCASTS.md`. Only the +receiver writes the handoff's explicit progress action, with +`ProgressRecord::explicit_generation(max(local, progress_generation))`; the +sender writes a final checkpoint at its session generation. Jam hosts send `music_dht::jam::redact_jam_snapshot`, which replaces podcast items with opaque `private` placeholders and drops the personal coordination diff --git a/PODCASTS.md b/PODCASTS.md index 71d5de1..241e235 100644 --- a/PODCASTS.md +++ b/PODCASTS.md @@ -30,7 +30,8 @@ offered for P2P transfer. RSS refreshes fetch metadata only. | Data | Trusted devices | Public federation | Local only | | --- | --- | --- | --- | -| Subscription state, channel metadata, feed aliases | `podcast_subscription_set` + snapshot | never | | +| Subscription state, channel metadata | `podcast_subscription_set` + snapshot | never | | +| Feed aliases | `podcast_feed_aliases_added`, `podcast_subscription_set` + snapshot | never | | | Share decision | `podcast_share_set` + snapshot | only the published card (future overlay) | | | Episode progress | `podcast_progress_set` + snapshot | never | | | Listening sessions, play counts | `podcast_listen_recorded` + snapshot | never | | @@ -92,19 +93,31 @@ It never changes afterwards. Redirect targets (permanent redirects), `itunes:new-feed-url` and URLs used to subscribe again are feed aliases of an existing podcast: -* the subscription record carries `feed_aliases`, a grow-only set merged - independently of the last-writer-wins register (`merge_feed_aliases`). The - URL that derives the podcast id is always kept, plus the 8 smallest other +* aliases form a grow-only set per podcast, merged independently of the + subscription register (`merge_feed_aliases`). The URL that derives the + podcast id is always kept, plus the 8 lexicographically smallest other URLs; this bounded union is commutative, associative and idempotent; -* `podcast.feed_url` is the URL currently fetched and is always added to the - aliases; +* aliases travel in their own operation, `podcast_feed_aliases_added`. A + background refresh that learns a URL emits it, but only when merging the + URL changes the stored set (a URL outside the bound is not sent again on + every refresh). The operation never touches the subscription or share + register, so it cannot resurrect a subscription or reset a share decision. + Subscribe and unsubscribe operations also carry the aliases known at that + time; +* `podcast.feed_url` of the subscription register is the URL used at the last + explicit subscribe or unsubscribe. It can fall outside the bounded alias + set, so the URLs that identify a podcast are + `SubscriptionRecord::feed_urls()`: the aliases plus `podcast.feed_url`; * before deriving a new id for a URL, clients call `resolve_podcast_id` with - every known `(podcast_id, url)` pair. A match reuses the existing podcast, - its progress and its history. With several matches the smallest id wins. + every known `(podcast_id, url)` pair, including `feed_urls()` and the URL + each device currently fetches. A match reuses the existing podcast, its + progress and its history. With several matches the smallest id wins. -Limitation: two devices that subscribe offline, through different URLs of -the same feed, before either learns the alias, create two podcast ids. The -contract does not merge podcast ids automatically in this version. +Limitations: two devices that subscribe offline, through different URLs of +the same feed, before either learns the alias, create two podcast ids; the +contract does not merge podcast ids automatically in this version. A podcast +with more than 8 aliases can lose a larger URL from the synced set; a device +that only knows that URL derives a new id. ### Episode id @@ -115,7 +128,7 @@ episode_id = "ep1:" + base32_lower_nopad(BLAKE3("furumi-podcast:episode:v1" || 0 | Key | Material | | --- | --- | | GUID | `guid:` + RSS `` / Atom `` trimmed, case-sensitive | -| link | `link:` + item `` trimmed | +| link | `link:` + item `` text (XML-unescaped once) or Atom `href`, trimmed of surrounding whitespace; no other cleanup | | date + title | `date-title:` + publication unix seconds + `\0` + normalized title | | enclosure | `enclosure:` + enclosure URL trimmed | @@ -131,11 +144,13 @@ For one parsed feed document: 1. Items with a GUID get the GUID id. A GUID never matches another known episode, so the result does not depend on local history. -2. An item without GUID first searches the known episodes of the podcast, - stage by stage: same enclosure URL, then same ``, then same - publication second and normalized title. The first stage with candidates - wins and the smallest candidate id is reused. A known episode can be - claimed by only one item of the document. +2. Items without GUID then search the known episodes of the podcast in three + stages, each run over the **whole document** before the next one starts: + same enclosure URL, then same ``, then same publication second and + normalized title. Among candidates the smallest id is reused. A known + episode can be claimed by only one item. Because a weak match of an + earlier item cannot take a known episode that a later item matches by + enclosure, the result does not depend on the order of items. 3. Otherwise a new id is derived from the first available key: ``, then date + title, then enclosure URL. @@ -145,15 +160,28 @@ Enclosure URLs and date + title pairs shared by several GUID-less items are not identifying either. Items without any usable key, and duplicates of an earlier item of the same document, get no id and are skipped. +The identity hints (GUID, enclosure, link, publication time, title) are the +ones `music_dht::podcast::feed` produces: which enclosure or `media:content` +is chosen, how relative URLs are resolved, which date field is used and how +it is parsed, how the title is cleaned. They are part of the rule; clients +must use that parser (or an implementation that passes its tests). Relative +media URLs are resolved against the URL the document was fetched from. + "Known episodes" include episodes from earlier refreshes **and** episodes learned through sync (`PodcastEpisodeRef::identity_hints`). Feed GUID-less episodes into it so a device that sees an episode first through sync keeps the id used by the other devices. -Limitation: for GUID-less episodes the id can depend on what a device saw -first. If a publisher changes several identifying fields of a GUID-less -episode at once, a device that never saw the old version can derive a -different id. +Limitations, GUID-less episodes only: + +* The id can depend on what a device saw first. If the publisher changes the + field the id was derived from (the ``, or the title or date for a + date + title id), a device that saw the old version keeps the old id + through the enclosure match, while a device that never saw it derives a + new id. One changed field is enough. +* Feeds with relative enclosure URLs fetched through different aliases give + different enclosure URLs, so the enclosure match and enclosure-derived ids + can differ between such devices. ## Metadata and bounds @@ -182,13 +210,21 @@ Episodes without a season are shown in their own group. Unknown All merge functions are commutative, associative and idempotent per entity; ties are broken by a total order over all fields. The same set of operations therefore yields the same state in any delivery order, with duplicates. -`hlc_ms` is the sync operation timestamp; it is not a true hybrid logical -clock, so ordering of user intent on progress uses a Lamport generation. +`hlc_ms` is the sync operation timestamp; clients advance it only past their +own operations, so it is not a hybrid logical clock and a device with a fast +clock stamps every write "later". Every register therefore orders by a +Lamport `generation` first and uses `hlc_ms` only between writes that did not +observe each other. ### Subscription * Register: `subscribed` + `podcast` metadata, last-writer-wins by - `(hlc_ms, op_id)`. + `(generation, hlc_ms, op_id)`. +* `generation` is a per-podcast Lamport counter shared with the share + register: every subscribe, unsubscribe and share write uses + `next_podcast_generation(subscription, share)` = one more than the greatest + generation of either register (1 when there is none). An action taken after + observing another device's write orders after it whatever the clocks say. * Unsubscribing writes `subscribed = false`. It is state, not a tombstone, and is never compacted, so it beats a stale subscription from a device that returns after being offline. @@ -197,20 +233,33 @@ clock, so ordering of user intent on progress uses a Lamport generation. separate explicit action (not part of this version). * Only explicit subscribe and unsubscribe actions write the register. RSS refreshes keep their metadata locally; a background refresh that wrote the - register could resurrect an unsubscribed podcast. + register could resurrect an unsubscribed podcast. Aliases learned by a + refresh use `podcast_feed_aliases_added`. * Subscribing to a podcast already subscribed locally must not emit a new operation. +| Events | Result | +| --- | --- | +| A (clock 10 min ahead) subscribes at g1; B sees it and unsubscribes at g2 | unsubscribed on every device | +| A and B, both offline after g1, unsubscribe and subscribe at g2 | the later `hlc_ms` (then `op_id`) wins | + ### Share -* `ShareRecord`, last-writer-wins by `(hlc_ms, op_id)`. `shared = false` is a - tombstone that revokes the publication. +* `ShareRecord`, last-writer-wins by `(generation, hlc_ms, op_id)`, with the + generation from `next_podcast_generation`. +* `shared = false` revokes the publication. Like an unsubscribe it is + privacy-protecting state, **not** a tombstone: it is never compacted and + its row is never deleted. Otherwise a stale `shared = true` still in some + device's log, or delivered again to a new device, would republish the + podcast. The register is one record per podcast, so keeping it is cheap. * Unsubscribing also writes `shared = false`. * `effective_share(subscription, share)` decides whether to publish: the user - is subscribed, chose to share, **and** the share decision is newer than the - latest subscription write. Every new subscription, including subscribing - again or subscribing from someone else's public card, is private until the - user shares it again, and a stale share decision cannot republish. + is subscribed, chose to share, **and** the share decision orders after the + latest subscription write by `(generation, hlc_ms, op_id)`. Every new + subscription, including subscribing again or subscribing from someone + else's public card, is private until the user shares it again; a stale + share decision cannot republish; a share decision taken after observing the + subscription always applies. ### Progress @@ -222,20 +271,38 @@ position: seeking back and "start over" are real changes. * Explicit actions use `generation = max_seen + 1` (`ProgressRecord::explicit_generation`): starting a listening session, - seeking, "start over", "mark played", "mark unplayed", handing playback to - another device. `max_seen` is the generation of the current register, - including received records. + seeking, "start over", "mark played", "mark unplayed", taking over playback + through a handoff. `max_seen` is the generation of the current register, + including received records. An explicit action during a session becomes + the generation of that session: its later checkpoints use it. * Periodic checkpoints and pause/stop saves within a session keep the generation of that session's latest explicit action. +* Handoff: queue items carry `PodcastEpisodeRef.progress_generation`, the + greatest generation the sender has seen for the episode, because the + playback state can arrive before the sender's progress operations. The + sender writes a final checkpoint at its session generation and does not + bump. Only the receiver writes the explicit action, with + `explicit_generation(max(local, progress_generation))`, so its writes order + after the sender's whatever the clocks say. +* A device that observes a greater generation for the episode it is playing + stops writing at its old generation: either playback moved away (the + coordination claim says so), or the user keeps listening locally and the + device writes its next record as an explicit action + (`explicit_generation(max_seen)`). * Write progress periodically (every 15–30 s) and on pause, seek, stop, - episode change and handoff, in the background. Coalesce pending checkpoints - so the outgoing log keeps only the latest per episode. + episode change and handoff, in the background. The outgoing log may drop a + pending progress operation only when another pending operation of the same + episode `supersedes` it (greater `(generation, hlc_ms, device_id, op_id)`). + Never coalesce by sequence order: a checkpoint of an older session queued + after "mark played" must not replace it. | Events | Result | | --- | --- | | A checkpoints session s1 at g3; B seeks back at g4 with a slower clock; A's late g3 checkpoint arrives | B's seek (g4) wins | | A restarts (g3, position 0); a queued g2 checkpoint of the old session arrives later | restart wins | | A and B both saw g5 and listen offline; both start sessions at g6 | the later `hlc_ms` (then `device_id`, `op_id`) wins on every device; both sessions are kept | +| A plays at g5 with a fast clock and hands off to B, which has only seen g2 | B starts at g6 (`max(2, 5) + 1`) and its checkpoints win | +| A marks played (g5), then a queued g3 checkpoint of its session is written | the outgoing log keeps g5; g3 is dominated | Ordinary Play resumes from `resume_position_ms()` (the position when in progress, otherwise 0). "Start over" is a separate explicit action. Publisher @@ -267,28 +334,65 @@ versions merge to the greatest `(completed, listened_ms, ended_at_ms, ...)`. ## Trusted-device sync (protocol v4) The ALPN remains `furumi/sync/2`; device profiles advertise -`protocol_version = 4`. New operation kinds: +`protocol_version = 4`. New operation kinds (none of them is a tombstone): -| Kind | Payload | Materialization | Tombstone | -| --- | --- | --- | --- | -| `podcast_subscription_set` | `podcast_id`, `subscribed`, `podcast`, `feed_aliases` | `SyncOpWire::podcast_subscription` | no | -| `podcast_share_set` | `podcast_id`, `shared` | `SyncOpWire::podcast_share` | `shared = false` | -| `podcast_progress_set` | `episode_id`, `podcast_id`, `position_ms`, `status`, `generation`, `session_id`, optional `episode` ref | `SyncOpWire::podcast_progress` | no | -| `podcast_listen_recorded` | `session`, optional `episode` ref | `SyncOpWire::podcast_listen_session` | no | +| Kind | Payload | Materialization | +| --- | --- | --- | +| `podcast_subscription_set` | `podcast_id`, `subscribed`, `podcast`, `feed_aliases`, `generation` | `SyncOpWire::podcast_subscription` | +| `podcast_share_set` | `podcast_id`, `shared`, `generation` | `SyncOpWire::podcast_share` | +| `podcast_feed_aliases_added` | `podcast_id`, `feed_aliases` (1..9 URLs) | `SyncOpWire::podcast_feed_aliases` | +| `podcast_progress_set` | `episode_id`, `podcast_id`, `position_ms`, `status`, `generation`, `session_id`, optional `episode` ref | `SyncOpWire::podcast_progress` | +| `podcast_listen_recorded` | `session`, optional `episode` ref | `SyncOpWire::podcast_listen_session` | Register stamps come from the operation: `hlc_ms`, `op_id`, and for progress `device_id = origin_device_id`. A session's `device_id` must equal the operation origin. `SyncOpWire::validate_podcast` checks this and the bounds; -receivers skip invalid podcast operations but still acknowledge them, so the +receivers skip operations that fail it but still acknowledge them, so the log keeps moving. -`DeviceProfileSet` gains an optional `protocol_version`. Each device reports -its own version. Receivers store -`profile_protocol_version(reported, previously_known)` and relays forward the -stored version of other devices in `DeviceProfileWire.protocol_version`. They -must never substitute their own constant: version gating would then send -podcast state to an old client. A device that never reported a version is -treated as version 1. +Ids (`PodcastId`, `EpisodeId`) and feed URLs (`FeedUrl`) are validated while +the message is deserialized: a malformed id or a non-canonical feed URL is a +schema violation, like a wrong JSON type, and the **whole message** fails to +parse. Conforming clients cannot produce one, because these types serialize +only canonical values, and their rules are frozen with the `v1` derivation +domains (changing them needs a new protocol version, see Versioning). + +### Peer versions + +Each device reports its own version: in the `profile` of its `Hello`, +`PairRequest` and `PairResponse`, in its own entry of the `devices` list (a +`SyncResponse` has no `profile`, so an initiator learns the responder's +version from the entry whose `device_id` is the responder), and in +`DeviceProfileSet.protocol_version` of operations it originates. A client +emits a `DeviceProfileSet` with `protocol_version` on the first start of a +new protocol version. + +* A self-report replaces the stored version: + `profile_protocol_version(reported, stored)` (`None` keeps the stored + value). +* Profiles of other devices relayed in `devices` only raise it: + `relayed_protocol_version(relayed, stored)` = `max(stored, relayed)`, or + `relayed` when nothing is stored. Deployed v1–v3 clients relay every + device with their own constant, so a relayed 3 must not downgrade a v4 + peer. +* Relays forward the stored version of other devices in + `DeviceProfileWire.protocol_version` and must never substitute their own + constant: version gating would then send podcast state to an old client. +* A device that never reported a version is treated as version 1. + Downgrading a device below v4 is not supported while relays still hold + v4 for it. + +### Pairing + +The invite consumer sends its `PairRequest` (operations, snapshot, playback) +before it hears from the inviter. Invites created by v4 clients carry the +inviter's version in `InviteWire.protocol_version` (JSON key `p`, omitted by +older clients). The consumer gates the whole request by +`InviteWire::inviter_protocol_version()`, which is version 1 for invites +without it: no podcast operations, `snapshot.for_peer(version)`, and no +playback snapshot with podcast items. Podcast state then catches up through +the snapshot of the next regular sync. The inviter gates its `PairResponse` +by the version in the request's `profile`. ### Snapshot recovery @@ -296,20 +400,33 @@ Sync vectors only grow, so operations hidden from a peer while it ran an older version are never re-requested after it upgrades. Podcast state is therefore part of `SyncSnapshot`: -| Section | Content | Limit | -| --- | --- | --- | -| `podcast_subscriptions` | all subscription registers, including unsubscribed | 1000 | -| `podcast_shares` | all share registers, including `shared = false` | 1000 | -| `podcast_progress` | progress registers | 5000 | -| `podcast_listen_sessions` | most recent sessions | 2000 | +| Section | Content | Limit | Budget share | +| --- | --- | --- | --- | +| `podcast_subscriptions` | subscription registers, including unsubscribed, with feed aliases; `podcast.description` is omitted | 1000 | 25 % | +| `podcast_shares` | share registers, including `shared = false` | 1000 | 5 % | +| `podcast_progress` | progress registers | 5000 | 45 % | +| `podcast_listen_sessions` | most recent sessions | 2000 | 25 % | -`SyncSnapshot::bound_podcast_state` orders each section newest first, caps it -and fits all sections into a 4 MiB JSON budget in the priority order above, -leaving room for music state inside the 8 MiB line limit. Records that do not -fit still propagate through operations. Receivers ignore podcast sections -that exceed the counts (`podcast_state_within_limits`), validate each record -and merge it with the same functions as operations. A new or long-absent -device thus converges from one snapshot. +`SyncSnapshot::bound_podcast_state_to(budget)` orders each section newest +first (registers by `(generation, hlc_ms)`, sessions by `ended_at_ms`), caps +it by count and by its share of the JSON byte budget; unused budget passes on +to the following sections in table order, so large subscription records +cannot crowd out progress. `bound_podcast_state()` uses the default 4 MiB +budget. Senders whose other message parts are large pass +`podcast_snapshot_budget(other_bytes)`, which keeps the whole line under the +8 MiB limit. Channel descriptions are not snapshotted: every device refreshes +channel metadata from the feed. + +Receivers ignore podcast sections that exceed the counts +(`podcast_state_within_limits`), validate each record and merge it with the +same functions as operations. + +Limitations: records cut by the bounds reach a peer only through operations +it has not acknowledged yet. A peer that upgrades from v3, or that received +operations through a v3 relay, already skipped the podcast operations: it +recovers only the newest records that fit, so older progress registers and +listening sessions are missing on that device and its play counts +(`completed_listen_count`) can differ from other devices. Empty sections are not serialized, so snapshots without podcast state are byte-identical to v3. @@ -341,10 +458,14 @@ as 1.0. When implemented, position extrapolation and handoff must use it. ## Version gating for peers below v4 -`SyncOpPayload` and `PlaybackSource` are closed serde enums. A v3 client that -receives an unknown `kind` fails to parse the **whole** message, not one -operation, which would stall synchronization. Additive `serde(default)` -fields alone do not make this compatible. Therefore: +`SyncOpPayload` is a closed serde enum. A v3 client that receives an unknown +operation `kind` fails to parse the **whole** message, not one operation, +which would stall synchronization. A v3 client does not know +`PlaybackTrack.source` and ignores it: it would treat an episode as a broken +music item (id 0, no content id) and forward the queue that way, losing the +episode for every device. Additive `serde(default)` fields alone therefore do +not make podcasts compatible. The peer version comes from "Peer versions" +above, and from the invite while pairing. Therefore: 1. Never send podcast operations to a peer below v4 (`op_allowed_for_peer`). @@ -359,6 +480,9 @@ fields alone do not make this compatible. Therefore: state would silently drop them. 5. Music operations and music-only playback keep working with v1–v3 peers. +`PlaybackSource` is closed for v4 clients as well: a v4 client fails to parse +a message with an unknown source `kind`. + ## Jam privacy Jam participants are outside the trusted-device group. @@ -383,7 +507,9 @@ limits, lenient dates and durations, relative media URLs resolved against the fetch URL. Items without an absolute `http(s)` audio URL are dropped. It extracts `itunes:season`, `itunes:episode`, `itunes:episodeType`, `guid isPermaLink` and `itunes:new-feed-url`, and produces identity hints for -`resolve_episode_ids`. It performs no I/O. +`resolve_episode_ids`; those hints are normative (see the no-GUID rule). +Links are kept verbatim (trimmed only) because they are identity material. +Text cleanup is linear in the input size. It performs no I/O. ## Versioning @@ -396,5 +522,8 @@ extracts `itunes:season`, `itunes:episode`, `itunes:episodeType`, domains, new prefixes, a migration and a new protocol version. * Adding an operation kind or a `PlaybackSource` variant requires a new protocol version and gating as above. +* Feed URL canonicalization, the identity bounds (`MAX_URL_BYTES`, + `MAX_GUID_BYTES`) and the no-GUID rule are frozen with the `v1` domains: + receivers reject non-canonical ids and URLs while parsing. * Public podcast cards will use a separate signed overlay with its own ALPN and version; the music DHT schema and `federation-net` do not change. diff --git a/crates/music-dht/README.md b/crates/music-dht/README.md index d807ffa..e2f16dd 100644 --- a/crates/music-dht/README.md +++ b/crates/music-dht/README.md @@ -226,8 +226,8 @@ let generation = ProgressRecord::explicit_generation(Some(4)); // 5 self-contained queue item. * `SubscriptionRecord`, `ShareRecord`, `ProgressRecord`, `ListenSession` with `merge_subscription`, `merge_share`, `merge_progress`, - `merge_listen_session`, `effective_share`, `reaches_completion` and - `completed_listen_count`. + `merge_listen_session`, `next_podcast_generation`, `effective_share`, + `reaches_completion` and `completed_listen_count`. Enable the `podcast-feed` feature for `music_dht::podcast::feed`, the shared tolerant RSS 2.0 / RSS 1.0 / Atom parser: @@ -237,12 +237,16 @@ music-dht = { version = "0.6", features = ["podcast-feed"] } ``` Device-sync protocol v4 adds `podcast_subscription_set`, `podcast_share_set`, -`podcast_progress_set` and `podcast_listen_recorded`, podcast snapshot -sections, `PlaybackTrack.source` and a reserved `playback_rate`. Old clients -cannot parse these, so always filter with `op_allowed_for_peer`, +`podcast_feed_aliases_added`, `podcast_progress_set` and +`podcast_listen_recorded`, podcast snapshot sections, `PlaybackTrack.source` +and a reserved `playback_rate`. Old clients cannot parse the new operation +kinds and ignore `source`, so always filter with `op_allowed_for_peer`, `playback_allowed_for_peer`, `SyncSnapshot::for_peer` and `playback_command_allowed_from_peer`, using the protocol version each device -reported itself (`profile_protocol_version`). Jam hosts use +reported itself (`profile_protocol_version`; relayed profiles go through +`relayed_protocol_version`, pairing requests use +`InviteWire::inviter_protocol_version`). Bound snapshots with +`SyncSnapshot::bound_podcast_state_to(podcast_snapshot_budget(..))`. Jam hosts use `jam::redact_jam_snapshot`, `jam_command_allowed` and `restore_jam_private_items`. -- 2.54.0