From 31d132bb3f90e1425379ba045682713ea25259ee Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 15:40:19 +0200 Subject: [PATCH] Add podcast-feed RSS and Atom parser Port the tolerant RSS 2.0 / RSS 1.0 / Atom parser of the original furumi-tui podcast work into music_dht::podcast::feed behind the optional podcast-feed feature (quick-xml, encoding_rs, url). Episode identity depends on what the parser extracts, so it must be shared by all clients. The port keeps its robustness (BOM and declared encodings, entities, mismatched tags, depth and size limits, lenient dates, http(s)-only media) and adds itunes:season, itunes:episode, itunes:episodeType, guid isPermaLink and itunes:new-feed-url. Missing GUIDs stay absent instead of falling back to the enclosure URL, and parsed items produce identity hints and contract metadata. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 29 + Cargo.toml | 3 + crates/music-dht/Cargo.toml | 7 + crates/music-dht/src/podcast/feed.rs | 1775 ++++++++++++++++++++ crates/music-dht/src/podcast/feed_tests.rs | 665 ++++++++ crates/music-dht/src/podcast/mod.rs | 9 + 6 files changed, 2488 insertions(+) create mode 100644 crates/music-dht/src/podcast/feed.rs create mode 100644 crates/music-dht/src/podcast/feed_tests.rs diff --git a/Cargo.lock b/Cargo.lock index fe24c1d..9c9ccc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,6 +389,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -790,6 +796,20 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "enum-assoc" version = "1.3.0" @@ -2071,6 +2091,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "music-dht" version = "0.5.0" @@ -2079,10 +2105,12 @@ dependencies = [ "async-trait", "blake3", "data-encoding", + "encoding_rs", "federation-net", "futures", "iroh", "postcard", + "quick-xml", "rand 0.9.5", "rusqlite", "serde", @@ -2092,6 +2120,7 @@ dependencies = [ "tokio", "tracing", "unicode-normalization", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 568c7b7..86aa1c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,3 +38,6 @@ lofty = "0.22" data-encoding = "2" rand = "0.9" tempfile = "3" +quick-xml = "0.41" +encoding_rs = "0.8.35" +url = "2.5" diff --git a/crates/music-dht/Cargo.toml b/crates/music-dht/Cargo.toml index a376ffd..cec4116 100644 --- a/crates/music-dht/Cargo.toml +++ b/crates/music-dht/Cargo.toml @@ -26,6 +26,13 @@ thiserror = { workspace = true } tracing = { workspace = true } rand = { workspace = true } data-encoding = { workspace = true } +quick-xml = { workspace = true, optional = true } +encoding_rs = { workspace = true, optional = true } +url = { workspace = true, optional = true } + +[features] +# RSS 2.0 / RSS 1.0 / Atom podcast feed parser (`music_dht::podcast::feed`). +podcast-feed = ["dep:quick-xml", "dep:encoding_rs", "dep:url"] [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/music-dht/src/podcast/feed.rs b/crates/music-dht/src/podcast/feed.rs new file mode 100644 index 0000000..8da47b8 --- /dev/null +++ b/crates/music-dht/src/podcast/feed.rs @@ -0,0 +1,1775 @@ +//! Shared tolerant RSS 2.0 / RSS 1.0 / Atom parser for podcast feeds. +//! +//! Available with the `podcast-feed` cargo feature. Every Furumi client uses +//! this parser because episode identity depends on what it extracts. +//! +//! Real-world feeds are frequently sloppy: undeclared namespace prefixes, +//! HTML entities that XML does not define, mismatched closing tags, legacy +//! single-byte encodings and creative date formats. The parser is a single +//! iterative pass over `quick-xml` events (no recursion), keeps only the +//! fields Furumi needs, bounds every captured value and never panics on +//! malformed input. It performs no I/O: fetching, redirects, conditional +//! requests and scheduling belong to the client runtime. +//! +//! ``` +//! use music_dht::podcast::feed::parse_feed_with_base; +//! use music_dht::podcast::{FeedUrl, PodcastId}; +//! +//! let xml = br#"Show +//! Pilotep-1 +//! +//! "#; +//! let url = FeedUrl::parse("https://example.test/feed.xml").unwrap(); +//! let feed = parse_feed_with_base(xml, Some(url.as_str())).unwrap(); +//! let podcast = PodcastId::from_feed_url(&url); +//! let ids = feed.resolve_episode_ids(&podcast, &[]); +//! let episode = feed.episodes[0].to_meta(&podcast, ids[0].clone().unwrap()); +//! assert_eq!(episode.audio_url, "https://example.test/audio/1.mp3"); +//! ``` + +use std::borrow::Cow; +use std::collections::HashSet; + +use quick_xml::events::{BytesStart, Event}; +use quick_xml::reader::Reader; +use url::Url; + +use super::{ + EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode, + MAX_DESCRIPTION_CHARS, MAX_GUID_BYTES, MAX_TITLE_CHARS, PodcastError, PodcastId, PodcastMeta, + PodcastResult, is_http_url, resolve_episode_ids, +}; + +/// Upper bound on episodes taken from a single feed document. +pub const MAX_EPISODES: usize = 20_000; +/// Raw text captured for one element is capped to keep memory bounded on +/// hostile or broken documents. +const MAX_CAPTURE_BYTES: usize = 512 * 1024; +/// Elements nested deeper than this are skipped (their events are ignored). +const MAX_DEPTH: usize = 256; +/// Durations longer than this (about 115 days) are treated as garbage. +const MAX_DURATION_SECONDS: f64 = 10_000_000.0; +const UNTITLED_EPISODE: &str = "Untitled episode"; +const UNTITLED_PODCAST: &str = "Untitled podcast"; + +/// Channel-level result of [`parse_feed`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedFeed { + /// Podcast title (never empty). + pub title: String, + /// Author (`itunes:author`, Atom author, `dc:creator`, `managingEditor`). + pub author: Option, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + pub description: Option, + /// Publisher web page. + pub link: Option, + /// Absolute http(s) artwork URL. + pub image_url: Option, + /// `itunes:new-feed-url`, verbatim. Clients canonicalize it with + /// [`FeedUrl::parse`] and record it as a feed alias. + pub new_feed_url: Option, + /// Playable items in document order, duplicates removed. + pub episodes: Vec, +} + +/// One playable feed item. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedEpisode { + /// RSS `` / Atom ``, trimmed; `None` when absent, blank or + /// longer than [`MAX_GUID_BYTES`]. + pub guid: Option, + /// RSS `guid isPermaLink` (default `true` per RSS 2.0); `None` without + /// an RSS GUID. Informational: identity uses the GUID text either way. + pub guid_is_permalink: Option, + /// Title (never empty). + pub title: String, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + pub description: Option, + /// Publication time in unix milliseconds. + pub published_at_ms: Option, + /// Announced duration in milliseconds. + pub duration_ms: Option, + /// `itunes:season`. + pub season: Option, + /// `itunes:episode`. + pub episode_number: Option, + /// `itunes:episodeType`. + pub episode_type: EpisodeType, + /// Absolute http(s) audio URL (never a local file). + pub audio_url: String, + /// Enclosure MIME type, lowercased, without parameters. + pub mime_type: Option, + /// Enclosure length in bytes. + pub byte_len: Option, + /// Item web page. + pub link: Option, + /// Absolute http(s) artwork URL. + pub image_url: Option, +} + +impl ParsedEpisode { + /// Identity hints for [`resolve_episode_ids`]. + pub fn identity_hints(&self) -> EpisodeIdentityHints { + EpisodeIdentityHints { + guid: self.guid.clone(), + enclosure_url: Some(self.audio_url.clone()), + link: self.link.clone(), + published_at_ms: self.published_at_ms, + title: Some(self.title.clone()), + } + } + + /// Contract metadata for this item under a resolved id. + pub fn to_meta(&self, podcast_id: &PodcastId, episode_id: EpisodeId) -> EpisodeMeta { + EpisodeMeta { + episode_id, + podcast_id: podcast_id.clone(), + guid: self.guid.clone(), + title: self.title.clone(), + description: self.description.clone(), + season: self.season, + episode_number: self.episode_number, + episode_type: self.episode_type, + published_at_ms: self.published_at_ms, + duration_ms: self.duration_ms, + audio_url: self.audio_url.clone(), + mime: self.mime_type.clone(), + byte_len: self.byte_len, + image_url: self.image_url.clone(), + link: self + .link + .clone() + .filter(|link| link.len() <= super::MAX_URL_BYTES), + } + } +} + +impl ParsedFeed { + /// Identity hints of every episode, in order. + pub fn identity_hints(&self) -> Vec { + self.episodes + .iter() + .map(ParsedEpisode::identity_hints) + .collect() + } + + /// Resolves stable ids for every episode (see [`resolve_episode_ids`]). + pub fn resolve_episode_ids( + &self, + podcast_id: &PodcastId, + known: &[KnownEpisode], + ) -> Vec> { + resolve_episode_ids( + podcast_id, + self.link.as_deref(), + &self.identity_hints(), + known, + ) + } + + /// Contract channel metadata for a feed fetched from `feed_url`. + pub fn to_podcast_meta(&self, feed_url: FeedUrl) -> PodcastMeta { + PodcastMeta { + feed_url, + title: self.title.clone(), + author: self.author.clone(), + description: self.description.clone(), + link: self.link.clone(), + image_url: self.image_url.clone(), + } + .bounded() + } +} + +/// Parses an RSS or Atom document. Fails for documents that are not feeds +/// (for example an HTML page) with "not an RSS or Atom feed". +/// +/// Without a base URL, relative media URLs cannot be resolved and such +/// episodes are dropped; prefer [`parse_feed_with_base`] when the feed URL is +/// known. +pub fn parse_feed(bytes: &[u8]) -> PodcastResult { + parse_feed_with_base(bytes, None) +} + +/// [`parse_feed`] resolving relative enclosure and image URLs against +/// `base_url` (the final URL the feed was fetched from). Episodes whose audio +/// URL does not resolve to an absolute `http`/`https` URL are dropped, so a +/// feed can never point playback at a local file. +pub fn parse_feed_with_base(bytes: &[u8], base_url: Option<&str>) -> PodcastResult { + let text = decode_document(bytes); + let parser = Parser { + base: base_url.and_then(|base| Url::parse(base).ok()), + ..Parser::default() + }; + parser.run(&text) +} + +fn not_a_feed(detail: impl std::fmt::Display) -> PodcastError { + PodcastError::NotAFeed(format!(" ({detail})")) +} + +// --------------------------------------------------------------------------- +// Character encoding + +/// Decodes the raw document to UTF-8. A BOM wins; otherwise an ASCII-compatible +/// encoding named in the XML declaration (windows-1251, koi8-r, ...) is +/// honoured; everything else is decoded as lossy UTF-8. +fn decode_document(bytes: &[u8]) -> Cow<'_, str> { + if let Some((encoding, bom_len)) = encoding_rs::Encoding::for_bom(bytes) { + let (text, _) = encoding.decode_without_bom_handling(&bytes[bom_len..]); + return text; + } + if let Some(encoding) = declared_encoding(bytes).and_then(encoding_rs::Encoding::for_label) + && encoding != encoding_rs::UTF_8 + && encoding.is_ascii_compatible() + { + let (text, _) = encoding.decode_without_bom_handling(bytes); + return text; + } + String::from_utf8_lossy(bytes) +} + +/// The `encoding="..."` label of a leading XML declaration, if any. +fn declared_encoding(bytes: &[u8]) -> Option<&[u8]> { + let head = &bytes[..bytes.len().min(512)]; + let start = head.iter().position(|b| !b.is_ascii_whitespace())?; + let head = &head[start..]; + if !head.starts_with(b"")?; + let decl = &head[..end]; + let at = find(decl, b"encoding")? + b"encoding".len(); + let rest = &decl[at..]; + let rest = &rest[rest.iter().position(|b| !b.is_ascii_whitespace())?..]; + let rest = rest.strip_prefix(b"=")?; + let rest = &rest[rest.iter().position(|b| !b.is_ascii_whitespace())?..]; + let quote = *rest.first()?; + if quote != b'"' && quote != b'\'' { + return None; + } + let rest = &rest[1..]; + let len = rest.iter().position(|&b| b == quote)?; + Some(&rest[..len]) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +// --------------------------------------------------------------------------- +// Namespaces + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ns { + /// No namespace, or the RSS 0.9x/1.0 namespace. + Plain, + Atom, + Itunes, + Content, + Media, + Dc, + Other, +} + +fn classify_uri(uri: &str) -> Ns { + let uri = uri.trim().to_ascii_lowercase(); + if uri.is_empty() { + Ns::Plain + } else if uri.contains("w3.org/2005/atom") { + Ns::Atom + } else if uri.contains("itunes.com/dtds/podcast") { + Ns::Itunes + } else if uri.contains("purl.org/rss/1.0/modules/content") { + Ns::Content + } else if uri.contains("search.yahoo.com/mrss") { + Ns::Media + } else if uri.contains("purl.org/dc/elements") { + Ns::Dc + } else if uri.contains("purl.org/rss/1.0") + || uri.contains("my.netscape.com/rdf") + || uri.contains("backend.userland.com/rss") + { + Ns::Plain + } else { + Ns::Other + } +} + +/// Classification of a prefix nobody declared (common in sloppy feeds). +fn classify_prefix(prefix: &str) -> Ns { + match prefix.to_ascii_lowercase().as_str() { + "" => Ns::Plain, + "itunes" => Ns::Itunes, + "content" => Ns::Content, + "media" => Ns::Media, + "dc" => Ns::Dc, + "atom" | "atom10" => Ns::Atom, + _ => Ns::Other, + } +} + +// --------------------------------------------------------------------------- +// Parser state + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Rss, + Atom, +} + +#[derive(Debug)] +struct Elem { + ns: Ns, + /// ASCII-lowercased local name. + local: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Field { + FeedTitle, + FeedDescription, + FeedItunesSummary, + FeedLink, + FeedItunesAuthor, + FeedAtomAuthor, + FeedDcCreator, + FeedManagingEditor, + FeedImageUrl, + FeedLogo, + FeedIcon, + FeedNewFeedUrl, + ItemTitle, + ItemItunesTitle, + ItemDescription, + ItemContent, + ItemSummary, + ItemGuid, + ItemPubDate, + ItemDcDate, + ItemUpdated, + ItemDuration, + ItemLink, + ItemSeason, + ItemEpisode, + ItemEpisodeType, +} + +#[derive(Debug)] +struct Capture { + /// Stack depth of the captured element. + depth: usize, + field: Field, + text: String, +} + +#[derive(Debug, Default)] +struct FeedAcc { + title: Option, + description: Option, + itunes_summary: Option, + link: Option, + itunes_author: Option, + atom_author: Option, + dc_creator: Option, + managing_editor: Option, + itunes_image: Option, + image_url: Option, + logo: Option, + icon: Option, + new_feed_url: Option, +} + +#[derive(Debug, Clone)] +struct Media { + url: String, + mime: Option, + len: Option, +} + +#[derive(Debug, Default)] +struct ItemAcc { + title: Option, + itunes_title: Option, + description: Option, + content: Option, + summary: Option, + guid: Option, + guid_is_permalink: Option, + pub_date: Option, + dc_date: Option, + updated: Option, + duration: Option, + season: Option, + episode: Option, + episode_type: Option, + link: Option, + enclosure: Option, + media_content: Option, + itunes_image: Option, + thumbnail: Option, +} + +#[derive(Debug, Default)] +struct Parser { + kind: Option, + stack: Vec, + /// Namespace declarations: (depth of declaring element, prefix, uri). + ns_decls: Vec<(usize, String, String)>, + /// Start tags ignored because they exceeded `MAX_DEPTH`. + skipped_depth: usize, + capture: Option, + feed: FeedAcc, + item: Option, + item_depth: usize, + episodes: Vec, + seen_guids: HashSet, + full: bool, + /// URL the document was fetched from, for relative media URLs. + base: Option, +} + +/// Sets `slot` unless it already holds a value (first occurrence wins). +fn set_once(slot: &mut Option, value: String) { + if slot.is_none() && !value.trim().is_empty() { + *slot = Some(value); + } +} + +impl Parser { + fn run(mut self, text: &str) -> PodcastResult { + let mut reader = Reader::from_str(text); + { + let config = reader.config_mut(); + config.check_end_names = false; + config.allow_unmatched_ends = true; + config.allow_dangling_amp = true; + config.expand_empty_elements = false; + config.trim_text(false); + } + let mut malformed = None; + loop { + match reader.read_event() { + Ok(Event::Eof) => break, + Ok(event) => { + self.handle(event)?; + if self.full { + break; + } + } + Err(err) => { + if self.kind.is_none() { + return Err(not_a_feed(format_args!("invalid XML: {err}"))); + } + malformed = Some(err.to_string()); + break; + } + } + } + if self.kind.is_none() { + return Err(PodcastError::NotAFeed(String::new())); + } + if let Some(err) = malformed { + if self.episodes.is_empty() { + return Err(PodcastError::MalformedFeed(err)); + } + tracing::warn!("podcast feed is malformed, keeping what parsed: {err}"); + } + Ok(self.finish()) + } + + fn handle(&mut self, event: Event<'_>) -> PodcastResult<()> { + match event { + Event::Start(start) => self.on_start(&start, false)?, + Event::Empty(start) => self.on_start(&start, true)?, + Event::End(end) => { + let name = end.name(); + let local = local_name(name.as_ref()); + self.on_end(&local); + } + Event::Text(text) => { + if let Some(capture) = self.capture.as_mut() { + match text.xml10_content() { + Ok(value) => push_capped(&mut capture.text, &value), + Err(_) => push_capped(&mut capture.text, &String::from_utf8_lossy(&text)), + } + } + } + Event::CData(data) => { + if let Some(capture) = self.capture.as_mut() { + match data.decode() { + Ok(value) => push_capped(&mut capture.text, &value), + Err(_) => push_capped(&mut capture.text, &String::from_utf8_lossy(&data)), + } + } + } + Event::GeneralRef(reference) => { + if let Some(capture) = self.capture.as_mut() { + let raw = String::from_utf8_lossy(&reference); + match resolve_entity(&raw) { + Some(ch) => push_capped(&mut capture.text, ch.encode_utf8(&mut [0; 4])), + None => { + // Keep unknown entities verbatim instead of failing. + push_capped(&mut capture.text, "&"); + push_capped(&mut capture.text, &raw); + push_capped(&mut capture.text, ";"); + } + } + } + } + Event::Comment(_) | Event::Decl(_) | Event::PI(_) | Event::DocType(_) | Event::Eof => {} + } + Ok(()) + } + + fn resolve_ns(&self, prefix: &str) -> Ns { + for (_, declared, uri) in self.ns_decls.iter().rev() { + if declared == prefix { + return classify_uri(uri); + } + } + classify_prefix(prefix) + } + + fn on_start(&mut self, start: &BytesStart<'_>, empty: bool) -> PodcastResult<()> { + if self.stack.len() >= MAX_DEPTH { + if !empty { + self.skipped_depth += 1; + } + return Ok(()); + } + let depth = self.stack.len() + 1; + let attrs = Attrs::collect(start); + for (key, value) in &attrs.raw { + if key == "xmlns" { + self.ns_decls.push((depth, String::new(), value.clone())); + } else if let Some(prefix) = key.strip_prefix("xmlns:") { + self.ns_decls + .push((depth, prefix.to_string(), value.clone())); + } + } + let qname = start.name(); + let (prefix, local) = split_qname(qname.as_ref()); + let mut ns = self.resolve_ns(&prefix); + if self.kind == Some(Kind::Rss) && prefix.is_empty() && matches!(ns, Ns::Atom | Ns::Other) { + // RSS has no namespace of its own; an unprefixed element is a core + // RSS element whatever default namespace the publisher declared. + ns = Ns::Plain; + } + self.stack.push(Elem { ns, local }); + + if self.kind.is_none() { + let local = self.stack[0].local.as_str(); + self.kind = match local { + "rss" | "rdf" => Some(Kind::Rss), + "feed" => Some(Kind::Atom), + _ => return Err(not_a_feed(format_args!("root element <{local}>"))), + }; + } else if let Some(capture) = self.capture.as_mut() { + // Markup nested in a captured field (e.g. Atom xhtml content): + // keep words apart. + push_capped(&mut capture.text, " "); + } else { + self.dispatch_start(&attrs); + } + + if empty { + self.close_top(); + } + Ok(()) + } + + fn dispatch_start(&mut self, attrs: &Attrs) { + let Some(kind) = self.kind else { return }; + let depth = self.stack.len(); + let local_at = |back: usize| { + depth + .checked_sub(back) + .and_then(|index| self.stack.get(index)) + .map(|elem| elem.local.clone()) + .unwrap_or_default() + }; + let ns = self.stack[depth - 1].ns; + let local = local_at(1); + let parent = local_at(2); + let grandparent = local_at(3); + let core = match kind { + Kind::Rss => ns == Ns::Plain, + Kind::Atom => matches!(ns, Ns::Plain | Ns::Atom), + }; + + if self.item.is_some() { + let direct = depth == self.item_depth + 1; + if let Some(field) = self.item_field(kind, ns, core, &local, direct, attrs) { + self.begin_capture(field); + } + return; + } + + let is_item = match kind { + Kind::Rss => core && local == "item", + Kind::Atom => core && local == "entry", + }; + if is_item { + self.item = Some(ItemAcc::default()); + self.item_depth = depth; + return; + } + + let feed_level = match kind { + Kind::Rss => parent == "channel", + Kind::Atom => depth == 2, + }; + let field = if feed_level { + self.feed_field(kind, ns, core, &local, attrs) + } else if kind == Kind::Rss + && core + && local == "url" + && parent == "image" + && matches!(grandparent.as_str(), "channel" | "rdf") + { + Some(Field::FeedImageUrl) + } else if kind == Kind::Atom && core && local == "name" && parent == "author" && depth == 3 + { + Some(Field::FeedAtomAuthor) + } else { + None + }; + if let Some(field) = field { + self.begin_capture(field); + } + } + + fn feed_field( + &mut self, + kind: Kind, + ns: Ns, + core: bool, + local: &str, + attrs: &Attrs, + ) -> Option { + if core { + return match (kind, local) { + (_, "title") => Some(Field::FeedTitle), + (Kind::Rss, "description") | (Kind::Atom, "subtitle") => { + Some(Field::FeedDescription) + } + (Kind::Rss, "link") => Some(Field::FeedLink), + (Kind::Rss, "managingeditor") => Some(Field::FeedManagingEditor), + (Kind::Atom, "link") => { + let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); + if rel.is_empty() || rel == "alternate" { + let href = attrs.get("href").unwrap_or_default(); + set_once(&mut self.feed.link, clean_short(&href)); + } + None + } + (Kind::Atom, "logo") => Some(Field::FeedLogo), + (Kind::Atom, "icon") => Some(Field::FeedIcon), + _ => None, + }; + } + match (ns, local) { + (Ns::Itunes, "author") => Some(Field::FeedItunesAuthor), + (Ns::Itunes, "summary") => Some(Field::FeedItunesSummary), + (Ns::Itunes, "image") => { + if let Some(href) = attrs.get("href") { + set_once(&mut self.feed.itunes_image, clean_short(&href)); + } + None + } + (Ns::Itunes, "new-feed-url") => Some(Field::FeedNewFeedUrl), + (Ns::Dc, "creator") => Some(Field::FeedDcCreator), + _ => None, + } + } + + fn item_field( + &mut self, + kind: Kind, + ns: Ns, + core: bool, + local: &str, + direct: bool, + attrs: &Attrs, + ) -> Option { + let item = self.item.as_mut()?; + // Media RSS may wrap content in , so accept any depth. + if ns == Ns::Media { + match local { + "content" => { + let url = attrs.get("url").unwrap_or_default(); + let mime = attrs.get("type"); + let medium = attrs.get("medium"); + if item.media_content.is_none() + && is_playable(&url, mime.as_deref(), medium.as_deref(), false) + { + item.media_content = Some(Media { + url: url.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("filesize")), + }); + } + } + "thumbnail" => { + if let Some(url) = attrs.get("url") { + set_once(&mut item.thumbnail, clean_short(&url)); + } + } + _ => {} + } + return None; + } + if !direct { + return None; + } + if core { + return match (kind, local) { + (_, "title") => Some(Field::ItemTitle), + (Kind::Rss, "description") => Some(Field::ItemDescription), + (Kind::Rss, "guid") => { + if item.guid_is_permalink.is_none() { + let permalink = attrs.get("ispermalink").unwrap_or_default(); + item.guid_is_permalink = + Some(!permalink.trim().eq_ignore_ascii_case("false")); + } + Some(Field::ItemGuid) + } + (Kind::Atom, "id") => Some(Field::ItemGuid), + (Kind::Rss, "pubdate") | (Kind::Atom, "published") => Some(Field::ItemPubDate), + (Kind::Atom, "updated") => Some(Field::ItemUpdated), + (Kind::Rss, "link") => Some(Field::ItemLink), + (Kind::Atom, "summary") => Some(Field::ItemDescription), + (Kind::Atom, "content") => Some(Field::ItemContent), + (Kind::Rss, "enclosure") => { + let url = attrs.get("url").unwrap_or_default(); + let mime = attrs.get("type"); + if item.enclosure.is_none() && is_playable(&url, mime.as_deref(), None, true) { + item.enclosure = Some(Media { + url: url.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("length")), + }); + } + None + } + (Kind::Atom, "link") => { + let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); + let href = attrs.get("href").unwrap_or_default(); + let mime = attrs.get("type"); + if rel == "enclosure" { + if item.enclosure.is_none() + && is_playable(&href, mime.as_deref(), None, true) + { + item.enclosure = Some(Media { + url: href.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("length")), + }); + } + } else if rel.is_empty() || rel == "alternate" { + set_once(&mut item.link, clean_short(&href)); + } + None + } + _ => None, + }; + } + match (ns, local) { + (Ns::Itunes, "title") => Some(Field::ItemItunesTitle), + (Ns::Itunes, "summary") => Some(Field::ItemSummary), + (Ns::Itunes, "duration") => Some(Field::ItemDuration), + (Ns::Itunes, "season") => Some(Field::ItemSeason), + (Ns::Itunes, "episode") => Some(Field::ItemEpisode), + (Ns::Itunes, "episodetype") => Some(Field::ItemEpisodeType), + (Ns::Itunes, "image") => { + if let Some(href) = attrs.get("href") { + set_once(&mut item.itunes_image, clean_short(&href)); + } + None + } + (Ns::Content, "encoded") => Some(Field::ItemContent), + (Ns::Dc, "date") => Some(Field::ItemDcDate), + _ => None, + } + } + + fn begin_capture(&mut self, field: Field) { + self.capture = Some(Capture { + depth: self.stack.len(), + field, + text: String::new(), + }); + } + + fn on_end(&mut self, local: &str) { + if self.skipped_depth > 0 { + self.skipped_depth -= 1; + return; + } + // Tolerate mismatched end tags: close up to the nearest open element + // with the same local name, ignore the tag if none is open. + let Some(index) = self.stack.iter().rposition(|elem| elem.local == local) else { + return; + }; + while self.stack.len() > index { + self.close_top(); + } + } + + fn close_top(&mut self) { + let depth = self.stack.len(); + if depth == 0 { + return; + } + if self + .capture + .as_ref() + .is_some_and(|capture| capture.depth == depth) + && let Some(capture) = self.capture.take() + { + self.store_capture(capture); + } + if depth == self.item_depth + && let Some(item) = self.item.take() + { + self.finish_item(item); + } + while self + .ns_decls + .last() + .is_some_and(|(decl_depth, _, _)| *decl_depth >= depth) + { + self.ns_decls.pop(); + } + self.stack.pop(); + } + + fn store_capture(&mut self, capture: Capture) { + let text = capture.text; + if let Some(item) = self.item.as_mut() { + let slot = match capture.field { + Field::ItemTitle => &mut item.title, + Field::ItemItunesTitle => &mut item.itunes_title, + Field::ItemDescription => &mut item.description, + Field::ItemContent => &mut item.content, + Field::ItemSummary => &mut item.summary, + Field::ItemGuid => &mut item.guid, + Field::ItemPubDate => &mut item.pub_date, + Field::ItemDcDate => &mut item.dc_date, + Field::ItemUpdated => &mut item.updated, + Field::ItemDuration => &mut item.duration, + Field::ItemLink => &mut item.link, + Field::ItemSeason => &mut item.season, + Field::ItemEpisode => &mut item.episode, + Field::ItemEpisodeType => &mut item.episode_type, + _ => return, + }; + set_once(slot, text); + return; + } + let feed = &mut self.feed; + let slot = match capture.field { + Field::FeedTitle => &mut feed.title, + Field::FeedDescription => &mut feed.description, + Field::FeedItunesSummary => &mut feed.itunes_summary, + Field::FeedLink => &mut feed.link, + Field::FeedItunesAuthor => &mut feed.itunes_author, + Field::FeedAtomAuthor => &mut feed.atom_author, + Field::FeedDcCreator => &mut feed.dc_creator, + Field::FeedManagingEditor => &mut feed.managing_editor, + Field::FeedImageUrl => &mut feed.image_url, + Field::FeedLogo => &mut feed.logo, + Field::FeedIcon => &mut feed.icon, + Field::FeedNewFeedUrl => &mut feed.new_feed_url, + _ => return, + }; + set_once(slot, text); + } + + fn finish_item(&mut self, item: ItemAcc) { + if self.full { + return; + } + let base = self.base.as_ref(); + let Some(media) = [item.enclosure, item.media_content] + .into_iter() + .flatten() + .find_map(|media| { + let url = resolve_http_url(&media.url, base)?; + Some(Media { url, ..media }) + }) + else { + return; + }; + let guid = item + .guid + .as_deref() + .map(str::trim) + .filter(|guid| !guid.is_empty() && guid.len() <= MAX_GUID_BYTES) + .map(str::to_string); + // Duplicate items (same GUID, or same audio URL without GUID) keep + // the first occurrence. + let dedup_key = match &guid { + Some(guid) => format!("guid:{guid}"), + None => format!("enclosure:{}", media.url), + }; + if !self.seen_guids.insert(dedup_key) { + return; + } + let guid_is_permalink = if guid.is_some() { + item.guid_is_permalink + } else { + None + }; + let title = [item.title.as_deref(), item.itunes_title.as_deref()] + .into_iter() + .flatten() + .map(clean_short) + .find(|title| !title.is_empty()) + .unwrap_or_else(|| UNTITLED_EPISODE.to_string()); + let description = [ + item.description.as_deref(), + item.content.as_deref(), + item.summary.as_deref(), + ] + .into_iter() + .flatten() + .map(clean_long) + .find(|text| !text.is_empty()); + let published_at_ms = [ + item.pub_date.as_deref(), + item.dc_date.as_deref(), + item.updated.as_deref(), + ] + .into_iter() + .flatten() + .find_map(parse_date) + .map(|seconds| seconds.saturating_mul(1_000)); + self.episodes.push(ParsedEpisode { + guid, + guid_is_permalink, + title, + description, + published_at_ms, + duration_ms: item + .duration + .as_deref() + .and_then(parse_duration) + .filter(|seconds| *seconds <= MAX_DURATION_SECONDS) + .map(|seconds| (seconds * 1_000.0).round() as i64), + season: item.season.as_deref().and_then(parse_number), + episode_number: item.episode.as_deref().and_then(parse_number), + episode_type: item + .episode_type + .as_deref() + .map(EpisodeType::parse) + .unwrap_or_default(), + audio_url: media.url, + mime_type: media.mime, + byte_len: media.len, + link: item + .link + .as_deref() + .map(clean_short) + .filter(|s| !s.is_empty()), + image_url: [item.itunes_image, item.thumbnail] + .into_iter() + .flatten() + .find_map(|url| resolve_http_url(&url, base)), + }); + if self.episodes.len() >= MAX_EPISODES { + self.full = true; + } + } + + fn finish(self) -> ParsedFeed { + let feed = self.feed; + let title = feed + .title + .as_deref() + .map(clean_short) + .filter(|title| !title.is_empty()) + .unwrap_or_else(|| UNTITLED_PODCAST.to_string()); + let author = [ + feed.itunes_author.as_deref(), + feed.atom_author.as_deref(), + feed.dc_creator.as_deref(), + feed.managing_editor.as_deref(), + ] + .into_iter() + .flatten() + .map(clean_short) + .find(|author| !author.is_empty()); + let description = [feed.description.as_deref(), feed.itunes_summary.as_deref()] + .into_iter() + .flatten() + .map(clean_long) + .find(|text| !text.is_empty()); + let image_url = [feed.itunes_image, feed.image_url, feed.logo, feed.icon] + .into_iter() + .flatten() + .find_map(|url| resolve_http_url(&url, self.base.as_ref())); + ParsedFeed { + title, + author, + description, + link: feed + .link + .as_deref() + .map(clean_short) + .filter(|s| !s.is_empty()), + image_url, + new_feed_url: feed + .new_feed_url + .as_deref() + .map(|url| url.trim().to_string()) + .filter(|url| !url.is_empty() && url.len() <= super::MAX_URL_BYTES), + episodes: self.episodes, + } + } +} + +// --------------------------------------------------------------------------- +// Element and attribute helpers + +fn split_qname(name: &[u8]) -> (String, String) { + let name = String::from_utf8_lossy(name); + match name.split_once(':') { + Some((prefix, local)) => (prefix.to_string(), local.to_ascii_lowercase()), + None => (String::new(), name.to_ascii_lowercase()), + } +} + +fn local_name(name: &[u8]) -> String { + split_qname(name).1 +} + +/// Attributes of one element: raw keys with entity-decoded values. +struct Attrs { + raw: Vec<(String, String)>, +} + +impl Attrs { + fn collect(start: &BytesStart<'_>) -> Attrs { + let mut attributes = start.attributes(); + attributes.with_checks(false); + let raw = attributes + .flatten() + .map(|attr| { + let key = String::from_utf8_lossy(attr.key.as_ref()).into_owned(); + let value = decode_entities(&String::from_utf8_lossy(&attr.value)); + (key, value) + }) + .collect(); + Attrs { raw } + } + + /// Value of the attribute whose local name matches `name` (any prefix). + fn get(&self, name: &str) -> Option { + self.raw + .iter() + .find(|(key, _)| { + let local = key.rsplit_once(':').map_or(key.as_str(), |(_, l)| l); + local.eq_ignore_ascii_case(name) && !key.starts_with("xmlns") + }) + .map(|(_, value)| value.clone()) + } +} + +fn push_capped(text: &mut String, value: &str) { + if text.len() >= MAX_CAPTURE_BYTES { + return; + } + let room = MAX_CAPTURE_BYTES - text.len(); + if value.len() <= room { + text.push_str(value); + } else { + let mut end = room; + while !value.is_char_boundary(end) { + end -= 1; + } + text.push_str(&value[..end]); + } +} + +fn parse_len(value: Option) -> Option { + value?.trim().parse::().ok().filter(|len| *len > 0) +} + +fn clean_mime(mime: Option<&str>) -> Option { + let mime = mime?.split(';').next()?.trim().to_ascii_lowercase(); + (!mime.is_empty()).then_some(mime) +} + +const MEDIA_EXTENSIONS: &[&str] = &[ + "mp3", "m4a", "m4b", "aac", "ogg", "oga", "opus", "flac", "wav", "mp4", "m4v", "mov", "webm", + "mka", +]; + +fn has_media_extension(url: &str) -> bool { + let path = url.split(['?', '#']).next().unwrap_or_default(); + let file = path.rsplit('/').next().unwrap_or_default(); + file.rsplit_once('.').is_some_and(|(_, ext)| { + MEDIA_EXTENSIONS + .iter() + .any(|known| known.eq_ignore_ascii_case(ext)) + }) +} + +/// Whether an enclosure/media element points at something playable. +/// `lenient_untyped`: accept a URL without type information (RSS +/// enclosures); media:content without type must at least look like audio. +fn is_playable(url: &str, mime: Option<&str>, medium: Option<&str>, lenient_untyped: bool) -> bool { + let url = url.trim(); + if url.is_empty() { + return false; + } + if let Some(mime) = clean_mime(mime) { + return mime.starts_with("audio/") + || mime.starts_with("video/") + || matches!( + mime.as_str(), + "application/octet-stream" + | "binary/octet-stream" + | "application/ogg" + | "application/mp4" + | "application/x-mpegurl" + | "application/vnd.apple.mpegurl" + ) + || (mime == "application/x-unknown" && has_media_extension(url)); + } + if let Some(medium) = medium.map(|m| m.trim().to_ascii_lowercase()) + && !medium.is_empty() + { + return medium == "audio" || medium == "video"; + } + lenient_untyped || has_media_extension(url) +} + +/// An absolute `http`/`https` URL for a media or image reference: absolute +/// http(s) URLs are kept as written, relative ones (including +/// protocol-relative `//host/...`) are resolved against `base`, and anything +/// else (`file:`, `ftp:`, unresolvable relative paths) yields `None`. +fn resolve_http_url(raw: &str, base: Option<&Url>) -> Option { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + let has_host = |url: &Url| url.host_str().is_some_and(|host| !host.is_empty()); + if is_remote_url(raw) { + // Keep the publisher's spelling when it already satisfies the + // contract; otherwise use the serialized (percent-encoded) form. + let parsed = Url::parse(raw).ok().filter(has_host)?; + return if is_http_url(raw) { + Some(raw.to_string()) + } else { + Some(parsed.to_string()).filter(|url| is_http_url(url)) + }; + } + let resolved = match base { + Some(base) => base.join(raw).ok()?, + None if raw.starts_with("//") => Url::parse(&format!("https:{raw}")).ok()?, + None => return None, + }; + (matches!(resolved.scheme(), "http" | "https") && has_host(&resolved)) + .then(|| resolved.to_string()) + .filter(|url| is_http_url(url)) +} + +fn is_remote_url(value: &str) -> bool { + let value = value.as_bytes(); + let has_prefix = |prefix: &[u8]| { + value.len() >= prefix.len() && value[..prefix.len()].eq_ignore_ascii_case(prefix) + }; + has_prefix(b"http://") || has_prefix(b"https://") +} + +/// Parses a non-negative integer such as `itunes:season`; garbage yields `None`. +fn parse_number(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + text.parse().ok() +} + +// --------------------------------------------------------------------------- +// Text cleanup + +fn clean_short(text: &str) -> String { + html_to_text(text, MAX_TITLE_CHARS) +} + +fn clean_long(text: &str) -> String { + html_to_text(text, MAX_DESCRIPTION_CHARS) +} + +const BLOCK_TAGS: &[&str] = &[ + "br", + "p", + "div", + "li", + "ul", + "ol", + "tr", + "td", + "th", + "table", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "blockquote", + "hr", + "pre", + "section", + "article", + "header", + "footer", + "dd", + "dt", +]; + +/// Converts (possibly HTML) text to plain text: strips tags (dropping +/// `]]> + Jane Host + + https://example.test/small.pngignored"#, + r#" + Episode one + One + <p>First &amp; best</p> + ep-1 + Tue, 02 Jan 2024 10:00:00 GMT + 1:02:03 + + https://example.test/ep1 + + + + + Only itunes title + Long show notes]]> + ep-2 + 754 + + + + Summary only + ep-3 + +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Test Cast"); + assert_eq!(feed.author.as_deref(), Some("Jane Host")); + assert_eq!(feed.description.as_deref(), Some("Weekly talk show.")); + assert_eq!(feed.link.as_deref(), Some("https://example.test/cast")); + assert_eq!( + feed.image_url.as_deref(), + Some("https://example.test/cover.jpg") + ); + assert_eq!(feed.episodes.len(), 3); + + let first = &feed.episodes[0]; + assert_eq!(first.guid.as_deref(), Some("ep-1")); + assert_eq!(first.title, "Episode one"); + assert_eq!(first.description.as_deref(), Some("First & best")); + assert_eq!(first.published_at_ms, Some(1_704_189_600_000)); + assert_eq!(first.duration_ms, Some(3_723_000)); + assert_eq!(first.audio_url, "https://cdn.example.test/1.mp3"); + assert_eq!(first.mime_type.as_deref(), Some("audio/mpeg")); + assert_eq!(first.byte_len, Some(5000)); + assert_eq!(first.link.as_deref(), Some("https://example.test/ep1")); + assert_eq!( + first.image_url.as_deref(), + Some("https://example.test/ep1.jpg") + ); + + let second = &feed.episodes[1]; + assert_eq!(second.title, "Only itunes title"); + assert_eq!(second.description.as_deref(), Some("Long show notes")); + assert_eq!(second.duration_ms, Some(754_000)); + assert_eq!(second.byte_len, None); + assert_eq!(second.published_at_ms, None); + + let third = &feed.episodes[2]; + assert_eq!(third.title, "Untitled episode"); + assert_eq!(third.description.as_deref(), Some("Summary only")); + assert_eq!(third.mime_type, None); +} + +#[test] +fn entities_numeric_refs_and_unknown_html_entities() { + let xml = rss( + "", + r#" + A & B — AП — &bogus; R&D © + x < y && z + e + +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let episode = &feed.episodes[0]; + assert_eq!(episode.title, "A & B — AП — &bogus; R&D ©"); + assert_eq!(episode.description.as_deref(), Some("x < y && z")); + assert_eq!(episode.audio_url, "https://cdn.example.test/a.mp3?x=1&y=2"); +} + +#[test] +fn atom_feed() { + let xml = r#" + + Atom Cast + About <b>things</b> + + + https://example.test/logo.png + Atom Author + +Entry one +urn:uuid:1 +2024-03-01T12:30:00+02:00 +2024-03-05T00:00:00Z +<p>Entry summary</p> + + + + +Entry two +urn:uuid:2 +2024-03-06T00:00:00Z +

Inline

xhtml

+ +
+ +No audio +urn:uuid:3 + + +
"#; + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Atom Cast"); + assert_eq!(feed.description.as_deref(), Some("About things")); + assert_eq!(feed.link.as_deref(), Some("https://example.test/")); + assert_eq!( + feed.image_url.as_deref(), + Some("https://example.test/logo.png") + ); + assert_eq!(feed.author.as_deref(), Some("Atom Author")); + assert_eq!(feed.episodes.len(), 2); + let first = &feed.episodes[0]; + assert_eq!(first.guid.as_deref(), Some("urn:uuid:1")); + assert_eq!(first.published_at_ms, Some(1_709_289_000_000)); + assert_eq!(first.description.as_deref(), Some("Entry summary")); + assert_eq!(first.link.as_deref(), Some("https://example.test/e1")); + assert_eq!(first.audio_url, "https://cdn.example.test/e1.ogg"); + assert_eq!(first.byte_len, Some(42)); + let second = &feed.episodes[1]; + assert_eq!(second.published_at_ms, Some(1_709_683_200_000)); + assert_eq!(second.description.as_deref(), Some("Inline xhtml")); +} + +#[test] +fn items_without_audio_are_skipped_and_media_content_is_a_fallback() { + let xml = r#"T +No enclosurea +Image enclosureb +Empty urlc +Mediad + + + + +Videoe +"#; + let feed = parse_feed(xml.as_bytes()).unwrap(); + let titles: Vec<_> = feed.episodes.iter().map(|e| e.title.as_str()).collect(); + assert_eq!(titles, ["Media", "Video"]); + assert_eq!(feed.episodes[0].audio_url, "https://x.test/d.mp3"); + assert_eq!(feed.episodes[0].byte_len, Some(77)); + assert_eq!( + feed.episodes[0].image_url.as_deref(), + Some("https://x.test/d-thumb.jpg") + ); +} + +#[test] +fn duplicate_guids_keep_the_first_and_missing_guids_stay_absent() { + let items = [ + item("dup", "First", "Mon, 01 Jan 2024 00:00:00 +0000"), + item("dup", "Second", "Tue, 02 Jan 2024 00:00:00 +0000"), + r#"No guid"#.to_string(), + r#"Blank guid "#.to_string(), + ] + .concat(); + let feed = parse_feed(rss("", &items).as_bytes()).unwrap(); + let pairs: Vec<_> = feed + .episodes + .iter() + .map(|e| (e.guid.as_deref(), e.title.as_str(), e.audio_url.as_str())) + .collect(); + assert_eq!( + pairs, + [ + (Some("dup"), "First", "https://cdn.example.test/dup.mp3"), + (None, "No guid", "https://cdn.example.test/noguid.mp3"), + (None, "Blank guid", "https://cdn.example.test/blank.mp3"), + ] + ); +} + +#[test] +fn durations() { + let cases = [ + ("01:02:03", Some(3723.0)), + ("1:02:03", Some(3723.0)), + ("62:03", Some(3723.0)), + ("3723", Some(3723.0)), + ("3723.5", Some(3723.5)), + ("00:00:01.25", Some(1.25)), + (" 45:00 ", Some(2700.0)), + ("0", None), + ("00:00:00", None), + ("", None), + ("abc", None), + ("1:2:3:4", None), + ("-5", None), + ("1e3", None), + ("NaN", None), + ("12:", None), + ]; + for (input, expected) in cases { + assert_eq!(parse_duration(input), expected, "{input:?}"); + } +} + +#[test] +fn dates() { + let reference = Some(1_704_189_600); // 2024-01-02 10:00:00 UTC + let cases = [ + ("Tue, 02 Jan 2024 10:00:00 GMT", reference), + ("Tue, 02 Jan 2024 10:00:00 +0000", reference), + ("02 Jan 2024 10:00:00 UT", reference), + ("Tue, 2 Jan 2024 13:00:00 +0300", reference), + ("Tue, 02 Jan 2024 05:00:00 EST", reference), + ("Tue, 02 Jan 2024 02:00:00 PST", reference), + ("Tuesday, 02 January 2024 10:00 GMT", reference), + ("Tue, 02 Jan 24 10:00:00 GMT", reference), + ("Tue, 02 Jan 2024 13:00:00 +03:00", reference), + ("Tue, 02 Jan 2024 10:00:00", reference), + ("Вт, 02 янв 2024 13:00:00 +0300", reference), + ("2024-01-02T10:00:00Z", reference), + ("2024-01-02T13:00:00+03:00", reference), + ("2024-01-02T10:00:00.123Z", reference), + ("2024-01-02 10:00:00", reference), + ("2024-01-02", Some(1_704_153_600)), + ("not a date", None), + ("Tue, 31 Feb 2024 10:00:00 GMT", None), + ("Tue, 02 Foo 2024 10:00:00 GMT", None), + ("2024-13-02T10:00:00Z", None), + ("", None), + ]; + for (input, expected) in cases { + assert_eq!(parse_date(input), expected, "{input:?}"); + } +} + +#[test] +fn non_ascii_zones_do_not_panic() { + for input in [ + "Mon, 07 Nov 2022 10:00:00 +й0", + "Mon, 07 Nov 2022 10:00:00 -йй", + "Mon, 07 Nov 2022 10:00+й0", + "Mon, 07 Nov 2022 10:00:00 +0й00", + "2022-11-07T10:00:00+0й0", + "2022-11-07T10:00:00+й", + "2022-11-07T10:00:00-€:00", + "2022-11-07T10:00:00+日本", + ] { + assert_eq!(parse_date(input), None, "{input:?}"); + } + // Exhaustive-ish: every short suffix of mixed ASCII / multi-byte + // characters after a sign. + let alphabet = ['0', '9', ':', 'й', '€', '日', '+', '-']; + for a in alphabet { + for b in alphabet { + for c in alphabet { + for prefix in ["Mon, 07 Nov 2022 10:00:00 ", "2022-11-07T10:00:00"] { + for sign in ['+', '-'] { + let _ = parse_date(&format!("{prefix}{sign}{a}{b}{c}")); + } + } + } + } + } + // In a feed, the bad date only loses the date. + let xml = rss("", &item("z", "Zone", "Mon, 07 Nov 2022 10:00:00 +й0")); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); + assert_eq!(feed.episodes[0].published_at_ms, None); +} + +#[test] +fn media_urls_are_resolved_and_must_be_http() { + let xml = rss( + r#""#, + r#"relRelative + + +protoProtocol relative + +fileLocal file + +ftpFTP + +fallbackMedia fallback + + +absAbsolute + "#, + ); + let feed = + parse_feed_with_base(xml.as_bytes(), Some("https://host.test/feeds/rss.xml")).unwrap(); + let urls: Vec<(&str, &str)> = feed + .episodes + .iter() + .map(|e| (e.guid.as_deref().unwrap_or_default(), e.audio_url.as_str())) + .collect(); + assert_eq!( + urls, + [ + ("rel", "https://host.test/audio/1.mp3"), + ("proto", "https://cdn.example.test/2.mp3"), + ("fallback", "https://cdn.example.test/4.mp3"), + ("abs", "HTTPS://cdn.example.test/5.mp3"), + ] + ); + assert_eq!( + feed.image_url.as_deref(), + Some("https://host.test/img/cover.jpg") + ); + assert_eq!( + feed.episodes[0].image_url.as_deref(), + Some("https://host.test/feeds/ep.png") + ); + for episode in &feed.episodes { + assert!(is_remote_url(&episode.audio_url), "{}", episode.audio_url); + } + + // Without a base, relative media cannot be resolved and is dropped; + // protocol-relative URLs default to https. + let feed = parse_feed(xml.as_bytes()).unwrap(); + let guids: Vec<&str> = feed + .episodes + .iter() + .map(|e| e.guid.as_deref().unwrap_or_default()) + .collect(); + assert_eq!(guids, ["proto", "fallback", "abs"]); + assert_eq!(feed.image_url, None); +} + +#[test] +fn rss_with_an_unrelated_default_namespace() { + for ns in ["http://example.test/x", "http://www.w3.org/2005/Atom"] { + let xml = format!( + r#"T + +1One +"# + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "T", "{ns}"); + assert_eq!(feed.episodes.len(), 1, "{ns}"); + assert_eq!(feed.episodes[0].title, "One"); + } +} + +#[test] +fn html_pages_and_garbage_are_rejected() { + let html = br#"Vivat

Hi   there

"#; + let err = parse_feed(html).unwrap_err().to_string(); + assert!(err.contains("not an RSS or Atom feed"), "{err}"); + for garbage in [ + &b""[..], + b"hello world", + b"\x00\xff\xfe garbage", + b"{\"json\": true}", + b"<", + b"<<<>>>", + ] { + let err = parse_feed(garbage).unwrap_err().to_string(); + assert!( + err.contains("not an RSS or Atom feed"), + "{garbage:?}: {err}" + ); + } +} + +#[test] +fn truncated_and_malformed_xml_never_panics() { + let items: String = (0..5) + .map(|i| item(&format!("g{i}"), &format!("T {i} &  "), "bad date")) + .collect(); + let xml = rss( + "", + &items, + ); + let bytes = xml.as_bytes(); + for len in 0..=bytes.len() { + let _ = parse_feed(&bytes[..len]); + } + // Mismatched end tags are tolerated. + let sloppy = r#"SA</b>1"#; + let feed = parse_feed(sloppy.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); + assert_eq!(feed.episodes[0].title, "A"); + // Truncation after complete items keeps them. + let third = xml.find("T 3").unwrap(); + let cut = &bytes[..xml[..third].rfind("<item>").unwrap()]; + let feed = parse_feed(cut).unwrap(); + assert_eq!(feed.episodes.len(), 3); +} + +#[test] +fn deep_nesting_does_not_overflow() { + let depth = 100_000; + let mut xml = String::from("<rss><channel><title>Deep"); + xml.push_str(&"".repeat(depth)); + xml.push_str(&"".repeat(depth)); + xml.push_str(&item("after", "After", "")); + xml.push_str(""); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); +} + +#[test] +fn cyrillic_titles_in_utf8_and_windows_1251() { + let xml = rss( + "Иван Петров", + &item( + "1", + "Выпуск №1: «Привет»", + "Mon, 01 Jan 2024 00:00:00 +0300", + ), + ) + .replace("Test Cast", "Подкаст"); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Подкаст"); + assert_eq!(feed.author.as_deref(), Some("Иван Петров")); + assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»"); + + let cp1251 = xml.replace("encoding=\"UTF-8\"", "encoding=\"windows-1251\""); + let (encoded, _, unmappable) = encoding_rs::WINDOWS_1251.encode(&cp1251); + assert!(!unmappable); + let feed = parse_feed(&encoded).unwrap(); + assert_eq!(feed.title, "Подкаст"); + assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»"); + + // Invalid UTF-8 without a declaration is decoded lossily. + let mut broken = xml.into_bytes(); + let at = broken + .windows(7) + .position(|w| w == b"Test Ca") + .unwrap_or(10); + broken.insert(at, 0xff); + assert!(parse_feed(&broken).is_ok()); +} + +#[test] +fn namespace_prefixes_are_matched_tolerantly() { + // Undeclared `itunes:` prefix and a custom prefix bound to the + // iTunes namespace. + let xml = format!( + r#"PUndeclared +1Custom prefix90 +"# + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.author.as_deref(), Some("Undeclared")); + assert_eq!(feed.episodes[0].title, "Custom prefix"); + assert_eq!(feed.episodes[0].duration_ms, Some(90_000)); +} + +#[test] +fn long_descriptions_are_truncated() { + let long = "word ".repeat(2000); + let text = html_to_text(&long, 4000); + assert!(text.chars().count() <= 4000); + assert!(text.ends_with('…')); + let xml = rss(&format!("{long}"), ""); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!(feed.description.unwrap().chars().count() <= 4000); +} + +#[test] +fn itunes_season_episode_and_type() { + let xml = rss( + "", + r#"aA + 2 7 + Trailer + +bB + two-1 + mystery + +cCbonus + 99999999999 + "#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let a = &feed.episodes[0]; + assert_eq!((a.season, a.episode_number), (Some(2), Some(7))); + assert_eq!(a.episode_type, EpisodeType::Trailer); + let b = &feed.episodes[1]; + assert_eq!((b.season, b.episode_number), (None, None)); + assert_eq!(b.episode_type, EpisodeType::Full); + let c = &feed.episodes[2]; + assert_eq!((c.season, c.episode_number), (None, None)); + assert_eq!(c.episode_type, EpisodeType::Bonus); +} + +#[test] +fn guid_permalink_flag() { + let xml = rss( + "", + r#"x +https://e.test/y +z +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let flags: Vec> = feed.episodes.iter().map(|e| e.guid_is_permalink).collect(); + assert_eq!(flags, [Some(false), Some(true), Some(false), None]); +} + +#[test] +fn new_feed_url_is_reported() { + let xml = rss( + " https://new.example.test/feed?x=1 ", + "", + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!( + feed.new_feed_url.as_deref(), + Some("https://new.example.test/feed?x=1") + ); + let alias = FeedUrl::parse(feed.new_feed_url.as_deref().unwrap()).unwrap(); + assert_eq!(alias.as_str(), "https://new.example.test/feed?x=1"); +} + +#[test] +fn oversized_guid_is_treated_as_absent() { + let long = "g".repeat(MAX_GUID_BYTES + 1); + let xml = rss( + "", + &format!(r#"{long}"#), + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes[0].guid, None); +} + +#[test] +fn media_urls_with_spaces_are_percent_encoded() { + let xml = rss( + "", + r#"s"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes[0].audio_url, "https://c.test/my%20file.mp3"); +} + +#[test] +fn parsed_feed_maps_to_valid_contract_metadata() { + let base = FeedUrl::parse("https://example.test/feed.xml").unwrap(); + let items = [ + item("g1", "One", "Tue, 02 Jan 2024 10:00:00 GMT"), + r#"No guidhttps://example.test/no-guid + Wed, 03 Jan 2024 10:00:00 GMT + "# + .to_string(), + // Same page as the channel link: not identifying. + r#"Home linkhttps://example.test/cast + "# + .to_string(), + ] + .concat(); + let feed = parse_feed_with_base(rss("", &items).as_bytes(), Some(base.as_str())).unwrap(); + let podcast_id = PodcastId::from_feed_url(&base); + let ids = feed.resolve_episode_ids(&podcast_id, &[]); + assert_eq!(ids[0], EpisodeId::from_guid(&podcast_id, "g1")); + assert_eq!( + ids[1], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::link("https://example.test/no-guid").unwrap() + )) + ); + assert_eq!( + ids[2], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::enclosure("https://cdn.example.test/home.mp3").unwrap() + )) + ); + + let podcast = feed.to_podcast_meta(base.clone()); + podcast.validate().unwrap(); + for (episode, id) in feed.episodes.iter().zip(ids) { + let meta = episode.to_meta(&podcast_id, id.unwrap()); + meta.validate().unwrap(); + meta.to_ref(&podcast).validate().unwrap(); + } + + // A later refresh with a changed title and audio URL keeps every id, + // using the stored hints of the previous refresh. + let known: Vec = feed + .episodes + .iter() + .zip(feed.resolve_episode_ids(&podcast_id, &[])) + .map(|(episode, id)| KnownEpisode { + episode_id: id.unwrap(), + hints: episode.identity_hints(), + }) + .collect(); + let refreshed = items + .replace("Home link", "Home link (edited)") + .replace("cdn.example.test/ng.mp3", "cdn2.example.test/ng.mp3"); + let feed2 = parse_feed_with_base(rss("", &refreshed).as_bytes(), Some(base.as_str())).unwrap(); + let ids2 = feed2.resolve_episode_ids(&podcast_id, &known); + let expected: Vec> = + known.iter().map(|k| Some(k.episode_id.clone())).collect(); + assert_eq!(ids2, expected); +} + +#[test] +fn hostile_sizes_stay_bounded() { + let huge_title = "t".repeat(MAX_CAPTURE_BYTES * 2); + let xml = rss( + "", + &format!( + r#"h{huge_title}"# + ), + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!(feed.episodes[0].title.chars().count() <= MAX_TITLE_CHARS); + let duration = rss( + "", + r#"d99999999999999999999"#, + ); + assert_eq!( + parse_feed(duration.as_bytes()).unwrap().episodes[0].duration_ms, + None + ); +} diff --git a/crates/music-dht/src/podcast/mod.rs b/crates/music-dht/src/podcast/mod.rs index a9d6e43..d4b2bb6 100644 --- a/crates/music-dht/src/podcast/mod.rs +++ b/crates/music-dht/src/podcast/mod.rs @@ -46,6 +46,12 @@ //! playback queues and progress operations. A receiving device can show and //! play the episode without being subscribed to the podcast. //! +//! # Feed parser +//! +//! With the `podcast-feed` cargo feature, `feed` provides the shared tolerant +//! RSS 2.0 / RSS 1.0 / Atom parser that produces the identity hints consumed +//! by [`resolve_episode_ids`]. +//! //! ``` //! use music_dht::podcast::{EpisodeId, FeedUrl, PodcastId}; //! @@ -63,6 +69,9 @@ mod identity; mod meta; mod state; +#[cfg(feature = "podcast-feed")] +pub mod feed; + pub use feed_url::{FeedUrl, merge_feed_aliases, resolve_podcast_id}; pub use id::{ EPISODE_ID_DOMAIN, EPISODE_ID_PREFIX, EpisodeId, EpisodeKey, PODCAST_ID_DOMAIN,