diff --git a/Cargo.lock b/Cargo.lock index fe24c1d..9c9ccc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,6 +389,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -790,6 +796,20 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "enum-assoc" version = "1.3.0" @@ -2071,6 +2091,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "music-dht" version = "0.5.0" @@ -2079,10 +2105,12 @@ dependencies = [ "async-trait", "blake3", "data-encoding", + "encoding_rs", "federation-net", "futures", "iroh", "postcard", + "quick-xml", "rand 0.9.5", "rusqlite", "serde", @@ -2092,6 +2120,7 @@ dependencies = [ "tokio", "tracing", "unicode-normalization", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 568c7b7..86aa1c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,3 +38,6 @@ lofty = "0.22" data-encoding = "2" rand = "0.9" tempfile = "3" +quick-xml = "0.41" +encoding_rs = "0.8.35" +url = "2.5" diff --git a/crates/music-dht/Cargo.toml b/crates/music-dht/Cargo.toml index a376ffd..cec4116 100644 --- a/crates/music-dht/Cargo.toml +++ b/crates/music-dht/Cargo.toml @@ -26,6 +26,13 @@ thiserror = { workspace = true } tracing = { workspace = true } rand = { workspace = true } data-encoding = { workspace = true } +quick-xml = { workspace = true, optional = true } +encoding_rs = { workspace = true, optional = true } +url = { workspace = true, optional = true } + +[features] +# RSS 2.0 / RSS 1.0 / Atom podcast feed parser (`music_dht::podcast::feed`). +podcast-feed = ["dep:quick-xml", "dep:encoding_rs", "dep:url"] [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/music-dht/src/podcast/feed.rs b/crates/music-dht/src/podcast/feed.rs new file mode 100644 index 0000000..8da47b8 --- /dev/null +++ b/crates/music-dht/src/podcast/feed.rs @@ -0,0 +1,1775 @@ +//! Shared tolerant RSS 2.0 / RSS 1.0 / Atom parser for podcast feeds. +//! +//! Available with the `podcast-feed` cargo feature. Every Furumi client uses +//! this parser because episode identity depends on what it extracts. +//! +//! Real-world feeds are frequently sloppy: undeclared namespace prefixes, +//! HTML entities that XML does not define, mismatched closing tags, legacy +//! single-byte encodings and creative date formats. The parser is a single +//! iterative pass over `quick-xml` events (no recursion), keeps only the +//! fields Furumi needs, bounds every captured value and never panics on +//! malformed input. It performs no I/O: fetching, redirects, conditional +//! requests and scheduling belong to the client runtime. +//! +//! ``` +//! use music_dht::podcast::feed::parse_feed_with_base; +//! use music_dht::podcast::{FeedUrl, PodcastId}; +//! +//! let xml = br#"Show +//! Pilotep-1 +//! +//! "#; +//! let url = FeedUrl::parse("https://example.test/feed.xml").unwrap(); +//! let feed = parse_feed_with_base(xml, Some(url.as_str())).unwrap(); +//! let podcast = PodcastId::from_feed_url(&url); +//! let ids = feed.resolve_episode_ids(&podcast, &[]); +//! let episode = feed.episodes[0].to_meta(&podcast, ids[0].clone().unwrap()); +//! assert_eq!(episode.audio_url, "https://example.test/audio/1.mp3"); +//! ``` + +use std::borrow::Cow; +use std::collections::HashSet; + +use quick_xml::events::{BytesStart, Event}; +use quick_xml::reader::Reader; +use url::Url; + +use super::{ + EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode, + MAX_DESCRIPTION_CHARS, MAX_GUID_BYTES, MAX_TITLE_CHARS, PodcastError, PodcastId, PodcastMeta, + PodcastResult, is_http_url, resolve_episode_ids, +}; + +/// Upper bound on episodes taken from a single feed document. +pub const MAX_EPISODES: usize = 20_000; +/// Raw text captured for one element is capped to keep memory bounded on +/// hostile or broken documents. +const MAX_CAPTURE_BYTES: usize = 512 * 1024; +/// Elements nested deeper than this are skipped (their events are ignored). +const MAX_DEPTH: usize = 256; +/// Durations longer than this (about 115 days) are treated as garbage. +const MAX_DURATION_SECONDS: f64 = 10_000_000.0; +const UNTITLED_EPISODE: &str = "Untitled episode"; +const UNTITLED_PODCAST: &str = "Untitled podcast"; + +/// Channel-level result of [`parse_feed`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedFeed { + /// Podcast title (never empty). + pub title: String, + /// Author (`itunes:author`, Atom author, `dc:creator`, `managingEditor`). + pub author: Option, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + pub description: Option, + /// Publisher web page. + pub link: Option, + /// Absolute http(s) artwork URL. + pub image_url: Option, + /// `itunes:new-feed-url`, verbatim. Clients canonicalize it with + /// [`FeedUrl::parse`] and record it as a feed alias. + pub new_feed_url: Option, + /// Playable items in document order, duplicates removed. + pub episodes: Vec, +} + +/// One playable feed item. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedEpisode { + /// RSS `` / Atom ``, trimmed; `None` when absent, blank or + /// longer than [`MAX_GUID_BYTES`]. + pub guid: Option, + /// RSS `guid isPermaLink` (default `true` per RSS 2.0); `None` without + /// an RSS GUID. Informational: identity uses the GUID text either way. + pub guid_is_permalink: Option, + /// Title (never empty). + pub title: String, + /// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`]. + pub description: Option, + /// Publication time in unix milliseconds. + pub published_at_ms: Option, + /// Announced duration in milliseconds. + pub duration_ms: Option, + /// `itunes:season`. + pub season: Option, + /// `itunes:episode`. + pub episode_number: Option, + /// `itunes:episodeType`. + pub episode_type: EpisodeType, + /// Absolute http(s) audio URL (never a local file). + pub audio_url: String, + /// Enclosure MIME type, lowercased, without parameters. + pub mime_type: Option, + /// Enclosure length in bytes. + pub byte_len: Option, + /// Item web page. + pub link: Option, + /// Absolute http(s) artwork URL. + pub image_url: Option, +} + +impl ParsedEpisode { + /// Identity hints for [`resolve_episode_ids`]. + pub fn identity_hints(&self) -> EpisodeIdentityHints { + EpisodeIdentityHints { + guid: self.guid.clone(), + enclosure_url: Some(self.audio_url.clone()), + link: self.link.clone(), + published_at_ms: self.published_at_ms, + title: Some(self.title.clone()), + } + } + + /// Contract metadata for this item under a resolved id. + pub fn to_meta(&self, podcast_id: &PodcastId, episode_id: EpisodeId) -> EpisodeMeta { + EpisodeMeta { + episode_id, + podcast_id: podcast_id.clone(), + guid: self.guid.clone(), + title: self.title.clone(), + description: self.description.clone(), + season: self.season, + episode_number: self.episode_number, + episode_type: self.episode_type, + published_at_ms: self.published_at_ms, + duration_ms: self.duration_ms, + audio_url: self.audio_url.clone(), + mime: self.mime_type.clone(), + byte_len: self.byte_len, + image_url: self.image_url.clone(), + link: self + .link + .clone() + .filter(|link| link.len() <= super::MAX_URL_BYTES), + } + } +} + +impl ParsedFeed { + /// Identity hints of every episode, in order. + pub fn identity_hints(&self) -> Vec { + self.episodes + .iter() + .map(ParsedEpisode::identity_hints) + .collect() + } + + /// Resolves stable ids for every episode (see [`resolve_episode_ids`]). + pub fn resolve_episode_ids( + &self, + podcast_id: &PodcastId, + known: &[KnownEpisode], + ) -> Vec> { + resolve_episode_ids( + podcast_id, + self.link.as_deref(), + &self.identity_hints(), + known, + ) + } + + /// Contract channel metadata for a feed fetched from `feed_url`. + pub fn to_podcast_meta(&self, feed_url: FeedUrl) -> PodcastMeta { + PodcastMeta { + feed_url, + title: self.title.clone(), + author: self.author.clone(), + description: self.description.clone(), + link: self.link.clone(), + image_url: self.image_url.clone(), + } + .bounded() + } +} + +/// Parses an RSS or Atom document. Fails for documents that are not feeds +/// (for example an HTML page) with "not an RSS or Atom feed". +/// +/// Without a base URL, relative media URLs cannot be resolved and such +/// episodes are dropped; prefer [`parse_feed_with_base`] when the feed URL is +/// known. +pub fn parse_feed(bytes: &[u8]) -> PodcastResult { + parse_feed_with_base(bytes, None) +} + +/// [`parse_feed`] resolving relative enclosure and image URLs against +/// `base_url` (the final URL the feed was fetched from). Episodes whose audio +/// URL does not resolve to an absolute `http`/`https` URL are dropped, so a +/// feed can never point playback at a local file. +pub fn parse_feed_with_base(bytes: &[u8], base_url: Option<&str>) -> PodcastResult { + let text = decode_document(bytes); + let parser = Parser { + base: base_url.and_then(|base| Url::parse(base).ok()), + ..Parser::default() + }; + parser.run(&text) +} + +fn not_a_feed(detail: impl std::fmt::Display) -> PodcastError { + PodcastError::NotAFeed(format!(" ({detail})")) +} + +// --------------------------------------------------------------------------- +// Character encoding + +/// Decodes the raw document to UTF-8. A BOM wins; otherwise an ASCII-compatible +/// encoding named in the XML declaration (windows-1251, koi8-r, ...) is +/// honoured; everything else is decoded as lossy UTF-8. +fn decode_document(bytes: &[u8]) -> Cow<'_, str> { + if let Some((encoding, bom_len)) = encoding_rs::Encoding::for_bom(bytes) { + let (text, _) = encoding.decode_without_bom_handling(&bytes[bom_len..]); + return text; + } + if let Some(encoding) = declared_encoding(bytes).and_then(encoding_rs::Encoding::for_label) + && encoding != encoding_rs::UTF_8 + && encoding.is_ascii_compatible() + { + let (text, _) = encoding.decode_without_bom_handling(bytes); + return text; + } + String::from_utf8_lossy(bytes) +} + +/// The `encoding="..."` label of a leading XML declaration, if any. +fn declared_encoding(bytes: &[u8]) -> Option<&[u8]> { + let head = &bytes[..bytes.len().min(512)]; + let start = head.iter().position(|b| !b.is_ascii_whitespace())?; + let head = &head[start..]; + if !head.starts_with(b"")?; + let decl = &head[..end]; + let at = find(decl, b"encoding")? + b"encoding".len(); + let rest = &decl[at..]; + let rest = &rest[rest.iter().position(|b| !b.is_ascii_whitespace())?..]; + let rest = rest.strip_prefix(b"=")?; + let rest = &rest[rest.iter().position(|b| !b.is_ascii_whitespace())?..]; + let quote = *rest.first()?; + if quote != b'"' && quote != b'\'' { + return None; + } + let rest = &rest[1..]; + let len = rest.iter().position(|&b| b == quote)?; + Some(&rest[..len]) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +// --------------------------------------------------------------------------- +// Namespaces + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ns { + /// No namespace, or the RSS 0.9x/1.0 namespace. + Plain, + Atom, + Itunes, + Content, + Media, + Dc, + Other, +} + +fn classify_uri(uri: &str) -> Ns { + let uri = uri.trim().to_ascii_lowercase(); + if uri.is_empty() { + Ns::Plain + } else if uri.contains("w3.org/2005/atom") { + Ns::Atom + } else if uri.contains("itunes.com/dtds/podcast") { + Ns::Itunes + } else if uri.contains("purl.org/rss/1.0/modules/content") { + Ns::Content + } else if uri.contains("search.yahoo.com/mrss") { + Ns::Media + } else if uri.contains("purl.org/dc/elements") { + Ns::Dc + } else if uri.contains("purl.org/rss/1.0") + || uri.contains("my.netscape.com/rdf") + || uri.contains("backend.userland.com/rss") + { + Ns::Plain + } else { + Ns::Other + } +} + +/// Classification of a prefix nobody declared (common in sloppy feeds). +fn classify_prefix(prefix: &str) -> Ns { + match prefix.to_ascii_lowercase().as_str() { + "" => Ns::Plain, + "itunes" => Ns::Itunes, + "content" => Ns::Content, + "media" => Ns::Media, + "dc" => Ns::Dc, + "atom" | "atom10" => Ns::Atom, + _ => Ns::Other, + } +} + +// --------------------------------------------------------------------------- +// Parser state + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Rss, + Atom, +} + +#[derive(Debug)] +struct Elem { + ns: Ns, + /// ASCII-lowercased local name. + local: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Field { + FeedTitle, + FeedDescription, + FeedItunesSummary, + FeedLink, + FeedItunesAuthor, + FeedAtomAuthor, + FeedDcCreator, + FeedManagingEditor, + FeedImageUrl, + FeedLogo, + FeedIcon, + FeedNewFeedUrl, + ItemTitle, + ItemItunesTitle, + ItemDescription, + ItemContent, + ItemSummary, + ItemGuid, + ItemPubDate, + ItemDcDate, + ItemUpdated, + ItemDuration, + ItemLink, + ItemSeason, + ItemEpisode, + ItemEpisodeType, +} + +#[derive(Debug)] +struct Capture { + /// Stack depth of the captured element. + depth: usize, + field: Field, + text: String, +} + +#[derive(Debug, Default)] +struct FeedAcc { + title: Option, + description: Option, + itunes_summary: Option, + link: Option, + itunes_author: Option, + atom_author: Option, + dc_creator: Option, + managing_editor: Option, + itunes_image: Option, + image_url: Option, + logo: Option, + icon: Option, + new_feed_url: Option, +} + +#[derive(Debug, Clone)] +struct Media { + url: String, + mime: Option, + len: Option, +} + +#[derive(Debug, Default)] +struct ItemAcc { + title: Option, + itunes_title: Option, + description: Option, + content: Option, + summary: Option, + guid: Option, + guid_is_permalink: Option, + pub_date: Option, + dc_date: Option, + updated: Option, + duration: Option, + season: Option, + episode: Option, + episode_type: Option, + link: Option, + enclosure: Option, + media_content: Option, + itunes_image: Option, + thumbnail: Option, +} + +#[derive(Debug, Default)] +struct Parser { + kind: Option, + stack: Vec, + /// Namespace declarations: (depth of declaring element, prefix, uri). + ns_decls: Vec<(usize, String, String)>, + /// Start tags ignored because they exceeded `MAX_DEPTH`. + skipped_depth: usize, + capture: Option, + feed: FeedAcc, + item: Option, + item_depth: usize, + episodes: Vec, + seen_guids: HashSet, + full: bool, + /// URL the document was fetched from, for relative media URLs. + base: Option, +} + +/// Sets `slot` unless it already holds a value (first occurrence wins). +fn set_once(slot: &mut Option, value: String) { + if slot.is_none() && !value.trim().is_empty() { + *slot = Some(value); + } +} + +impl Parser { + fn run(mut self, text: &str) -> PodcastResult { + let mut reader = Reader::from_str(text); + { + let config = reader.config_mut(); + config.check_end_names = false; + config.allow_unmatched_ends = true; + config.allow_dangling_amp = true; + config.expand_empty_elements = false; + config.trim_text(false); + } + let mut malformed = None; + loop { + match reader.read_event() { + Ok(Event::Eof) => break, + Ok(event) => { + self.handle(event)?; + if self.full { + break; + } + } + Err(err) => { + if self.kind.is_none() { + return Err(not_a_feed(format_args!("invalid XML: {err}"))); + } + malformed = Some(err.to_string()); + break; + } + } + } + if self.kind.is_none() { + return Err(PodcastError::NotAFeed(String::new())); + } + if let Some(err) = malformed { + if self.episodes.is_empty() { + return Err(PodcastError::MalformedFeed(err)); + } + tracing::warn!("podcast feed is malformed, keeping what parsed: {err}"); + } + Ok(self.finish()) + } + + fn handle(&mut self, event: Event<'_>) -> PodcastResult<()> { + match event { + Event::Start(start) => self.on_start(&start, false)?, + Event::Empty(start) => self.on_start(&start, true)?, + Event::End(end) => { + let name = end.name(); + let local = local_name(name.as_ref()); + self.on_end(&local); + } + Event::Text(text) => { + if let Some(capture) = self.capture.as_mut() { + match text.xml10_content() { + Ok(value) => push_capped(&mut capture.text, &value), + Err(_) => push_capped(&mut capture.text, &String::from_utf8_lossy(&text)), + } + } + } + Event::CData(data) => { + if let Some(capture) = self.capture.as_mut() { + match data.decode() { + Ok(value) => push_capped(&mut capture.text, &value), + Err(_) => push_capped(&mut capture.text, &String::from_utf8_lossy(&data)), + } + } + } + Event::GeneralRef(reference) => { + if let Some(capture) = self.capture.as_mut() { + let raw = String::from_utf8_lossy(&reference); + match resolve_entity(&raw) { + Some(ch) => push_capped(&mut capture.text, ch.encode_utf8(&mut [0; 4])), + None => { + // Keep unknown entities verbatim instead of failing. + push_capped(&mut capture.text, "&"); + push_capped(&mut capture.text, &raw); + push_capped(&mut capture.text, ";"); + } + } + } + } + Event::Comment(_) | Event::Decl(_) | Event::PI(_) | Event::DocType(_) | Event::Eof => {} + } + Ok(()) + } + + fn resolve_ns(&self, prefix: &str) -> Ns { + for (_, declared, uri) in self.ns_decls.iter().rev() { + if declared == prefix { + return classify_uri(uri); + } + } + classify_prefix(prefix) + } + + fn on_start(&mut self, start: &BytesStart<'_>, empty: bool) -> PodcastResult<()> { + if self.stack.len() >= MAX_DEPTH { + if !empty { + self.skipped_depth += 1; + } + return Ok(()); + } + let depth = self.stack.len() + 1; + let attrs = Attrs::collect(start); + for (key, value) in &attrs.raw { + if key == "xmlns" { + self.ns_decls.push((depth, String::new(), value.clone())); + } else if let Some(prefix) = key.strip_prefix("xmlns:") { + self.ns_decls + .push((depth, prefix.to_string(), value.clone())); + } + } + let qname = start.name(); + let (prefix, local) = split_qname(qname.as_ref()); + let mut ns = self.resolve_ns(&prefix); + if self.kind == Some(Kind::Rss) && prefix.is_empty() && matches!(ns, Ns::Atom | Ns::Other) { + // RSS has no namespace of its own; an unprefixed element is a core + // RSS element whatever default namespace the publisher declared. + ns = Ns::Plain; + } + self.stack.push(Elem { ns, local }); + + if self.kind.is_none() { + let local = self.stack[0].local.as_str(); + self.kind = match local { + "rss" | "rdf" => Some(Kind::Rss), + "feed" => Some(Kind::Atom), + _ => return Err(not_a_feed(format_args!("root element <{local}>"))), + }; + } else if let Some(capture) = self.capture.as_mut() { + // Markup nested in a captured field (e.g. Atom xhtml content): + // keep words apart. + push_capped(&mut capture.text, " "); + } else { + self.dispatch_start(&attrs); + } + + if empty { + self.close_top(); + } + Ok(()) + } + + fn dispatch_start(&mut self, attrs: &Attrs) { + let Some(kind) = self.kind else { return }; + let depth = self.stack.len(); + let local_at = |back: usize| { + depth + .checked_sub(back) + .and_then(|index| self.stack.get(index)) + .map(|elem| elem.local.clone()) + .unwrap_or_default() + }; + let ns = self.stack[depth - 1].ns; + let local = local_at(1); + let parent = local_at(2); + let grandparent = local_at(3); + let core = match kind { + Kind::Rss => ns == Ns::Plain, + Kind::Atom => matches!(ns, Ns::Plain | Ns::Atom), + }; + + if self.item.is_some() { + let direct = depth == self.item_depth + 1; + if let Some(field) = self.item_field(kind, ns, core, &local, direct, attrs) { + self.begin_capture(field); + } + return; + } + + let is_item = match kind { + Kind::Rss => core && local == "item", + Kind::Atom => core && local == "entry", + }; + if is_item { + self.item = Some(ItemAcc::default()); + self.item_depth = depth; + return; + } + + let feed_level = match kind { + Kind::Rss => parent == "channel", + Kind::Atom => depth == 2, + }; + let field = if feed_level { + self.feed_field(kind, ns, core, &local, attrs) + } else if kind == Kind::Rss + && core + && local == "url" + && parent == "image" + && matches!(grandparent.as_str(), "channel" | "rdf") + { + Some(Field::FeedImageUrl) + } else if kind == Kind::Atom && core && local == "name" && parent == "author" && depth == 3 + { + Some(Field::FeedAtomAuthor) + } else { + None + }; + if let Some(field) = field { + self.begin_capture(field); + } + } + + fn feed_field( + &mut self, + kind: Kind, + ns: Ns, + core: bool, + local: &str, + attrs: &Attrs, + ) -> Option { + if core { + return match (kind, local) { + (_, "title") => Some(Field::FeedTitle), + (Kind::Rss, "description") | (Kind::Atom, "subtitle") => { + Some(Field::FeedDescription) + } + (Kind::Rss, "link") => Some(Field::FeedLink), + (Kind::Rss, "managingeditor") => Some(Field::FeedManagingEditor), + (Kind::Atom, "link") => { + let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); + if rel.is_empty() || rel == "alternate" { + let href = attrs.get("href").unwrap_or_default(); + set_once(&mut self.feed.link, clean_short(&href)); + } + None + } + (Kind::Atom, "logo") => Some(Field::FeedLogo), + (Kind::Atom, "icon") => Some(Field::FeedIcon), + _ => None, + }; + } + match (ns, local) { + (Ns::Itunes, "author") => Some(Field::FeedItunesAuthor), + (Ns::Itunes, "summary") => Some(Field::FeedItunesSummary), + (Ns::Itunes, "image") => { + if let Some(href) = attrs.get("href") { + set_once(&mut self.feed.itunes_image, clean_short(&href)); + } + None + } + (Ns::Itunes, "new-feed-url") => Some(Field::FeedNewFeedUrl), + (Ns::Dc, "creator") => Some(Field::FeedDcCreator), + _ => None, + } + } + + fn item_field( + &mut self, + kind: Kind, + ns: Ns, + core: bool, + local: &str, + direct: bool, + attrs: &Attrs, + ) -> Option { + let item = self.item.as_mut()?; + // Media RSS may wrap content in , so accept any depth. + if ns == Ns::Media { + match local { + "content" => { + let url = attrs.get("url").unwrap_or_default(); + let mime = attrs.get("type"); + let medium = attrs.get("medium"); + if item.media_content.is_none() + && is_playable(&url, mime.as_deref(), medium.as_deref(), false) + { + item.media_content = Some(Media { + url: url.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("filesize")), + }); + } + } + "thumbnail" => { + if let Some(url) = attrs.get("url") { + set_once(&mut item.thumbnail, clean_short(&url)); + } + } + _ => {} + } + return None; + } + if !direct { + return None; + } + if core { + return match (kind, local) { + (_, "title") => Some(Field::ItemTitle), + (Kind::Rss, "description") => Some(Field::ItemDescription), + (Kind::Rss, "guid") => { + if item.guid_is_permalink.is_none() { + let permalink = attrs.get("ispermalink").unwrap_or_default(); + item.guid_is_permalink = + Some(!permalink.trim().eq_ignore_ascii_case("false")); + } + Some(Field::ItemGuid) + } + (Kind::Atom, "id") => Some(Field::ItemGuid), + (Kind::Rss, "pubdate") | (Kind::Atom, "published") => Some(Field::ItemPubDate), + (Kind::Atom, "updated") => Some(Field::ItemUpdated), + (Kind::Rss, "link") => Some(Field::ItemLink), + (Kind::Atom, "summary") => Some(Field::ItemDescription), + (Kind::Atom, "content") => Some(Field::ItemContent), + (Kind::Rss, "enclosure") => { + let url = attrs.get("url").unwrap_or_default(); + let mime = attrs.get("type"); + if item.enclosure.is_none() && is_playable(&url, mime.as_deref(), None, true) { + item.enclosure = Some(Media { + url: url.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("length")), + }); + } + None + } + (Kind::Atom, "link") => { + let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase(); + let href = attrs.get("href").unwrap_or_default(); + let mime = attrs.get("type"); + if rel == "enclosure" { + if item.enclosure.is_none() + && is_playable(&href, mime.as_deref(), None, true) + { + item.enclosure = Some(Media { + url: href.trim().to_string(), + mime: clean_mime(mime.as_deref()), + len: parse_len(attrs.get("length")), + }); + } + } else if rel.is_empty() || rel == "alternate" { + set_once(&mut item.link, clean_short(&href)); + } + None + } + _ => None, + }; + } + match (ns, local) { + (Ns::Itunes, "title") => Some(Field::ItemItunesTitle), + (Ns::Itunes, "summary") => Some(Field::ItemSummary), + (Ns::Itunes, "duration") => Some(Field::ItemDuration), + (Ns::Itunes, "season") => Some(Field::ItemSeason), + (Ns::Itunes, "episode") => Some(Field::ItemEpisode), + (Ns::Itunes, "episodetype") => Some(Field::ItemEpisodeType), + (Ns::Itunes, "image") => { + if let Some(href) = attrs.get("href") { + set_once(&mut item.itunes_image, clean_short(&href)); + } + None + } + (Ns::Content, "encoded") => Some(Field::ItemContent), + (Ns::Dc, "date") => Some(Field::ItemDcDate), + _ => None, + } + } + + fn begin_capture(&mut self, field: Field) { + self.capture = Some(Capture { + depth: self.stack.len(), + field, + text: String::new(), + }); + } + + fn on_end(&mut self, local: &str) { + if self.skipped_depth > 0 { + self.skipped_depth -= 1; + return; + } + // Tolerate mismatched end tags: close up to the nearest open element + // with the same local name, ignore the tag if none is open. + let Some(index) = self.stack.iter().rposition(|elem| elem.local == local) else { + return; + }; + while self.stack.len() > index { + self.close_top(); + } + } + + fn close_top(&mut self) { + let depth = self.stack.len(); + if depth == 0 { + return; + } + if self + .capture + .as_ref() + .is_some_and(|capture| capture.depth == depth) + && let Some(capture) = self.capture.take() + { + self.store_capture(capture); + } + if depth == self.item_depth + && let Some(item) = self.item.take() + { + self.finish_item(item); + } + while self + .ns_decls + .last() + .is_some_and(|(decl_depth, _, _)| *decl_depth >= depth) + { + self.ns_decls.pop(); + } + self.stack.pop(); + } + + fn store_capture(&mut self, capture: Capture) { + let text = capture.text; + if let Some(item) = self.item.as_mut() { + let slot = match capture.field { + Field::ItemTitle => &mut item.title, + Field::ItemItunesTitle => &mut item.itunes_title, + Field::ItemDescription => &mut item.description, + Field::ItemContent => &mut item.content, + Field::ItemSummary => &mut item.summary, + Field::ItemGuid => &mut item.guid, + Field::ItemPubDate => &mut item.pub_date, + Field::ItemDcDate => &mut item.dc_date, + Field::ItemUpdated => &mut item.updated, + Field::ItemDuration => &mut item.duration, + Field::ItemLink => &mut item.link, + Field::ItemSeason => &mut item.season, + Field::ItemEpisode => &mut item.episode, + Field::ItemEpisodeType => &mut item.episode_type, + _ => return, + }; + set_once(slot, text); + return; + } + let feed = &mut self.feed; + let slot = match capture.field { + Field::FeedTitle => &mut feed.title, + Field::FeedDescription => &mut feed.description, + Field::FeedItunesSummary => &mut feed.itunes_summary, + Field::FeedLink => &mut feed.link, + Field::FeedItunesAuthor => &mut feed.itunes_author, + Field::FeedAtomAuthor => &mut feed.atom_author, + Field::FeedDcCreator => &mut feed.dc_creator, + Field::FeedManagingEditor => &mut feed.managing_editor, + Field::FeedImageUrl => &mut feed.image_url, + Field::FeedLogo => &mut feed.logo, + Field::FeedIcon => &mut feed.icon, + Field::FeedNewFeedUrl => &mut feed.new_feed_url, + _ => return, + }; + set_once(slot, text); + } + + fn finish_item(&mut self, item: ItemAcc) { + if self.full { + return; + } + let base = self.base.as_ref(); + let Some(media) = [item.enclosure, item.media_content] + .into_iter() + .flatten() + .find_map(|media| { + let url = resolve_http_url(&media.url, base)?; + Some(Media { url, ..media }) + }) + else { + return; + }; + let guid = item + .guid + .as_deref() + .map(str::trim) + .filter(|guid| !guid.is_empty() && guid.len() <= MAX_GUID_BYTES) + .map(str::to_string); + // Duplicate items (same GUID, or same audio URL without GUID) keep + // the first occurrence. + let dedup_key = match &guid { + Some(guid) => format!("guid:{guid}"), + None => format!("enclosure:{}", media.url), + }; + if !self.seen_guids.insert(dedup_key) { + return; + } + let guid_is_permalink = if guid.is_some() { + item.guid_is_permalink + } else { + None + }; + let title = [item.title.as_deref(), item.itunes_title.as_deref()] + .into_iter() + .flatten() + .map(clean_short) + .find(|title| !title.is_empty()) + .unwrap_or_else(|| UNTITLED_EPISODE.to_string()); + let description = [ + item.description.as_deref(), + item.content.as_deref(), + item.summary.as_deref(), + ] + .into_iter() + .flatten() + .map(clean_long) + .find(|text| !text.is_empty()); + let published_at_ms = [ + item.pub_date.as_deref(), + item.dc_date.as_deref(), + item.updated.as_deref(), + ] + .into_iter() + .flatten() + .find_map(parse_date) + .map(|seconds| seconds.saturating_mul(1_000)); + self.episodes.push(ParsedEpisode { + guid, + guid_is_permalink, + title, + description, + published_at_ms, + duration_ms: item + .duration + .as_deref() + .and_then(parse_duration) + .filter(|seconds| *seconds <= MAX_DURATION_SECONDS) + .map(|seconds| (seconds * 1_000.0).round() as i64), + season: item.season.as_deref().and_then(parse_number), + episode_number: item.episode.as_deref().and_then(parse_number), + episode_type: item + .episode_type + .as_deref() + .map(EpisodeType::parse) + .unwrap_or_default(), + audio_url: media.url, + mime_type: media.mime, + byte_len: media.len, + link: item + .link + .as_deref() + .map(clean_short) + .filter(|s| !s.is_empty()), + image_url: [item.itunes_image, item.thumbnail] + .into_iter() + .flatten() + .find_map(|url| resolve_http_url(&url, base)), + }); + if self.episodes.len() >= MAX_EPISODES { + self.full = true; + } + } + + fn finish(self) -> ParsedFeed { + let feed = self.feed; + let title = feed + .title + .as_deref() + .map(clean_short) + .filter(|title| !title.is_empty()) + .unwrap_or_else(|| UNTITLED_PODCAST.to_string()); + let author = [ + feed.itunes_author.as_deref(), + feed.atom_author.as_deref(), + feed.dc_creator.as_deref(), + feed.managing_editor.as_deref(), + ] + .into_iter() + .flatten() + .map(clean_short) + .find(|author| !author.is_empty()); + let description = [feed.description.as_deref(), feed.itunes_summary.as_deref()] + .into_iter() + .flatten() + .map(clean_long) + .find(|text| !text.is_empty()); + let image_url = [feed.itunes_image, feed.image_url, feed.logo, feed.icon] + .into_iter() + .flatten() + .find_map(|url| resolve_http_url(&url, self.base.as_ref())); + ParsedFeed { + title, + author, + description, + link: feed + .link + .as_deref() + .map(clean_short) + .filter(|s| !s.is_empty()), + image_url, + new_feed_url: feed + .new_feed_url + .as_deref() + .map(|url| url.trim().to_string()) + .filter(|url| !url.is_empty() && url.len() <= super::MAX_URL_BYTES), + episodes: self.episodes, + } + } +} + +// --------------------------------------------------------------------------- +// Element and attribute helpers + +fn split_qname(name: &[u8]) -> (String, String) { + let name = String::from_utf8_lossy(name); + match name.split_once(':') { + Some((prefix, local)) => (prefix.to_string(), local.to_ascii_lowercase()), + None => (String::new(), name.to_ascii_lowercase()), + } +} + +fn local_name(name: &[u8]) -> String { + split_qname(name).1 +} + +/// Attributes of one element: raw keys with entity-decoded values. +struct Attrs { + raw: Vec<(String, String)>, +} + +impl Attrs { + fn collect(start: &BytesStart<'_>) -> Attrs { + let mut attributes = start.attributes(); + attributes.with_checks(false); + let raw = attributes + .flatten() + .map(|attr| { + let key = String::from_utf8_lossy(attr.key.as_ref()).into_owned(); + let value = decode_entities(&String::from_utf8_lossy(&attr.value)); + (key, value) + }) + .collect(); + Attrs { raw } + } + + /// Value of the attribute whose local name matches `name` (any prefix). + fn get(&self, name: &str) -> Option { + self.raw + .iter() + .find(|(key, _)| { + let local = key.rsplit_once(':').map_or(key.as_str(), |(_, l)| l); + local.eq_ignore_ascii_case(name) && !key.starts_with("xmlns") + }) + .map(|(_, value)| value.clone()) + } +} + +fn push_capped(text: &mut String, value: &str) { + if text.len() >= MAX_CAPTURE_BYTES { + return; + } + let room = MAX_CAPTURE_BYTES - text.len(); + if value.len() <= room { + text.push_str(value); + } else { + let mut end = room; + while !value.is_char_boundary(end) { + end -= 1; + } + text.push_str(&value[..end]); + } +} + +fn parse_len(value: Option) -> Option { + value?.trim().parse::().ok().filter(|len| *len > 0) +} + +fn clean_mime(mime: Option<&str>) -> Option { + let mime = mime?.split(';').next()?.trim().to_ascii_lowercase(); + (!mime.is_empty()).then_some(mime) +} + +const MEDIA_EXTENSIONS: &[&str] = &[ + "mp3", "m4a", "m4b", "aac", "ogg", "oga", "opus", "flac", "wav", "mp4", "m4v", "mov", "webm", + "mka", +]; + +fn has_media_extension(url: &str) -> bool { + let path = url.split(['?', '#']).next().unwrap_or_default(); + let file = path.rsplit('/').next().unwrap_or_default(); + file.rsplit_once('.').is_some_and(|(_, ext)| { + MEDIA_EXTENSIONS + .iter() + .any(|known| known.eq_ignore_ascii_case(ext)) + }) +} + +/// Whether an enclosure/media element points at something playable. +/// `lenient_untyped`: accept a URL without type information (RSS +/// enclosures); media:content without type must at least look like audio. +fn is_playable(url: &str, mime: Option<&str>, medium: Option<&str>, lenient_untyped: bool) -> bool { + let url = url.trim(); + if url.is_empty() { + return false; + } + if let Some(mime) = clean_mime(mime) { + return mime.starts_with("audio/") + || mime.starts_with("video/") + || matches!( + mime.as_str(), + "application/octet-stream" + | "binary/octet-stream" + | "application/ogg" + | "application/mp4" + | "application/x-mpegurl" + | "application/vnd.apple.mpegurl" + ) + || (mime == "application/x-unknown" && has_media_extension(url)); + } + if let Some(medium) = medium.map(|m| m.trim().to_ascii_lowercase()) + && !medium.is_empty() + { + return medium == "audio" || medium == "video"; + } + lenient_untyped || has_media_extension(url) +} + +/// An absolute `http`/`https` URL for a media or image reference: absolute +/// http(s) URLs are kept as written, relative ones (including +/// protocol-relative `//host/...`) are resolved against `base`, and anything +/// else (`file:`, `ftp:`, unresolvable relative paths) yields `None`. +fn resolve_http_url(raw: &str, base: Option<&Url>) -> Option { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + let has_host = |url: &Url| url.host_str().is_some_and(|host| !host.is_empty()); + if is_remote_url(raw) { + // Keep the publisher's spelling when it already satisfies the + // contract; otherwise use the serialized (percent-encoded) form. + let parsed = Url::parse(raw).ok().filter(has_host)?; + return if is_http_url(raw) { + Some(raw.to_string()) + } else { + Some(parsed.to_string()).filter(|url| is_http_url(url)) + }; + } + let resolved = match base { + Some(base) => base.join(raw).ok()?, + None if raw.starts_with("//") => Url::parse(&format!("https:{raw}")).ok()?, + None => return None, + }; + (matches!(resolved.scheme(), "http" | "https") && has_host(&resolved)) + .then(|| resolved.to_string()) + .filter(|url| is_http_url(url)) +} + +fn is_remote_url(value: &str) -> bool { + let value = value.as_bytes(); + let has_prefix = |prefix: &[u8]| { + value.len() >= prefix.len() && value[..prefix.len()].eq_ignore_ascii_case(prefix) + }; + has_prefix(b"http://") || has_prefix(b"https://") +} + +/// Parses a non-negative integer such as `itunes:season`; garbage yields `None`. +fn parse_number(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + text.parse().ok() +} + +// --------------------------------------------------------------------------- +// Text cleanup + +fn clean_short(text: &str) -> String { + html_to_text(text, MAX_TITLE_CHARS) +} + +fn clean_long(text: &str) -> String { + html_to_text(text, MAX_DESCRIPTION_CHARS) +} + +const BLOCK_TAGS: &[&str] = &[ + "br", + "p", + "div", + "li", + "ul", + "ol", + "tr", + "td", + "th", + "table", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "blockquote", + "hr", + "pre", + "section", + "article", + "header", + "footer", + "dd", + "dt", +]; + +/// Converts (possibly HTML) text to plain text: strips tags (dropping +/// `]]> + Jane Host + + https://example.test/small.pngignored"#, + r#" + Episode one + One + <p>First &amp; best</p> + ep-1 + Tue, 02 Jan 2024 10:00:00 GMT + 1:02:03 + + https://example.test/ep1 + + + + + Only itunes title + Long show notes]]> + ep-2 + 754 + + + + Summary only + ep-3 + +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Test Cast"); + assert_eq!(feed.author.as_deref(), Some("Jane Host")); + assert_eq!(feed.description.as_deref(), Some("Weekly talk show.")); + assert_eq!(feed.link.as_deref(), Some("https://example.test/cast")); + assert_eq!( + feed.image_url.as_deref(), + Some("https://example.test/cover.jpg") + ); + assert_eq!(feed.episodes.len(), 3); + + let first = &feed.episodes[0]; + assert_eq!(first.guid.as_deref(), Some("ep-1")); + assert_eq!(first.title, "Episode one"); + assert_eq!(first.description.as_deref(), Some("First & best")); + assert_eq!(first.published_at_ms, Some(1_704_189_600_000)); + assert_eq!(first.duration_ms, Some(3_723_000)); + assert_eq!(first.audio_url, "https://cdn.example.test/1.mp3"); + assert_eq!(first.mime_type.as_deref(), Some("audio/mpeg")); + assert_eq!(first.byte_len, Some(5000)); + assert_eq!(first.link.as_deref(), Some("https://example.test/ep1")); + assert_eq!( + first.image_url.as_deref(), + Some("https://example.test/ep1.jpg") + ); + + let second = &feed.episodes[1]; + assert_eq!(second.title, "Only itunes title"); + assert_eq!(second.description.as_deref(), Some("Long show notes")); + assert_eq!(second.duration_ms, Some(754_000)); + assert_eq!(second.byte_len, None); + assert_eq!(second.published_at_ms, None); + + let third = &feed.episodes[2]; + assert_eq!(third.title, "Untitled episode"); + assert_eq!(third.description.as_deref(), Some("Summary only")); + assert_eq!(third.mime_type, None); +} + +#[test] +fn entities_numeric_refs_and_unknown_html_entities() { + let xml = rss( + "", + r#" + A & B — AП — &bogus; R&D © + x < y && z + e + +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let episode = &feed.episodes[0]; + assert_eq!(episode.title, "A & B — AП — &bogus; R&D ©"); + assert_eq!(episode.description.as_deref(), Some("x < y && z")); + assert_eq!(episode.audio_url, "https://cdn.example.test/a.mp3?x=1&y=2"); +} + +#[test] +fn atom_feed() { + let xml = r#" + + Atom Cast + About <b>things</b> + + + https://example.test/logo.png + Atom Author + +Entry one +urn:uuid:1 +2024-03-01T12:30:00+02:00 +2024-03-05T00:00:00Z +<p>Entry summary</p> + + + + +Entry two +urn:uuid:2 +2024-03-06T00:00:00Z +

Inline

xhtml

+ +
+ +No audio +urn:uuid:3 + + +
"#; + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Atom Cast"); + assert_eq!(feed.description.as_deref(), Some("About things")); + assert_eq!(feed.link.as_deref(), Some("https://example.test/")); + assert_eq!( + feed.image_url.as_deref(), + Some("https://example.test/logo.png") + ); + assert_eq!(feed.author.as_deref(), Some("Atom Author")); + assert_eq!(feed.episodes.len(), 2); + let first = &feed.episodes[0]; + assert_eq!(first.guid.as_deref(), Some("urn:uuid:1")); + assert_eq!(first.published_at_ms, Some(1_709_289_000_000)); + assert_eq!(first.description.as_deref(), Some("Entry summary")); + assert_eq!(first.link.as_deref(), Some("https://example.test/e1")); + assert_eq!(first.audio_url, "https://cdn.example.test/e1.ogg"); + assert_eq!(first.byte_len, Some(42)); + let second = &feed.episodes[1]; + assert_eq!(second.published_at_ms, Some(1_709_683_200_000)); + assert_eq!(second.description.as_deref(), Some("Inline xhtml")); +} + +#[test] +fn items_without_audio_are_skipped_and_media_content_is_a_fallback() { + let xml = r#"T +No enclosurea +Image enclosureb +Empty urlc +Mediad + + + + +Videoe +"#; + let feed = parse_feed(xml.as_bytes()).unwrap(); + let titles: Vec<_> = feed.episodes.iter().map(|e| e.title.as_str()).collect(); + assert_eq!(titles, ["Media", "Video"]); + assert_eq!(feed.episodes[0].audio_url, "https://x.test/d.mp3"); + assert_eq!(feed.episodes[0].byte_len, Some(77)); + assert_eq!( + feed.episodes[0].image_url.as_deref(), + Some("https://x.test/d-thumb.jpg") + ); +} + +#[test] +fn duplicate_guids_keep_the_first_and_missing_guids_stay_absent() { + let items = [ + item("dup", "First", "Mon, 01 Jan 2024 00:00:00 +0000"), + item("dup", "Second", "Tue, 02 Jan 2024 00:00:00 +0000"), + r#"No guid"#.to_string(), + r#"Blank guid "#.to_string(), + ] + .concat(); + let feed = parse_feed(rss("", &items).as_bytes()).unwrap(); + let pairs: Vec<_> = feed + .episodes + .iter() + .map(|e| (e.guid.as_deref(), e.title.as_str(), e.audio_url.as_str())) + .collect(); + assert_eq!( + pairs, + [ + (Some("dup"), "First", "https://cdn.example.test/dup.mp3"), + (None, "No guid", "https://cdn.example.test/noguid.mp3"), + (None, "Blank guid", "https://cdn.example.test/blank.mp3"), + ] + ); +} + +#[test] +fn durations() { + let cases = [ + ("01:02:03", Some(3723.0)), + ("1:02:03", Some(3723.0)), + ("62:03", Some(3723.0)), + ("3723", Some(3723.0)), + ("3723.5", Some(3723.5)), + ("00:00:01.25", Some(1.25)), + (" 45:00 ", Some(2700.0)), + ("0", None), + ("00:00:00", None), + ("", None), + ("abc", None), + ("1:2:3:4", None), + ("-5", None), + ("1e3", None), + ("NaN", None), + ("12:", None), + ]; + for (input, expected) in cases { + assert_eq!(parse_duration(input), expected, "{input:?}"); + } +} + +#[test] +fn dates() { + let reference = Some(1_704_189_600); // 2024-01-02 10:00:00 UTC + let cases = [ + ("Tue, 02 Jan 2024 10:00:00 GMT", reference), + ("Tue, 02 Jan 2024 10:00:00 +0000", reference), + ("02 Jan 2024 10:00:00 UT", reference), + ("Tue, 2 Jan 2024 13:00:00 +0300", reference), + ("Tue, 02 Jan 2024 05:00:00 EST", reference), + ("Tue, 02 Jan 2024 02:00:00 PST", reference), + ("Tuesday, 02 January 2024 10:00 GMT", reference), + ("Tue, 02 Jan 24 10:00:00 GMT", reference), + ("Tue, 02 Jan 2024 13:00:00 +03:00", reference), + ("Tue, 02 Jan 2024 10:00:00", reference), + ("Вт, 02 янв 2024 13:00:00 +0300", reference), + ("2024-01-02T10:00:00Z", reference), + ("2024-01-02T13:00:00+03:00", reference), + ("2024-01-02T10:00:00.123Z", reference), + ("2024-01-02 10:00:00", reference), + ("2024-01-02", Some(1_704_153_600)), + ("not a date", None), + ("Tue, 31 Feb 2024 10:00:00 GMT", None), + ("Tue, 02 Foo 2024 10:00:00 GMT", None), + ("2024-13-02T10:00:00Z", None), + ("", None), + ]; + for (input, expected) in cases { + assert_eq!(parse_date(input), expected, "{input:?}"); + } +} + +#[test] +fn non_ascii_zones_do_not_panic() { + for input in [ + "Mon, 07 Nov 2022 10:00:00 +й0", + "Mon, 07 Nov 2022 10:00:00 -йй", + "Mon, 07 Nov 2022 10:00+й0", + "Mon, 07 Nov 2022 10:00:00 +0й00", + "2022-11-07T10:00:00+0й0", + "2022-11-07T10:00:00+й", + "2022-11-07T10:00:00-€:00", + "2022-11-07T10:00:00+日本", + ] { + assert_eq!(parse_date(input), None, "{input:?}"); + } + // Exhaustive-ish: every short suffix of mixed ASCII / multi-byte + // characters after a sign. + let alphabet = ['0', '9', ':', 'й', '€', '日', '+', '-']; + for a in alphabet { + for b in alphabet { + for c in alphabet { + for prefix in ["Mon, 07 Nov 2022 10:00:00 ", "2022-11-07T10:00:00"] { + for sign in ['+', '-'] { + let _ = parse_date(&format!("{prefix}{sign}{a}{b}{c}")); + } + } + } + } + } + // In a feed, the bad date only loses the date. + let xml = rss("", &item("z", "Zone", "Mon, 07 Nov 2022 10:00:00 +й0")); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); + assert_eq!(feed.episodes[0].published_at_ms, None); +} + +#[test] +fn media_urls_are_resolved_and_must_be_http() { + let xml = rss( + r#""#, + r#"relRelative + + +protoProtocol relative + +fileLocal file + +ftpFTP + +fallbackMedia fallback + + +absAbsolute + "#, + ); + let feed = + parse_feed_with_base(xml.as_bytes(), Some("https://host.test/feeds/rss.xml")).unwrap(); + let urls: Vec<(&str, &str)> = feed + .episodes + .iter() + .map(|e| (e.guid.as_deref().unwrap_or_default(), e.audio_url.as_str())) + .collect(); + assert_eq!( + urls, + [ + ("rel", "https://host.test/audio/1.mp3"), + ("proto", "https://cdn.example.test/2.mp3"), + ("fallback", "https://cdn.example.test/4.mp3"), + ("abs", "HTTPS://cdn.example.test/5.mp3"), + ] + ); + assert_eq!( + feed.image_url.as_deref(), + Some("https://host.test/img/cover.jpg") + ); + assert_eq!( + feed.episodes[0].image_url.as_deref(), + Some("https://host.test/feeds/ep.png") + ); + for episode in &feed.episodes { + assert!(is_remote_url(&episode.audio_url), "{}", episode.audio_url); + } + + // Without a base, relative media cannot be resolved and is dropped; + // protocol-relative URLs default to https. + let feed = parse_feed(xml.as_bytes()).unwrap(); + let guids: Vec<&str> = feed + .episodes + .iter() + .map(|e| e.guid.as_deref().unwrap_or_default()) + .collect(); + assert_eq!(guids, ["proto", "fallback", "abs"]); + assert_eq!(feed.image_url, None); +} + +#[test] +fn rss_with_an_unrelated_default_namespace() { + for ns in ["http://example.test/x", "http://www.w3.org/2005/Atom"] { + let xml = format!( + r#"T + +1One +"# + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "T", "{ns}"); + assert_eq!(feed.episodes.len(), 1, "{ns}"); + assert_eq!(feed.episodes[0].title, "One"); + } +} + +#[test] +fn html_pages_and_garbage_are_rejected() { + let html = br#"Vivat

Hi   there

"#; + let err = parse_feed(html).unwrap_err().to_string(); + assert!(err.contains("not an RSS or Atom feed"), "{err}"); + for garbage in [ + &b""[..], + b"hello world", + b"\x00\xff\xfe garbage", + b"{\"json\": true}", + b"<", + b"<<<>>>", + ] { + let err = parse_feed(garbage).unwrap_err().to_string(); + assert!( + err.contains("not an RSS or Atom feed"), + "{garbage:?}: {err}" + ); + } +} + +#[test] +fn truncated_and_malformed_xml_never_panics() { + let items: String = (0..5) + .map(|i| item(&format!("g{i}"), &format!("T {i} &  "), "bad date")) + .collect(); + let xml = rss( + "", + &items, + ); + let bytes = xml.as_bytes(); + for len in 0..=bytes.len() { + let _ = parse_feed(&bytes[..len]); + } + // Mismatched end tags are tolerated. + let sloppy = r#"SA</b>1"#; + let feed = parse_feed(sloppy.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); + assert_eq!(feed.episodes[0].title, "A"); + // Truncation after complete items keeps them. + let third = xml.find("T 3").unwrap(); + let cut = &bytes[..xml[..third].rfind("<item>").unwrap()]; + let feed = parse_feed(cut).unwrap(); + assert_eq!(feed.episodes.len(), 3); +} + +#[test] +fn deep_nesting_does_not_overflow() { + let depth = 100_000; + let mut xml = String::from("<rss><channel><title>Deep"); + xml.push_str(&"".repeat(depth)); + xml.push_str(&"".repeat(depth)); + xml.push_str(&item("after", "After", "")); + xml.push_str(""); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes.len(), 1); +} + +#[test] +fn cyrillic_titles_in_utf8_and_windows_1251() { + let xml = rss( + "Иван Петров", + &item( + "1", + "Выпуск №1: «Привет»", + "Mon, 01 Jan 2024 00:00:00 +0300", + ), + ) + .replace("Test Cast", "Подкаст"); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.title, "Подкаст"); + assert_eq!(feed.author.as_deref(), Some("Иван Петров")); + assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»"); + + let cp1251 = xml.replace("encoding=\"UTF-8\"", "encoding=\"windows-1251\""); + let (encoded, _, unmappable) = encoding_rs::WINDOWS_1251.encode(&cp1251); + assert!(!unmappable); + let feed = parse_feed(&encoded).unwrap(); + assert_eq!(feed.title, "Подкаст"); + assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»"); + + // Invalid UTF-8 without a declaration is decoded lossily. + let mut broken = xml.into_bytes(); + let at = broken + .windows(7) + .position(|w| w == b"Test Ca") + .unwrap_or(10); + broken.insert(at, 0xff); + assert!(parse_feed(&broken).is_ok()); +} + +#[test] +fn namespace_prefixes_are_matched_tolerantly() { + // Undeclared `itunes:` prefix and a custom prefix bound to the + // iTunes namespace. + let xml = format!( + r#"PUndeclared +1Custom prefix90 +"# + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.author.as_deref(), Some("Undeclared")); + assert_eq!(feed.episodes[0].title, "Custom prefix"); + assert_eq!(feed.episodes[0].duration_ms, Some(90_000)); +} + +#[test] +fn long_descriptions_are_truncated() { + let long = "word ".repeat(2000); + let text = html_to_text(&long, 4000); + assert!(text.chars().count() <= 4000); + assert!(text.ends_with('…')); + let xml = rss(&format!("{long}"), ""); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!(feed.description.unwrap().chars().count() <= 4000); +} + +#[test] +fn itunes_season_episode_and_type() { + let xml = rss( + "", + r#"aA + 2 7 + Trailer + +bB + two-1 + mystery + +cCbonus + 99999999999 + "#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let a = &feed.episodes[0]; + assert_eq!((a.season, a.episode_number), (Some(2), Some(7))); + assert_eq!(a.episode_type, EpisodeType::Trailer); + let b = &feed.episodes[1]; + assert_eq!((b.season, b.episode_number), (None, None)); + assert_eq!(b.episode_type, EpisodeType::Full); + let c = &feed.episodes[2]; + assert_eq!((c.season, c.episode_number), (None, None)); + assert_eq!(c.episode_type, EpisodeType::Bonus); +} + +#[test] +fn guid_permalink_flag() { + let xml = rss( + "", + r#"x +https://e.test/y +z +"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + let flags: Vec> = feed.episodes.iter().map(|e| e.guid_is_permalink).collect(); + assert_eq!(flags, [Some(false), Some(true), Some(false), None]); +} + +#[test] +fn new_feed_url_is_reported() { + let xml = rss( + " https://new.example.test/feed?x=1 ", + "", + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!( + feed.new_feed_url.as_deref(), + Some("https://new.example.test/feed?x=1") + ); + let alias = FeedUrl::parse(feed.new_feed_url.as_deref().unwrap()).unwrap(); + assert_eq!(alias.as_str(), "https://new.example.test/feed?x=1"); +} + +#[test] +fn oversized_guid_is_treated_as_absent() { + let long = "g".repeat(MAX_GUID_BYTES + 1); + let xml = rss( + "", + &format!(r#"{long}"#), + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes[0].guid, None); +} + +#[test] +fn media_urls_with_spaces_are_percent_encoded() { + let xml = rss( + "", + r#"s"#, + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert_eq!(feed.episodes[0].audio_url, "https://c.test/my%20file.mp3"); +} + +#[test] +fn parsed_feed_maps_to_valid_contract_metadata() { + let base = FeedUrl::parse("https://example.test/feed.xml").unwrap(); + let items = [ + item("g1", "One", "Tue, 02 Jan 2024 10:00:00 GMT"), + r#"No guidhttps://example.test/no-guid + Wed, 03 Jan 2024 10:00:00 GMT + "# + .to_string(), + // Same page as the channel link: not identifying. + r#"Home linkhttps://example.test/cast + "# + .to_string(), + ] + .concat(); + let feed = parse_feed_with_base(rss("", &items).as_bytes(), Some(base.as_str())).unwrap(); + let podcast_id = PodcastId::from_feed_url(&base); + let ids = feed.resolve_episode_ids(&podcast_id, &[]); + assert_eq!(ids[0], EpisodeId::from_guid(&podcast_id, "g1")); + assert_eq!( + ids[1], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::link("https://example.test/no-guid").unwrap() + )) + ); + assert_eq!( + ids[2], + Some(EpisodeId::derive( + &podcast_id, + &crate::podcast::EpisodeKey::enclosure("https://cdn.example.test/home.mp3").unwrap() + )) + ); + + let podcast = feed.to_podcast_meta(base.clone()); + podcast.validate().unwrap(); + for (episode, id) in feed.episodes.iter().zip(ids) { + let meta = episode.to_meta(&podcast_id, id.unwrap()); + meta.validate().unwrap(); + meta.to_ref(&podcast).validate().unwrap(); + } + + // A later refresh with a changed title and audio URL keeps every id, + // using the stored hints of the previous refresh. + let known: Vec = feed + .episodes + .iter() + .zip(feed.resolve_episode_ids(&podcast_id, &[])) + .map(|(episode, id)| KnownEpisode { + episode_id: id.unwrap(), + hints: episode.identity_hints(), + }) + .collect(); + let refreshed = items + .replace("Home link", "Home link (edited)") + .replace("cdn.example.test/ng.mp3", "cdn2.example.test/ng.mp3"); + let feed2 = parse_feed_with_base(rss("", &refreshed).as_bytes(), Some(base.as_str())).unwrap(); + let ids2 = feed2.resolve_episode_ids(&podcast_id, &known); + let expected: Vec> = + known.iter().map(|k| Some(k.episode_id.clone())).collect(); + assert_eq!(ids2, expected); +} + +#[test] +fn hostile_sizes_stay_bounded() { + let huge_title = "t".repeat(MAX_CAPTURE_BYTES * 2); + let xml = rss( + "", + &format!( + r#"h{huge_title}"# + ), + ); + let feed = parse_feed(xml.as_bytes()).unwrap(); + assert!(feed.episodes[0].title.chars().count() <= MAX_TITLE_CHARS); + let duration = rss( + "", + r#"d99999999999999999999"#, + ); + assert_eq!( + parse_feed(duration.as_bytes()).unwrap().episodes[0].duration_ms, + None + ); +} diff --git a/crates/music-dht/src/podcast/mod.rs b/crates/music-dht/src/podcast/mod.rs index a9d6e43..d4b2bb6 100644 --- a/crates/music-dht/src/podcast/mod.rs +++ b/crates/music-dht/src/podcast/mod.rs @@ -46,6 +46,12 @@ //! playback queues and progress operations. A receiving device can show and //! play the episode without being subscribed to the podcast. //! +//! # Feed parser +//! +//! With the `podcast-feed` cargo feature, `feed` provides the shared tolerant +//! RSS 2.0 / RSS 1.0 / Atom parser that produces the identity hints consumed +//! by [`resolve_episode_ids`]. +//! //! ``` //! use music_dht::podcast::{EpisodeId, FeedUrl, PodcastId}; //! @@ -63,6 +69,9 @@ mod identity; mod meta; mod state; +#[cfg(feature = "podcast-feed")] +pub mod feed; + pub use feed_url::{FeedUrl, merge_feed_aliases, resolve_podcast_id}; pub use id::{ EPISODE_ID_DOMAIN, EPISODE_ID_PREFIX, EpisodeId, EpisodeKey, PODCAST_ID_DOMAIN,