Make podcast episode identity order-independent and linear

Run the no-GUID matching stages across the whole document so a weak
date+title match of an earlier item cannot take a known episode that a
later item matches by enclosure. Keep feed links verbatim (trimmed only)
because they are identity material, and stop html_to_text from scanning
to the end of the text at every '<' when no '>' follows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-19 16:10:33 +02:00
1 parent 4f7e999280
commit 629857dbfe
3 files changed
+219 -49

No files matched your search

+27 -15
View File
@@ -61,7 +61,8 @@ pub struct ParsedFeed {
pub author: Option<String>,
/// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`].
pub description: Option<String>,
/// Publisher web page.
/// Publisher web page: `<link>` text or Atom `href`, trimmed, verbatim
/// (identity material, see [`resolve_episode_ids`]).
pub link: Option<String>,
/// Absolute http(s) artwork URL.
pub image_url: Option<String>,
@@ -101,7 +102,8 @@ pub struct ParsedEpisode {
pub mime_type: Option<String>,
/// Enclosure length in bytes.
pub byte_len: Option<u64>,
/// Item web page.
/// Item web page: `<link>` text or Atom `href`, trimmed, verbatim
/// (identity material, see [`resolve_episode_ids`]).
pub link: Option<String>,
/// Absolute http(s) artwork URL.
pub image_url: Option<String>,
@@ -662,7 +664,7 @@ impl Parser {
let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase();
if rel.is_empty() || rel == "alternate" {
let href = attrs.get("href").unwrap_or_default();
set_once(&mut self.feed.link, clean_short(&href));
set_once(&mut self.feed.link, href.trim().to_string());
}
None
}
@@ -770,7 +772,7 @@ impl Parser {
});
}
} else if rel.is_empty() || rel == "alternate" {
set_once(&mut item.link, clean_short(&href));
set_once(&mut item.link, href.trim().to_string());
}
None
}
@@ -970,11 +972,7 @@ impl Parser {
audio_url: media.url,
mime_type: media.mime,
byte_len: media.len,
link: item
.link
.as_deref()
.map(clean_short)
.filter(|s| !s.is_empty()),
link: item.link.as_deref().and_then(raw_link),
image_url: [item.itunes_image, item.thumbnail]
.into_iter()
.flatten()
@@ -1016,11 +1014,7 @@ impl Parser {
title,
author,
description,
link: feed
.link
.as_deref()
.map(clean_short)
.filter(|s| !s.is_empty()),
link: feed.link.as_deref().and_then(raw_link),
image_url,
new_feed_url: feed
.new_feed_url
@@ -1199,6 +1193,17 @@ fn parse_number(text: &str) -> Option<u32> {
// ---------------------------------------------------------------------------
// Text cleanup
/// `<link>` text (already XML-unescaped) or Atom `href`, trimmed.
///
/// Links are identity material for GUID-less episodes (`EpisodeKey::Link`),
/// so they are deliberately **not** passed through the display text cleaner:
/// no tag stripping, second entity decoding, whitespace collapsing or
/// truncation. Links that are empty or longer than `MAX_URL_BYTES` are absent.
fn raw_link(text: &str) -> Option<String> {
let link = text.trim();
(!link.is_empty() && link.len() <= super::MAX_URL_BYTES).then(|| link.to_string())
}
fn clean_short(text: &str) -> String {
html_to_text(text, MAX_TITLE_CHARS)
}
@@ -1248,10 +1253,17 @@ pub(crate) fn html_to_text(input: &str, max_chars: usize) -> String {
.chars()
.next()
.is_some_and(|c| c.is_ascii_alphabetic() || matches!(c, '/' | '!' | '?'));
let Some(gt) = after.find('>').filter(|_| looks_like_tag) else {
if !looks_like_tag {
stripped.push('<');
rest = after;
continue;
}
let Some(gt) = after.find('>') else {
// No '>' anywhere after this point: nothing else can be a tag.
// Stopping here keeps the scan linear on text full of '<'.
stripped.push('<');
rest = after;
break;
};
let tag = &after[..gt];
rest = &after[gt + 1..];
@@ -663,3 +663,69 @@ fn hostile_sizes_stay_bounded() {
None
);
}
#[test]
fn text_full_of_unclosed_angle_brackets_is_linear() {
// '<a' repeated up to the capture cap, with no '>' anywhere: the old
// scanner searched for '>' to the end of the text at every '<'.
let body = "&lt;a".repeat(MAX_CAPTURE_BYTES / 2);
let xml = rss(
"",
&format!(
r#"<item><guid>lt</guid><title>{body}</title><description>{body}</description><enclosure url="https://c.test/lt.mp3"/></item>"#
),
);
let started = std::time::Instant::now();
let feed = parse_feed(xml.as_bytes()).unwrap();
assert!(
started.elapsed() < std::time::Duration::from_secs(2),
"took {:?}",
started.elapsed()
);
let episode = &feed.episodes[0];
assert!(episode.title.starts_with("<a<a"));
assert!(episode.title.chars().count() <= MAX_TITLE_CHARS);
assert_eq!(
html_to_text("a < b and <b>bold</b> <c", 100),
"a < b and bold <c"
);
}
#[test]
fn links_are_identity_material_and_kept_verbatim() {
// Entities are decoded once by the XML reader and never again; tag-like
// text, inner whitespace and long links are not rewritten by the display
// cleaner.
let long_path = "p".repeat(1_500);
let items = format!(
r#"<item><title>A</title><link> https://e.test/a?x=1&amp;amp;y=&lt;b&gt; </link><enclosure url="https://c.test/a.mp3"/></item>
<item><title>B</title><link>https://e.test/{long_path}</link><enclosure url="https://c.test/b.mp3"/></item>"#
);
let feed = parse_feed(rss("", &items).as_bytes()).unwrap();
assert_eq!(
feed.episodes[0].link.as_deref(),
Some("https://e.test/a?x=1&amp;y=<b>")
);
let long = format!("https://e.test/{long_path}");
assert_eq!(feed.episodes[1].link.as_deref(), Some(long.as_str()));
let podcast_id =
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed").unwrap());
let ids = feed.resolve_episode_ids(&podcast_id, &[]);
assert_eq!(
ids[1],
Some(EpisodeId::derive(
&podcast_id,
&crate::podcast::EpisodeKey::link(&long).unwrap()
))
);
let atom = r#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>T</title>
<link href=" https://e.test/home?a=1&amp;b=2 "/>
<entry><title>E</title><link rel="alternate" href="https://e.test/e?q=&lt;x&gt;"/>
<link rel="enclosure" href="https://c.test/e.mp3" type="audio/mpeg"/></entry></feed>"#;
let feed = parse_feed(atom.as_bytes()).unwrap();
assert_eq!(feed.link.as_deref(), Some("https://e.test/home?a=1&b=2"));
assert_eq!(
feed.episodes[0].link.as_deref(),
Some("https://e.test/e?q=<x>")
);
}
+126 -34
View File
@@ -57,6 +57,9 @@ impl EpisodeIdentityHints {
}
}
/// One matching stage of the no-GUID rule: does `known` match `item`?
type StageMatcher<'f> = &'f dyn Fn(&Usable<'_>, &EpisodeIdentityHints) -> bool;
struct Usable<'a> {
link: Option<&'a str>,
enclosure: Option<&'a str>,
@@ -103,11 +106,13 @@ pub fn resolve_episode_id(
///
/// 1. An item with a GUID gets `EpisodeId::from_guid`. A GUID never matches
/// a different known episode, so the id is independent of local history.
/// 2. An item without a GUID first looks for a known episode, in this order:
/// same enclosure URL, then same `<link>`, then same publication second and
/// [`normalize_episode_title`]. The first stage with candidates wins; among
/// candidates the smallest id is reused. A known episode already claimed by
/// another item of this document is not a candidate.
/// 2. Items without a GUID then look for known episodes in three stages run
/// over the whole document: same enclosure URL, then same `<link>`, then
/// same publication second and [`normalize_episode_title`]. A stage
/// assigns matches for all remaining items before the next stage starts;
/// among candidates the smallest id is reused. A known episode already
/// claimed (by a GUID item or an earlier stage) is not a candidate, so the
/// result does not depend on item order.
/// 3. Otherwise a new id is derived from the first available key: `<link>`,
/// then `(date, title)`, then enclosure URL.
///
@@ -166,40 +171,47 @@ pub fn resolve_episode_ids(
}
}
// Pass 2: GUID-less items reuse a matching known episode.
let mut pending = Vec::new();
for (index, item) in items.iter().enumerate() {
if item.guid_key().is_some() {
continue;
}
let own = &usable[index];
let stages: [&dyn Fn(&EpisodeIdentityHints) -> bool; 3] = [
&|candidate| {
own.enclosure
.is_some_and(|url| candidate.enclosure() == Some(url))
},
&|candidate| own.link.is_some_and(|link| candidate.link() == Some(link)),
&|candidate| {
own.date_title
.as_ref()
.is_some_and(|pair| candidate.date_title().as_ref() == Some(pair))
},
];
let reused = stages.iter().find_map(|matches| {
known
// Pass 2: GUID-less items reuse a matching known episode. Stages run
// across the whole document: every enclosure match is assigned before any
// link match, and every link match before any (date, title) match, so a
// weak match of an earlier item cannot steal the known episode that a
// later item matches strongly. Within one stage an item's key is unique in
// the document, so two items never compete for the same known episode
// and the result does not depend on item order.
let mut pending: Vec<usize> = (0..items.len())
.filter(|index| items[*index].guid_key().is_none())
.collect();
let stages: [StageMatcher<'_>; 3] = [
&|own, candidate| {
own.enclosure
.is_some_and(|url| candidate.enclosure() == Some(url))
},
&|own, candidate| own.link.is_some_and(|link| candidate.link() == Some(link)),
&|own, candidate| {
own.date_title
.as_ref()
.is_some_and(|pair| candidate.date_title().as_ref() == Some(pair))
},
];
for matches in stages {
let mut unmatched = Vec::with_capacity(pending.len());
for index in pending {
let own = &usable[index];
let reused = known
.iter()
.filter(|known| !claimed.contains(&known.episode_id) && matches(&known.hints))
.filter(|known| !claimed.contains(&known.episode_id) && matches(own, &known.hints))
.map(|known| &known.episode_id)
.min()
.cloned()
});
match reused {
Some(id) => {
claimed.insert(id.clone());
out[index] = Some(id);
.cloned();
match reused {
Some(id) => {
claimed.insert(id.clone());
out[index] = Some(id);
}
None => unmatched.push(index),
}
None => pending.push(index),
}
pending = unmatched;
}
// Pass 3: derive new ids from the first available fallback key.
@@ -509,4 +521,84 @@ mod tests {
assert_eq!(resolve_episode_ids(&p, None, &items, &known), forward);
assert!(forward.iter().all(Option::is_some));
}
#[test]
fn strong_matches_beat_earlier_weak_matches_in_any_item_order() {
let p = podcast();
let k1 = EpisodeId::from_guid(&p, "legacy-k1").unwrap();
let known = [KnownEpisode {
episode_id: k1.clone(),
hints: hints(
None,
Some("https://cdn/e1.mp3"),
None,
Some(5_000),
Some("Same"),
),
}];
// A new item that only matches K1 by (date, title), and the real K1
// (same enclosure) renamed and re-dated by the publisher.
let weak = hints(
None,
Some("https://cdn/new.mp3"),
None,
Some(5_000),
Some("Same"),
);
let strong = hints(
None,
Some("https://cdn/e1.mp3"),
None,
Some(9_000),
Some("Renamed"),
);
let forward = resolve_episode_ids(&p, None, &[weak.clone(), strong.clone()], &known);
assert_eq!(forward[1].as_ref(), Some(&k1), "enclosure match keeps K1");
assert_ne!(forward[0].as_ref(), Some(&k1));
let reverse = resolve_episode_ids(&p, None, &[strong, weak], &known);
assert_eq!(reverse, vec![forward[1].clone(), forward[0].clone()]);
}
#[test]
fn result_is_independent_of_item_order() {
let p = podcast();
let items: Vec<EpisodeIdentityHints> = (0..5)
.map(|i| {
hints(
None,
Some(&format!("https://cdn/{i}.mp3")),
(i % 2 == 0).then(|| format!("https://e/{i}")).as_deref(),
Some((i % 3) * 1_000),
Some(&format!("Ep {}", i % 2)),
)
})
.collect();
// Known episodes whose hints overlap different items at different
// stages.
let known: Vec<KnownEpisode> = (0..5)
.map(|i| KnownEpisode {
episode_id: EpisodeId::from_guid(&p, &format!("legacy-{i}")).unwrap(),
hints: hints(
None,
Some(&format!("https://cdn/{}.mp3", (i + 2) % 7)),
Some(&format!("https://e/{}", (i + 1) % 5)),
Some((i % 3) * 1_000),
Some(&format!("Ep {}", i % 2)),
),
})
.collect();
let reference = resolve_episode_ids(&p, None, &items, &known);
let mut order: Vec<usize> = (0..items.len()).collect();
for _ in 0..20 {
order.rotate_left(1);
let middle = order.len() / 2;
order.swap(0, middle);
let shuffled: Vec<EpisodeIdentityHints> =
order.iter().map(|i| items[*i].clone()).collect();
let result = resolve_episode_ids(&p, None, &shuffled, &known);
for (position, index) in order.iter().enumerate() {
assert_eq!(result[position], reference[*index], "order {order:?}");
}
}
}
}