Make podcast episode identity order-independent and linear
Run the no-GUID matching stages across the whole document so a weak date+title match of an earlier item cannot take a known episode that a later item matches by enclosure. Keep feed links verbatim (trimmed only) because they are identity material, and stop html_to_text from scanning to the end of the text at every '<' when no '>' follows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4f7e999280
commit
629857dbfe
3 files changed
+219
-49
No files matched your search
@@ -61,7 +61,8 @@ pub struct ParsedFeed {
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`].
|
||||
pub description: Option<String>,
|
||||
/// Publisher web page.
|
||||
/// Publisher web page: `<link>` text or Atom `href`, trimmed, verbatim
|
||||
/// (identity material, see [`resolve_episode_ids`]).
|
||||
pub link: Option<String>,
|
||||
/// Absolute http(s) artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
@@ -101,7 +102,8 @@ pub struct ParsedEpisode {
|
||||
pub mime_type: Option<String>,
|
||||
/// Enclosure length in bytes.
|
||||
pub byte_len: Option<u64>,
|
||||
/// Item web page.
|
||||
/// Item web page: `<link>` text or Atom `href`, trimmed, verbatim
|
||||
/// (identity material, see [`resolve_episode_ids`]).
|
||||
pub link: Option<String>,
|
||||
/// Absolute http(s) artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
@@ -662,7 +664,7 @@ impl Parser {
|
||||
let rel = attrs.get("rel").unwrap_or_default().to_ascii_lowercase();
|
||||
if rel.is_empty() || rel == "alternate" {
|
||||
let href = attrs.get("href").unwrap_or_default();
|
||||
set_once(&mut self.feed.link, clean_short(&href));
|
||||
set_once(&mut self.feed.link, href.trim().to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -770,7 +772,7 @@ impl Parser {
|
||||
});
|
||||
}
|
||||
} else if rel.is_empty() || rel == "alternate" {
|
||||
set_once(&mut item.link, clean_short(&href));
|
||||
set_once(&mut item.link, href.trim().to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -970,11 +972,7 @@ impl Parser {
|
||||
audio_url: media.url,
|
||||
mime_type: media.mime,
|
||||
byte_len: media.len,
|
||||
link: item
|
||||
.link
|
||||
.as_deref()
|
||||
.map(clean_short)
|
||||
.filter(|s| !s.is_empty()),
|
||||
link: item.link.as_deref().and_then(raw_link),
|
||||
image_url: [item.itunes_image, item.thumbnail]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
@@ -1016,11 +1014,7 @@ impl Parser {
|
||||
title,
|
||||
author,
|
||||
description,
|
||||
link: feed
|
||||
.link
|
||||
.as_deref()
|
||||
.map(clean_short)
|
||||
.filter(|s| !s.is_empty()),
|
||||
link: feed.link.as_deref().and_then(raw_link),
|
||||
image_url,
|
||||
new_feed_url: feed
|
||||
.new_feed_url
|
||||
@@ -1199,6 +1193,17 @@ fn parse_number(text: &str) -> Option<u32> {
|
||||
// ---------------------------------------------------------------------------
|
||||
// Text cleanup
|
||||
|
||||
/// `<link>` text (already XML-unescaped) or Atom `href`, trimmed.
|
||||
///
|
||||
/// Links are identity material for GUID-less episodes (`EpisodeKey::Link`),
|
||||
/// so they are deliberately **not** passed through the display text cleaner:
|
||||
/// no tag stripping, second entity decoding, whitespace collapsing or
|
||||
/// truncation. Links that are empty or longer than `MAX_URL_BYTES` are absent.
|
||||
fn raw_link(text: &str) -> Option<String> {
|
||||
let link = text.trim();
|
||||
(!link.is_empty() && link.len() <= super::MAX_URL_BYTES).then(|| link.to_string())
|
||||
}
|
||||
|
||||
fn clean_short(text: &str) -> String {
|
||||
html_to_text(text, MAX_TITLE_CHARS)
|
||||
}
|
||||
@@ -1248,10 +1253,17 @@ pub(crate) fn html_to_text(input: &str, max_chars: usize) -> String {
|
||||
.chars()
|
||||
.next()
|
||||
.is_some_and(|c| c.is_ascii_alphabetic() || matches!(c, '/' | '!' | '?'));
|
||||
let Some(gt) = after.find('>').filter(|_| looks_like_tag) else {
|
||||
if !looks_like_tag {
|
||||
stripped.push('<');
|
||||
rest = after;
|
||||
continue;
|
||||
}
|
||||
let Some(gt) = after.find('>') else {
|
||||
// No '>' anywhere after this point: nothing else can be a tag.
|
||||
// Stopping here keeps the scan linear on text full of '<'.
|
||||
stripped.push('<');
|
||||
rest = after;
|
||||
break;
|
||||
};
|
||||
let tag = &after[..gt];
|
||||
rest = &after[gt + 1..];
|
||||
|
||||
@@ -663,3 +663,69 @@ fn hostile_sizes_stay_bounded() {
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_full_of_unclosed_angle_brackets_is_linear() {
|
||||
// '<a' repeated up to the capture cap, with no '>' anywhere: the old
|
||||
// scanner searched for '>' to the end of the text at every '<'.
|
||||
let body = "<a".repeat(MAX_CAPTURE_BYTES / 2);
|
||||
let xml = rss(
|
||||
"",
|
||||
&format!(
|
||||
r#"<item><guid>lt</guid><title>{body}</title><description>{body}</description><enclosure url="https://c.test/lt.mp3"/></item>"#
|
||||
),
|
||||
);
|
||||
let started = std::time::Instant::now();
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(2),
|
||||
"took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
let episode = &feed.episodes[0];
|
||||
assert!(episode.title.starts_with("<a<a"));
|
||||
assert!(episode.title.chars().count() <= MAX_TITLE_CHARS);
|
||||
assert_eq!(
|
||||
html_to_text("a < b and <b>bold</b> <c", 100),
|
||||
"a < b and bold <c"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn links_are_identity_material_and_kept_verbatim() {
|
||||
// Entities are decoded once by the XML reader and never again; tag-like
|
||||
// text, inner whitespace and long links are not rewritten by the display
|
||||
// cleaner.
|
||||
let long_path = "p".repeat(1_500);
|
||||
let items = format!(
|
||||
r#"<item><title>A</title><link> https://e.test/a?x=1&amp;y=<b> </link><enclosure url="https://c.test/a.mp3"/></item>
|
||||
<item><title>B</title><link>https://e.test/{long_path}</link><enclosure url="https://c.test/b.mp3"/></item>"#
|
||||
);
|
||||
let feed = parse_feed(rss("", &items).as_bytes()).unwrap();
|
||||
assert_eq!(
|
||||
feed.episodes[0].link.as_deref(),
|
||||
Some("https://e.test/a?x=1&y=<b>")
|
||||
);
|
||||
let long = format!("https://e.test/{long_path}");
|
||||
assert_eq!(feed.episodes[1].link.as_deref(), Some(long.as_str()));
|
||||
let podcast_id =
|
||||
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed").unwrap());
|
||||
let ids = feed.resolve_episode_ids(&podcast_id, &[]);
|
||||
assert_eq!(
|
||||
ids[1],
|
||||
Some(EpisodeId::derive(
|
||||
&podcast_id,
|
||||
&crate::podcast::EpisodeKey::link(&long).unwrap()
|
||||
))
|
||||
);
|
||||
let atom = r#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>T</title>
|
||||
<link href=" https://e.test/home?a=1&b=2 "/>
|
||||
<entry><title>E</title><link rel="alternate" href="https://e.test/e?q=<x>"/>
|
||||
<link rel="enclosure" href="https://c.test/e.mp3" type="audio/mpeg"/></entry></feed>"#;
|
||||
let feed = parse_feed(atom.as_bytes()).unwrap();
|
||||
assert_eq!(feed.link.as_deref(), Some("https://e.test/home?a=1&b=2"));
|
||||
assert_eq!(
|
||||
feed.episodes[0].link.as_deref(),
|
||||
Some("https://e.test/e?q=<x>")
|
||||
);
|
||||
}
|
||||
@@ -57,6 +57,9 @@ impl EpisodeIdentityHints {
|
||||
}
|
||||
}
|
||||
|
||||
/// One matching stage of the no-GUID rule: does `known` match `item`?
|
||||
type StageMatcher<'f> = &'f dyn Fn(&Usable<'_>, &EpisodeIdentityHints) -> bool;
|
||||
|
||||
struct Usable<'a> {
|
||||
link: Option<&'a str>,
|
||||
enclosure: Option<&'a str>,
|
||||
@@ -103,11 +106,13 @@ pub fn resolve_episode_id(
|
||||
///
|
||||
/// 1. An item with a GUID gets `EpisodeId::from_guid`. A GUID never matches
|
||||
/// a different known episode, so the id is independent of local history.
|
||||
/// 2. An item without a GUID first looks for a known episode, in this order:
|
||||
/// same enclosure URL, then same `<link>`, then same publication second and
|
||||
/// [`normalize_episode_title`]. The first stage with candidates wins; among
|
||||
/// candidates the smallest id is reused. A known episode already claimed by
|
||||
/// another item of this document is not a candidate.
|
||||
/// 2. Items without a GUID then look for known episodes in three stages run
|
||||
/// over the whole document: same enclosure URL, then same `<link>`, then
|
||||
/// same publication second and [`normalize_episode_title`]. A stage
|
||||
/// assigns matches for all remaining items before the next stage starts;
|
||||
/// among candidates the smallest id is reused. A known episode already
|
||||
/// claimed (by a GUID item or an earlier stage) is not a candidate, so the
|
||||
/// result does not depend on item order.
|
||||
/// 3. Otherwise a new id is derived from the first available key: `<link>`,
|
||||
/// then `(date, title)`, then enclosure URL.
|
||||
///
|
||||
@@ -166,40 +171,47 @@ pub fn resolve_episode_ids(
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: GUID-less items reuse a matching known episode.
|
||||
let mut pending = Vec::new();
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if item.guid_key().is_some() {
|
||||
continue;
|
||||
}
|
||||
let own = &usable[index];
|
||||
let stages: [&dyn Fn(&EpisodeIdentityHints) -> bool; 3] = [
|
||||
&|candidate| {
|
||||
own.enclosure
|
||||
.is_some_and(|url| candidate.enclosure() == Some(url))
|
||||
},
|
||||
&|candidate| own.link.is_some_and(|link| candidate.link() == Some(link)),
|
||||
&|candidate| {
|
||||
own.date_title
|
||||
.as_ref()
|
||||
.is_some_and(|pair| candidate.date_title().as_ref() == Some(pair))
|
||||
},
|
||||
];
|
||||
let reused = stages.iter().find_map(|matches| {
|
||||
known
|
||||
// Pass 2: GUID-less items reuse a matching known episode. Stages run
|
||||
// across the whole document: every enclosure match is assigned before any
|
||||
// link match, and every link match before any (date, title) match, so a
|
||||
// weak match of an earlier item cannot steal the known episode that a
|
||||
// later item matches strongly. Within one stage an item's key is unique in
|
||||
// the document, so two items never compete for the same known episode
|
||||
// and the result does not depend on item order.
|
||||
let mut pending: Vec<usize> = (0..items.len())
|
||||
.filter(|index| items[*index].guid_key().is_none())
|
||||
.collect();
|
||||
let stages: [StageMatcher<'_>; 3] = [
|
||||
&|own, candidate| {
|
||||
own.enclosure
|
||||
.is_some_and(|url| candidate.enclosure() == Some(url))
|
||||
},
|
||||
&|own, candidate| own.link.is_some_and(|link| candidate.link() == Some(link)),
|
||||
&|own, candidate| {
|
||||
own.date_title
|
||||
.as_ref()
|
||||
.is_some_and(|pair| candidate.date_title().as_ref() == Some(pair))
|
||||
},
|
||||
];
|
||||
for matches in stages {
|
||||
let mut unmatched = Vec::with_capacity(pending.len());
|
||||
for index in pending {
|
||||
let own = &usable[index];
|
||||
let reused = known
|
||||
.iter()
|
||||
.filter(|known| !claimed.contains(&known.episode_id) && matches(&known.hints))
|
||||
.filter(|known| !claimed.contains(&known.episode_id) && matches(own, &known.hints))
|
||||
.map(|known| &known.episode_id)
|
||||
.min()
|
||||
.cloned()
|
||||
});
|
||||
match reused {
|
||||
Some(id) => {
|
||||
claimed.insert(id.clone());
|
||||
out[index] = Some(id);
|
||||
.cloned();
|
||||
match reused {
|
||||
Some(id) => {
|
||||
claimed.insert(id.clone());
|
||||
out[index] = Some(id);
|
||||
}
|
||||
None => unmatched.push(index),
|
||||
}
|
||||
None => pending.push(index),
|
||||
}
|
||||
pending = unmatched;
|
||||
}
|
||||
|
||||
// Pass 3: derive new ids from the first available fallback key.
|
||||
@@ -509,4 +521,84 @@ mod tests {
|
||||
assert_eq!(resolve_episode_ids(&p, None, &items, &known), forward);
|
||||
assert!(forward.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strong_matches_beat_earlier_weak_matches_in_any_item_order() {
|
||||
let p = podcast();
|
||||
let k1 = EpisodeId::from_guid(&p, "legacy-k1").unwrap();
|
||||
let known = [KnownEpisode {
|
||||
episode_id: k1.clone(),
|
||||
hints: hints(
|
||||
None,
|
||||
Some("https://cdn/e1.mp3"),
|
||||
None,
|
||||
Some(5_000),
|
||||
Some("Same"),
|
||||
),
|
||||
}];
|
||||
// A new item that only matches K1 by (date, title), and the real K1
|
||||
// (same enclosure) renamed and re-dated by the publisher.
|
||||
let weak = hints(
|
||||
None,
|
||||
Some("https://cdn/new.mp3"),
|
||||
None,
|
||||
Some(5_000),
|
||||
Some("Same"),
|
||||
);
|
||||
let strong = hints(
|
||||
None,
|
||||
Some("https://cdn/e1.mp3"),
|
||||
None,
|
||||
Some(9_000),
|
||||
Some("Renamed"),
|
||||
);
|
||||
let forward = resolve_episode_ids(&p, None, &[weak.clone(), strong.clone()], &known);
|
||||
assert_eq!(forward[1].as_ref(), Some(&k1), "enclosure match keeps K1");
|
||||
assert_ne!(forward[0].as_ref(), Some(&k1));
|
||||
let reverse = resolve_episode_ids(&p, None, &[strong, weak], &known);
|
||||
assert_eq!(reverse, vec![forward[1].clone(), forward[0].clone()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_is_independent_of_item_order() {
|
||||
let p = podcast();
|
||||
let items: Vec<EpisodeIdentityHints> = (0..5)
|
||||
.map(|i| {
|
||||
hints(
|
||||
None,
|
||||
Some(&format!("https://cdn/{i}.mp3")),
|
||||
(i % 2 == 0).then(|| format!("https://e/{i}")).as_deref(),
|
||||
Some((i % 3) * 1_000),
|
||||
Some(&format!("Ep {}", i % 2)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
// Known episodes whose hints overlap different items at different
|
||||
// stages.
|
||||
let known: Vec<KnownEpisode> = (0..5)
|
||||
.map(|i| KnownEpisode {
|
||||
episode_id: EpisodeId::from_guid(&p, &format!("legacy-{i}")).unwrap(),
|
||||
hints: hints(
|
||||
None,
|
||||
Some(&format!("https://cdn/{}.mp3", (i + 2) % 7)),
|
||||
Some(&format!("https://e/{}", (i + 1) % 5)),
|
||||
Some((i % 3) * 1_000),
|
||||
Some(&format!("Ep {}", i % 2)),
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
let reference = resolve_episode_ids(&p, None, &items, &known);
|
||||
let mut order: Vec<usize> = (0..items.len()).collect();
|
||||
for _ in 0..20 {
|
||||
order.rotate_left(1);
|
||||
let middle = order.len() / 2;
|
||||
order.swap(0, middle);
|
||||
let shuffled: Vec<EpisodeIdentityHints> =
|
||||
order.iter().map(|i| items[*i].clone()).collect();
|
||||
let result = resolve_episode_ids(&p, None, &shuffled, &known);
|
||||
for (position, index) in order.iter().enumerate() {
|
||||
assert_eq!(result[position], reference[*index], "order {order:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user