diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 488c497..29b1a1b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -320,10 +320,14 @@ registers (subscription, share, progress) and listening sessions in The TUI owns what the contract leaves to clients: - **Fetching.** Feeds are fetched with conditional requests (ETag, - Last-Modified), bounded size and time, and redirects followed by hand so the - store learns whether every hop was permanent; only then does the new URL - become a feed alias and the fetch URL. Validators are dropped after a - permanent redirect because they belong to the old address. + Last-Modified), bounded size and time, and redirects followed by hand. The + address that the leading permanent hops (301/308) lead to becomes a feed + alias and the fetch URL, even when a temporary hop (CDN, tracker) follows + it; a chain that starts with a temporary hop never moves the feed. + Validators are dropped after a permanent redirect because they belong to + the old address. The prototype database's validators are dropped once at + import, so the first check parses the feed (seasons and episode numbers + were not stored by the prototype). - **Scheduling.** The store keeps each feed's local fetch state (validators, last error, consecutive failures, next check). A timer asks it which subscriptions are due; feeds are checked at most a few at a time, a success diff --git a/src/podcasts/fetch.rs b/src/podcasts/fetch.rs index cddcf50..bf2078b 100644 --- a/src/podcasts/fetch.rs +++ b/src/podcasts/fetch.rs @@ -1,8 +1,10 @@ //! HTTP for podcast feeds and cover images. //! //! Redirects are followed by hand so the caller learns where the feed -//! really lives and whether every hop was permanent (301/308): only then -//! does the store record the new URL as a feed alias. Conditional requests +//! really lives: the address reached by the leading permanent (301/308) +//! hops is where the feed moved, and the store records it as the new fetch +//! URL (the old one becomes an alias). Temporary hops after it (a CDN or a +//! tracking prefix) do not move the feed further. Conditional requests //! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and //! the user agent are fixed here; parsing and storage happen elsewhere. @@ -57,12 +59,14 @@ pub enum FeedBody { #[derive(Debug)] pub struct FeedResponse { pub body: FeedBody, - /// URL that finally answered, canonicalized. + /// URL that finally answered, canonicalized; the base for relative + /// URLs in the document. pub final_url: FeedUrl, - /// At least one redirect was followed and every one of them was - /// permanent (301/308). Temporary redirects (tracking prefixes, CDNs) - /// never move a feed. - pub moved_permanently: bool, + /// Where the leading permanent redirects (301/308, one or more, before + /// any temporary one) lead: the feed's new address. `None` when the + /// first hop was temporary or there was none. Temporary redirects + /// (tracking prefixes, CDNs) never move a feed. + pub moved_permanently_to: Option, /// Redirect hops followed. pub redirects: usize, } @@ -116,7 +120,7 @@ impl Fetcher { let Followed { response, url: final_url, - moved_permanently, + moved_permanently_to, redirects, conditional, } = self @@ -128,6 +132,9 @@ impl Fetcher { redact(&final_url) ) })?; + // An address that is not a valid feed URL is not worth moving to. + let moved_permanently_to = + moved_permanently_to.and_then(|url| FeedUrl::parse(url.as_str()).ok()); let status = response.status(); if status == StatusCode::NOT_MODIFIED { if !conditional { @@ -136,7 +143,7 @@ impl Fetcher { return Ok(FeedResponse { body: FeedBody::NotModified, final_url, - moved_permanently, + moved_permanently_to, redirects, }); } @@ -153,7 +160,7 @@ impl Fetcher { last_modified, }, final_url, - moved_permanently, + moved_permanently_to, redirects, }) } @@ -187,7 +194,9 @@ impl Fetcher { ) -> Result { let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?; check_scheme(¤t)?; - let mut all_permanent = true; + // Still in the leading run of permanent hops. + let mut permanent_run = true; + let mut moved_permanently_to = None; let mut send_validators = true; let mut hops = 0; loop { @@ -216,7 +225,7 @@ impl Fetcher { return Ok(Followed { response, url: current, - moved_permanently: hops > 0 && all_permanent, + moved_permanently_to, redirects: hops, conditional, }); @@ -238,7 +247,10 @@ impl Fetcher { status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT ); - all_permanent &= permanent; + permanent_run &= permanent; + if permanent_run { + moved_permanently_to = Some(next.clone()); + } if permanent { send_validators = false; } @@ -251,8 +263,8 @@ struct Followed { response: reqwest::Response, /// URL that answered. url: Url, - /// At least one hop and all of them permanent. - moved_permanently: bool, + /// Target of the leading permanent hops. + moved_permanently_to: Option, redirects: usize, /// The final request carried validators. conditional: bool, diff --git a/src/podcasts/refresh.rs b/src/podcasts/refresh.rs index 22ca5f8..f4ddfd9 100644 --- a/src/podcasts/refresh.rs +++ b/src/podcasts/refresh.rs @@ -127,20 +127,22 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result< }; let final_url = response.final_url; let feed = parse(body, &final_url).await?; + // The feed's own address: where its permanent redirects lead. + let moved_permanently = response.moved_permanently_to.is_some(); + let landed = response.moved_permanently_to.unwrap_or(final_url); let store = Arc::clone(&services.store); let log = Arc::clone(&services.log); - let moved_permanently = response.moved_permanently; let image_url = feed.image_url.clone(); let report = blocking(move || { // Identity comes from the URL of the first subscription. Reuse a // podcast this device already knows under the typed or the final // URL; otherwise the typed URL defines the id and a permanent // redirect becomes an alias below. - let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? { + let identity_url = if knows(&store, &url)? || !knows(&store, &landed)? { url.clone() } else { - final_url.clone() + landed.clone() }; let meta = feed.to_podcast_meta(identity_url); let mut subscribed = None; @@ -159,7 +161,7 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result< &FeedRefresh { podcast_id: &podcast_id, feed: &feed, - fetched_from: &final_url, + fetched_from: &landed, moved_permanently, }, )?; @@ -252,7 +254,7 @@ async fn fetch_and_apply( tracing::debug!( podcast = %podcast_id, redirects = response.redirects, - moved_permanently = response.moved_permanently, + moved_permanently = response.moved_permanently_to.is_some(), not_modified = matches!(response.body, FeedBody::NotModified), "podcast feed checked" ); @@ -275,21 +277,25 @@ async fn fetch_and_apply( }; let final_url = response.final_url; let feed = parse(body, &final_url).await?; + let moved_permanently = response.moved_permanently_to.is_some(); + let landed = response.moved_permanently_to.unwrap_or(final_url); let image_url = feed.image_url.clone(); let store = Arc::clone(&services.store); let log = Arc::clone(&services.log); let id = podcast_id.clone(); let previous_url = url.clone(); - let moved_permanently = response.moved_permanently; let report = blocking(move || { let mut refreshed = None; log.record(&mut |stamps| { let outcome = store.apply_feed( stamps, + // `fetched_from` is the address the permanent redirects led + // to; the document itself may come from a temporary hop + // behind it (parsed against that URL above). &FeedRefresh { podcast_id: &id, feed: &feed, - fetched_from: &final_url, + fetched_from: &landed, moved_permanently, }, )?; diff --git a/src/podcasts/tests.rs b/src/podcasts/tests.rs index b8a7c1c..3758393 100644 --- a/src/podcasts/tests.rs +++ b/src/podcasts/tests.rs @@ -254,6 +254,70 @@ async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() { assert_eq!(again.podcast_id, report.podcast_id); } +/// A feed that moved (301) to a host that answers through a temporary +/// redirect (302, a CDN or tracker) moves to the permanent target only; the +/// following checks start there and are conditional again. +#[tokio::test] +async fn a_permanent_move_behind_a_temporary_redirect_still_moves_the_feed() { + let f = fixture(); + f.server.set( + "/old.xml", + Route::redirect(301, &f.server.url("/moved.xml")), + ); + f.server + .set("/moved.xml", Route::redirect(302, "/cdn/feed.xml")); + f.server.set( + "/cdn/feed.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml").etag("\"v1\""), + ); + let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap(); + let moved = FeedUrl::parse(&f.server.url("/moved.xml")).unwrap(); + let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap(); + let store = &f.services.store; + let summary = store.podcast(&report.podcast_id).unwrap().unwrap(); + assert_eq!( + summary.meta.feed_url, moved, + "the permanent target, not the CDN" + ); + assert!( + store + .feed_aliases(&report.podcast_id) + .unwrap() + .contains(&moved) + ); + // Relative enclosures resolve against the document's own address. + let episodes = store + .episodes(&EpisodeQuery::new(report.podcast_id.clone())) + .unwrap(); + assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3")); + + // The next check starts at the new address and keeps its validators + // through the temporary hop. + let checked = refresh(&f.services, &report.podcast_id, &moved, NOW + 1) + .await + .unwrap(); + assert!(checked.not_modified); + assert_eq!(f.server.requests("/old.xml").len(), 1); + let cdn = f.server.requests("/cdn/feed.xml"); + assert_eq!(cdn.len(), 2); + assert_eq!( + cdn[1].get("if-none-match").map(String::as_str), + Some("\"v1\"") + ); + + // A temporary hop first never moves the feed, whatever follows it. + f.server.set( + "/tracked.xml", + Route::redirect(302, &f.server.url("/old.xml")), + ); + let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap(); + let other = subscribe(&f.services, tracked.as_str(), NOW + 2) + .await + .unwrap(); + let summary = store.podcast(&other.podcast_id).unwrap().unwrap(); + assert_eq!(summary.meta.feed_url, tracked); +} + #[tokio::test] async fn temporary_redirect_keeps_the_feed_url() { let f = fixture();