Move a feed to its permanent address behind a temporary redirect

A 301 to a new host followed by a 302 (CDN, tracker) never moved the
feed, because only all-permanent chains counted; every check then went
through the old address without validators and re-applied the whole
feed. The target of the leading permanent hops is now recorded as the
new fetch URL (the old one becomes an alias); the document is still
parsed against the URL that served it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-19 21:50:44 +02:00
1 parent fe373e4169
commit e0da2ddf2b
4 files changed
+112 -26

No files matched your search

+27 -15
View File
@@ -1,8 +1,10 @@
//! HTTP for podcast feeds and cover images.
//!
//! Redirects are followed by hand so the caller learns where the feed
//! really lives and whether every hop was permanent (301/308): only then
//! does the store record the new URL as a feed alias. Conditional requests
//! really lives: the address reached by the leading permanent (301/308)
//! hops is where the feed moved, and the store records it as the new fetch
//! URL (the old one becomes an alias). Temporary hops after it (a CDN or a
//! tracking prefix) do not move the feed further. Conditional requests
//! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and
//! the user agent are fixed here; parsing and storage happen elsewhere.
@@ -57,12 +59,14 @@ pub enum FeedBody {
#[derive(Debug)]
pub struct FeedResponse {
pub body: FeedBody,
/// URL that finally answered, canonicalized.
/// URL that finally answered, canonicalized; the base for relative
/// URLs in the document.
pub final_url: FeedUrl,
/// At least one redirect was followed and every one of them was
/// permanent (301/308). Temporary redirects (tracking prefixes, CDNs)
/// never move a feed.
pub moved_permanently: bool,
/// Where the leading permanent redirects (301/308, one or more, before
/// any temporary one) lead: the feed's new address. `None` when the
/// first hop was temporary or there was none. Temporary redirects
/// (tracking prefixes, CDNs) never move a feed.
pub moved_permanently_to: Option<FeedUrl>,
/// Redirect hops followed.
pub redirects: usize,
}
@@ -116,7 +120,7 @@ impl Fetcher {
let Followed {
response,
url: final_url,
moved_permanently,
moved_permanently_to,
redirects,
conditional,
} = self
@@ -128,6 +132,9 @@ impl Fetcher {
redact(&final_url)
)
})?;
// An address that is not a valid feed URL is not worth moving to.
let moved_permanently_to =
moved_permanently_to.and_then(|url| FeedUrl::parse(url.as_str()).ok());
let status = response.status();
if status == StatusCode::NOT_MODIFIED {
if !conditional {
@@ -136,7 +143,7 @@ impl Fetcher {
return Ok(FeedResponse {
body: FeedBody::NotModified,
final_url,
moved_permanently,
moved_permanently_to,
redirects,
});
}
@@ -153,7 +160,7 @@ impl Fetcher {
last_modified,
},
final_url,
moved_permanently,
moved_permanently_to,
redirects,
})
}
@@ -187,7 +194,9 @@ impl Fetcher {
) -> Result<Followed> {
let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?;
check_scheme(&current)?;
let mut all_permanent = true;
// Still in the leading run of permanent hops.
let mut permanent_run = true;
let mut moved_permanently_to = None;
let mut send_validators = true;
let mut hops = 0;
loop {
@@ -216,7 +225,7 @@ impl Fetcher {
return Ok(Followed {
response,
url: current,
moved_permanently: hops > 0 && all_permanent,
moved_permanently_to,
redirects: hops,
conditional,
});
@@ -238,7 +247,10 @@ impl Fetcher {
status,
StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT
);
all_permanent &= permanent;
permanent_run &= permanent;
if permanent_run {
moved_permanently_to = Some(next.clone());
}
if permanent {
send_validators = false;
}
@@ -251,8 +263,8 @@ struct Followed {
response: reqwest::Response,
/// URL that answered.
url: Url,
/// At least one hop and all of them permanent.
moved_permanently: bool,
/// Target of the leading permanent hops.
moved_permanently_to: Option<Url>,
redirects: usize,
/// The final request carried validators.
conditional: bool,
+13 -7
View File
@@ -127,20 +127,22 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<
};
let final_url = response.final_url;
let feed = parse(body, &final_url).await?;
// The feed's own address: where its permanent redirects lead.
let moved_permanently = response.moved_permanently_to.is_some();
let landed = response.moved_permanently_to.unwrap_or(final_url);
let store = Arc::clone(&services.store);
let log = Arc::clone(&services.log);
let moved_permanently = response.moved_permanently;
let image_url = feed.image_url.clone();
let report = blocking(move || {
// Identity comes from the URL of the first subscription. Reuse a
// podcast this device already knows under the typed or the final
// URL; otherwise the typed URL defines the id and a permanent
// redirect becomes an alias below.
let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? {
let identity_url = if knows(&store, &url)? || !knows(&store, &landed)? {
url.clone()
} else {
final_url.clone()
landed.clone()
};
let meta = feed.to_podcast_meta(identity_url);
let mut subscribed = None;
@@ -159,7 +161,7 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<
&FeedRefresh {
podcast_id: &podcast_id,
feed: &feed,
fetched_from: &final_url,
fetched_from: &landed,
moved_permanently,
},
)?;
@@ -252,7 +254,7 @@ async fn fetch_and_apply(
tracing::debug!(
podcast = %podcast_id,
redirects = response.redirects,
moved_permanently = response.moved_permanently,
moved_permanently = response.moved_permanently_to.is_some(),
not_modified = matches!(response.body, FeedBody::NotModified),
"podcast feed checked"
);
@@ -275,21 +277,25 @@ async fn fetch_and_apply(
};
let final_url = response.final_url;
let feed = parse(body, &final_url).await?;
let moved_permanently = response.moved_permanently_to.is_some();
let landed = response.moved_permanently_to.unwrap_or(final_url);
let image_url = feed.image_url.clone();
let store = Arc::clone(&services.store);
let log = Arc::clone(&services.log);
let id = podcast_id.clone();
let previous_url = url.clone();
let moved_permanently = response.moved_permanently;
let report = blocking(move || {
let mut refreshed = None;
log.record(&mut |stamps| {
let outcome = store.apply_feed(
stamps,
// `fetched_from` is the address the permanent redirects led
// to; the document itself may come from a temporary hop
// behind it (parsed against that URL above).
&FeedRefresh {
podcast_id: &id,
feed: &feed,
fetched_from: &final_url,
fetched_from: &landed,
moved_permanently,
},
)?;
+64
View File
@@ -254,6 +254,70 @@ async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() {
assert_eq!(again.podcast_id, report.podcast_id);
}
/// A feed that moved (301) to a host that answers through a temporary
/// redirect (302, a CDN or tracker) moves to the permanent target only; the
/// following checks start there and are conditional again.
#[tokio::test]
async fn a_permanent_move_behind_a_temporary_redirect_still_moves_the_feed() {
let f = fixture();
f.server.set(
"/old.xml",
Route::redirect(301, &f.server.url("/moved.xml")),
);
f.server
.set("/moved.xml", Route::redirect(302, "/cdn/feed.xml"));
f.server.set(
"/cdn/feed.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml").etag("\"v1\""),
);
let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap();
let moved = FeedUrl::parse(&f.server.url("/moved.xml")).unwrap();
let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap();
let store = &f.services.store;
let summary = store.podcast(&report.podcast_id).unwrap().unwrap();
assert_eq!(
summary.meta.feed_url, moved,
"the permanent target, not the CDN"
);
assert!(
store
.feed_aliases(&report.podcast_id)
.unwrap()
.contains(&moved)
);
// Relative enclosures resolve against the document's own address.
let episodes = store
.episodes(&EpisodeQuery::new(report.podcast_id.clone()))
.unwrap();
assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3"));
// The next check starts at the new address and keeps its validators
// through the temporary hop.
let checked = refresh(&f.services, &report.podcast_id, &moved, NOW + 1)
.await
.unwrap();
assert!(checked.not_modified);
assert_eq!(f.server.requests("/old.xml").len(), 1);
let cdn = f.server.requests("/cdn/feed.xml");
assert_eq!(cdn.len(), 2);
assert_eq!(
cdn[1].get("if-none-match").map(String::as_str),
Some("\"v1\"")
);
// A temporary hop first never moves the feed, whatever follows it.
f.server.set(
"/tracked.xml",
Route::redirect(302, &f.server.url("/old.xml")),
);
let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap();
let other = subscribe(&f.services, tracked.as_str(), NOW + 2)
.await
.unwrap();
let summary = store.podcast(&other.podcast_id).unwrap().unwrap();
assert_eq!(summary.meta.feed_url, tracked);
}
#[tokio::test]
async fn temporary_redirect_keeps_the_feed_url() {
let f = fixture();