Move a feed to its permanent address behind a temporary redirect

A 301 to a new host followed by a 302 (CDN, tracker) never moved the
feed, because only all-permanent chains counted; every check then went
through the old address without validators and re-applied the whole
feed. The target of the leading permanent hops is now recorded as the
new fetch URL (the old one becomes an alias); the document is still
parsed against the URL that served it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-19 21:50:44 +02:00
1 parent fe373e4169
commit e0da2ddf2b
4 files changed
+112 -26

No files matched your search

+8 -4
View File
@@ -320,10 +320,14 @@ registers (subscription, share, progress) and listening sessions in
The TUI owns what the contract leaves to clients:
- **Fetching.** Feeds are fetched with conditional requests (ETag,
Last-Modified), bounded size and time, and redirects followed by hand so the
store learns whether every hop was permanent; only then does the new URL
become a feed alias and the fetch URL. Validators are dropped after a
permanent redirect because they belong to the old address.
Last-Modified), bounded size and time, and redirects followed by hand. The
address that the leading permanent hops (301/308) lead to becomes a feed
alias and the fetch URL, even when a temporary hop (CDN, tracker) follows
it; a chain that starts with a temporary hop never moves the feed.
Validators are dropped after a permanent redirect because they belong to
the old address. The prototype database's validators are dropped once at
import, so the first check parses the feed (seasons and episode numbers
were not stored by the prototype).
- **Scheduling.** The store keeps each feed's local fetch state (validators,
last error, consecutive failures, next check). A timer asks it which
subscriptions are due; feeds are checked at most a few at a time, a success
+27 -15
View File
@@ -1,8 +1,10 @@
//! HTTP for podcast feeds and cover images.
//!
//! Redirects are followed by hand so the caller learns where the feed
//! really lives and whether every hop was permanent (301/308): only then
//! does the store record the new URL as a feed alias. Conditional requests
//! really lives: the address reached by the leading permanent (301/308)
//! hops is where the feed moved, and the store records it as the new fetch
//! URL (the old one becomes an alias). Temporary hops after it (a CDN or a
//! tracking prefix) do not move the feed further. Conditional requests
//! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and
//! the user agent are fixed here; parsing and storage happen elsewhere.
@@ -57,12 +59,14 @@ pub enum FeedBody {
#[derive(Debug)]
pub struct FeedResponse {
pub body: FeedBody,
/// URL that finally answered, canonicalized.
/// URL that finally answered, canonicalized; the base for relative
/// URLs in the document.
pub final_url: FeedUrl,
/// At least one redirect was followed and every one of them was
/// permanent (301/308). Temporary redirects (tracking prefixes, CDNs)
/// never move a feed.
pub moved_permanently: bool,
/// Where the leading permanent redirects (301/308, one or more, before
/// any temporary one) lead: the feed's new address. `None` when the
/// first hop was temporary or there was none. Temporary redirects
/// (tracking prefixes, CDNs) never move a feed.
pub moved_permanently_to: Option<FeedUrl>,
/// Redirect hops followed.
pub redirects: usize,
}
@@ -116,7 +120,7 @@ impl Fetcher {
let Followed {
response,
url: final_url,
moved_permanently,
moved_permanently_to,
redirects,
conditional,
} = self
@@ -128,6 +132,9 @@ impl Fetcher {
redact(&final_url)
)
})?;
// An address that is not a valid feed URL is not worth moving to.
let moved_permanently_to =
moved_permanently_to.and_then(|url| FeedUrl::parse(url.as_str()).ok());
let status = response.status();
if status == StatusCode::NOT_MODIFIED {
if !conditional {
@@ -136,7 +143,7 @@ impl Fetcher {
return Ok(FeedResponse {
body: FeedBody::NotModified,
final_url,
moved_permanently,
moved_permanently_to,
redirects,
});
}
@@ -153,7 +160,7 @@ impl Fetcher {
last_modified,
},
final_url,
moved_permanently,
moved_permanently_to,
redirects,
})
}
@@ -187,7 +194,9 @@ impl Fetcher {
) -> Result<Followed> {
let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?;
check_scheme(&current)?;
let mut all_permanent = true;
// Still in the leading run of permanent hops.
let mut permanent_run = true;
let mut moved_permanently_to = None;
let mut send_validators = true;
let mut hops = 0;
loop {
@@ -216,7 +225,7 @@ impl Fetcher {
return Ok(Followed {
response,
url: current,
moved_permanently: hops > 0 && all_permanent,
moved_permanently_to,
redirects: hops,
conditional,
});
@@ -238,7 +247,10 @@ impl Fetcher {
status,
StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT
);
all_permanent &= permanent;
permanent_run &= permanent;
if permanent_run {
moved_permanently_to = Some(next.clone());
}
if permanent {
send_validators = false;
}
@@ -251,8 +263,8 @@ struct Followed {
response: reqwest::Response,
/// URL that answered.
url: Url,
/// At least one hop and all of them permanent.
moved_permanently: bool,
/// Target of the leading permanent hops.
moved_permanently_to: Option<Url>,
redirects: usize,
/// The final request carried validators.
conditional: bool,
+13 -7
View File
@@ -127,20 +127,22 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<
};
let final_url = response.final_url;
let feed = parse(body, &final_url).await?;
// The feed's own address: where its permanent redirects lead.
let moved_permanently = response.moved_permanently_to.is_some();
let landed = response.moved_permanently_to.unwrap_or(final_url);
let store = Arc::clone(&services.store);
let log = Arc::clone(&services.log);
let moved_permanently = response.moved_permanently;
let image_url = feed.image_url.clone();
let report = blocking(move || {
// Identity comes from the URL of the first subscription. Reuse a
// podcast this device already knows under the typed or the final
// URL; otherwise the typed URL defines the id and a permanent
// redirect becomes an alias below.
let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? {
let identity_url = if knows(&store, &url)? || !knows(&store, &landed)? {
url.clone()
} else {
final_url.clone()
landed.clone()
};
let meta = feed.to_podcast_meta(identity_url);
let mut subscribed = None;
@@ -159,7 +161,7 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<
&FeedRefresh {
podcast_id: &podcast_id,
feed: &feed,
fetched_from: &final_url,
fetched_from: &landed,
moved_permanently,
},
)?;
@@ -252,7 +254,7 @@ async fn fetch_and_apply(
tracing::debug!(
podcast = %podcast_id,
redirects = response.redirects,
moved_permanently = response.moved_permanently,
moved_permanently = response.moved_permanently_to.is_some(),
not_modified = matches!(response.body, FeedBody::NotModified),
"podcast feed checked"
);
@@ -275,21 +277,25 @@ async fn fetch_and_apply(
};
let final_url = response.final_url;
let feed = parse(body, &final_url).await?;
let moved_permanently = response.moved_permanently_to.is_some();
let landed = response.moved_permanently_to.unwrap_or(final_url);
let image_url = feed.image_url.clone();
let store = Arc::clone(&services.store);
let log = Arc::clone(&services.log);
let id = podcast_id.clone();
let previous_url = url.clone();
let moved_permanently = response.moved_permanently;
let report = blocking(move || {
let mut refreshed = None;
log.record(&mut |stamps| {
let outcome = store.apply_feed(
stamps,
// `fetched_from` is the address the permanent redirects led
// to; the document itself may come from a temporary hop
// behind it (parsed against that URL above).
&FeedRefresh {
podcast_id: &id,
feed: &feed,
fetched_from: &final_url,
fetched_from: &landed,
moved_permanently,
},
)?;
+64
View File
@@ -254,6 +254,70 @@ async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() {
assert_eq!(again.podcast_id, report.podcast_id);
}
/// A feed that moved (301) to a host that answers through a temporary
/// redirect (302, a CDN or tracker) moves to the permanent target only; the
/// following checks start there and are conditional again.
#[tokio::test]
async fn a_permanent_move_behind_a_temporary_redirect_still_moves_the_feed() {
let f = fixture();
f.server.set(
"/old.xml",
Route::redirect(301, &f.server.url("/moved.xml")),
);
f.server
.set("/moved.xml", Route::redirect(302, "/cdn/feed.xml"));
f.server.set(
"/cdn/feed.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml").etag("\"v1\""),
);
let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap();
let moved = FeedUrl::parse(&f.server.url("/moved.xml")).unwrap();
let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap();
let store = &f.services.store;
let summary = store.podcast(&report.podcast_id).unwrap().unwrap();
assert_eq!(
summary.meta.feed_url, moved,
"the permanent target, not the CDN"
);
assert!(
store
.feed_aliases(&report.podcast_id)
.unwrap()
.contains(&moved)
);
// Relative enclosures resolve against the document's own address.
let episodes = store
.episodes(&EpisodeQuery::new(report.podcast_id.clone()))
.unwrap();
assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3"));
// The next check starts at the new address and keeps its validators
// through the temporary hop.
let checked = refresh(&f.services, &report.podcast_id, &moved, NOW + 1)
.await
.unwrap();
assert!(checked.not_modified);
assert_eq!(f.server.requests("/old.xml").len(), 1);
let cdn = f.server.requests("/cdn/feed.xml");
assert_eq!(cdn.len(), 2);
assert_eq!(
cdn[1].get("if-none-match").map(String::as_str),
Some("\"v1\"")
);
// A temporary hop first never moves the feed, whatever follows it.
f.server.set(
"/tracked.xml",
Route::redirect(302, &f.server.url("/old.xml")),
);
let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap();
let other = subscribe(&f.services, tracked.as_str(), NOW + 2)
.await
.unwrap();
let summary = store.podcast(&other.podcast_id).unwrap().unwrap();
assert_eq!(summary.meta.feed_url, tracked);
}
#[tokio::test]
async fn temporary_redirect_keeps_the_feed_url() {
let f = fixture();