forked from Furumi/furumi-tui
Move a feed to its permanent address behind a temporary redirect
A 301 to a new host followed by a 302 (CDN, tracker) never moved the feed, because only all-permanent chains counted; every check then went through the old address without validators and re-applied the whole feed. The target of the leading permanent hops is now recorded as the new fetch URL (the old one becomes an alias); the document is still parsed against the URL that served it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fe373e4169
commit
e0da2ddf2b
4 files changed
+112
-26
No files matched your search
+27
-15
@@ -1,8 +1,10 @@
|
||||
//! HTTP for podcast feeds and cover images.
|
||||
//!
|
||||
//! Redirects are followed by hand so the caller learns where the feed
|
||||
//! really lives and whether every hop was permanent (301/308): only then
|
||||
//! does the store record the new URL as a feed alias. Conditional requests
|
||||
//! really lives: the address reached by the leading permanent (301/308)
|
||||
//! hops is where the feed moved, and the store records it as the new fetch
|
||||
//! URL (the old one becomes an alias). Temporary hops after it (a CDN or a
|
||||
//! tracking prefix) do not move the feed further. Conditional requests
|
||||
//! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and
|
||||
//! the user agent are fixed here; parsing and storage happen elsewhere.
|
||||
|
||||
@@ -57,12 +59,14 @@ pub enum FeedBody {
|
||||
#[derive(Debug)]
|
||||
pub struct FeedResponse {
|
||||
pub body: FeedBody,
|
||||
/// URL that finally answered, canonicalized.
|
||||
/// URL that finally answered, canonicalized; the base for relative
|
||||
/// URLs in the document.
|
||||
pub final_url: FeedUrl,
|
||||
/// At least one redirect was followed and every one of them was
|
||||
/// permanent (301/308). Temporary redirects (tracking prefixes, CDNs)
|
||||
/// never move a feed.
|
||||
pub moved_permanently: bool,
|
||||
/// Where the leading permanent redirects (301/308, one or more, before
|
||||
/// any temporary one) lead: the feed's new address. `None` when the
|
||||
/// first hop was temporary or there was none. Temporary redirects
|
||||
/// (tracking prefixes, CDNs) never move a feed.
|
||||
pub moved_permanently_to: Option<FeedUrl>,
|
||||
/// Redirect hops followed.
|
||||
pub redirects: usize,
|
||||
}
|
||||
@@ -116,7 +120,7 @@ impl Fetcher {
|
||||
let Followed {
|
||||
response,
|
||||
url: final_url,
|
||||
moved_permanently,
|
||||
moved_permanently_to,
|
||||
redirects,
|
||||
conditional,
|
||||
} = self
|
||||
@@ -128,6 +132,9 @@ impl Fetcher {
|
||||
redact(&final_url)
|
||||
)
|
||||
})?;
|
||||
// An address that is not a valid feed URL is not worth moving to.
|
||||
let moved_permanently_to =
|
||||
moved_permanently_to.and_then(|url| FeedUrl::parse(url.as_str()).ok());
|
||||
let status = response.status();
|
||||
if status == StatusCode::NOT_MODIFIED {
|
||||
if !conditional {
|
||||
@@ -136,7 +143,7 @@ impl Fetcher {
|
||||
return Ok(FeedResponse {
|
||||
body: FeedBody::NotModified,
|
||||
final_url,
|
||||
moved_permanently,
|
||||
moved_permanently_to,
|
||||
redirects,
|
||||
});
|
||||
}
|
||||
@@ -153,7 +160,7 @@ impl Fetcher {
|
||||
last_modified,
|
||||
},
|
||||
final_url,
|
||||
moved_permanently,
|
||||
moved_permanently_to,
|
||||
redirects,
|
||||
})
|
||||
}
|
||||
@@ -187,7 +194,9 @@ impl Fetcher {
|
||||
) -> Result<Followed> {
|
||||
let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?;
|
||||
check_scheme(¤t)?;
|
||||
let mut all_permanent = true;
|
||||
// Still in the leading run of permanent hops.
|
||||
let mut permanent_run = true;
|
||||
let mut moved_permanently_to = None;
|
||||
let mut send_validators = true;
|
||||
let mut hops = 0;
|
||||
loop {
|
||||
@@ -216,7 +225,7 @@ impl Fetcher {
|
||||
return Ok(Followed {
|
||||
response,
|
||||
url: current,
|
||||
moved_permanently: hops > 0 && all_permanent,
|
||||
moved_permanently_to,
|
||||
redirects: hops,
|
||||
conditional,
|
||||
});
|
||||
@@ -238,7 +247,10 @@ impl Fetcher {
|
||||
status,
|
||||
StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT
|
||||
);
|
||||
all_permanent &= permanent;
|
||||
permanent_run &= permanent;
|
||||
if permanent_run {
|
||||
moved_permanently_to = Some(next.clone());
|
||||
}
|
||||
if permanent {
|
||||
send_validators = false;
|
||||
}
|
||||
@@ -251,8 +263,8 @@ struct Followed {
|
||||
response: reqwest::Response,
|
||||
/// URL that answered.
|
||||
url: Url,
|
||||
/// At least one hop and all of them permanent.
|
||||
moved_permanently: bool,
|
||||
/// Target of the leading permanent hops.
|
||||
moved_permanently_to: Option<Url>,
|
||||
redirects: usize,
|
||||
/// The final request carried validators.
|
||||
conditional: bool,
|
||||
|
||||
+13
-7
@@ -127,20 +127,22 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<
|
||||
};
|
||||
let final_url = response.final_url;
|
||||
let feed = parse(body, &final_url).await?;
|
||||
// The feed's own address: where its permanent redirects lead.
|
||||
let moved_permanently = response.moved_permanently_to.is_some();
|
||||
let landed = response.moved_permanently_to.unwrap_or(final_url);
|
||||
|
||||
let store = Arc::clone(&services.store);
|
||||
let log = Arc::clone(&services.log);
|
||||
let moved_permanently = response.moved_permanently;
|
||||
let image_url = feed.image_url.clone();
|
||||
let report = blocking(move || {
|
||||
// Identity comes from the URL of the first subscription. Reuse a
|
||||
// podcast this device already knows under the typed or the final
|
||||
// URL; otherwise the typed URL defines the id and a permanent
|
||||
// redirect becomes an alias below.
|
||||
let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? {
|
||||
let identity_url = if knows(&store, &url)? || !knows(&store, &landed)? {
|
||||
url.clone()
|
||||
} else {
|
||||
final_url.clone()
|
||||
landed.clone()
|
||||
};
|
||||
let meta = feed.to_podcast_meta(identity_url);
|
||||
let mut subscribed = None;
|
||||
@@ -159,7 +161,7 @@ pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<
|
||||
&FeedRefresh {
|
||||
podcast_id: &podcast_id,
|
||||
feed: &feed,
|
||||
fetched_from: &final_url,
|
||||
fetched_from: &landed,
|
||||
moved_permanently,
|
||||
},
|
||||
)?;
|
||||
@@ -252,7 +254,7 @@ async fn fetch_and_apply(
|
||||
tracing::debug!(
|
||||
podcast = %podcast_id,
|
||||
redirects = response.redirects,
|
||||
moved_permanently = response.moved_permanently,
|
||||
moved_permanently = response.moved_permanently_to.is_some(),
|
||||
not_modified = matches!(response.body, FeedBody::NotModified),
|
||||
"podcast feed checked"
|
||||
);
|
||||
@@ -275,21 +277,25 @@ async fn fetch_and_apply(
|
||||
};
|
||||
let final_url = response.final_url;
|
||||
let feed = parse(body, &final_url).await?;
|
||||
let moved_permanently = response.moved_permanently_to.is_some();
|
||||
let landed = response.moved_permanently_to.unwrap_or(final_url);
|
||||
let image_url = feed.image_url.clone();
|
||||
let store = Arc::clone(&services.store);
|
||||
let log = Arc::clone(&services.log);
|
||||
let id = podcast_id.clone();
|
||||
let previous_url = url.clone();
|
||||
let moved_permanently = response.moved_permanently;
|
||||
let report = blocking(move || {
|
||||
let mut refreshed = None;
|
||||
log.record(&mut |stamps| {
|
||||
let outcome = store.apply_feed(
|
||||
stamps,
|
||||
// `fetched_from` is the address the permanent redirects led
|
||||
// to; the document itself may come from a temporary hop
|
||||
// behind it (parsed against that URL above).
|
||||
&FeedRefresh {
|
||||
podcast_id: &id,
|
||||
feed: &feed,
|
||||
fetched_from: &final_url,
|
||||
fetched_from: &landed,
|
||||
moved_permanently,
|
||||
},
|
||||
)?;
|
||||
|
||||
@@ -254,6 +254,70 @@ async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() {
|
||||
assert_eq!(again.podcast_id, report.podcast_id);
|
||||
}
|
||||
|
||||
/// A feed that moved (301) to a host that answers through a temporary
|
||||
/// redirect (302, a CDN or tracker) moves to the permanent target only; the
|
||||
/// following checks start there and are conditional again.
|
||||
#[tokio::test]
|
||||
async fn a_permanent_move_behind_a_temporary_redirect_still_moves_the_feed() {
|
||||
let f = fixture();
|
||||
f.server.set(
|
||||
"/old.xml",
|
||||
Route::redirect(301, &f.server.url("/moved.xml")),
|
||||
);
|
||||
f.server
|
||||
.set("/moved.xml", Route::redirect(302, "/cdn/feed.xml"));
|
||||
f.server.set(
|
||||
"/cdn/feed.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml").etag("\"v1\""),
|
||||
);
|
||||
let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap();
|
||||
let moved = FeedUrl::parse(&f.server.url("/moved.xml")).unwrap();
|
||||
let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap();
|
||||
let store = &f.services.store;
|
||||
let summary = store.podcast(&report.podcast_id).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
summary.meta.feed_url, moved,
|
||||
"the permanent target, not the CDN"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.feed_aliases(&report.podcast_id)
|
||||
.unwrap()
|
||||
.contains(&moved)
|
||||
);
|
||||
// Relative enclosures resolve against the document's own address.
|
||||
let episodes = store
|
||||
.episodes(&EpisodeQuery::new(report.podcast_id.clone()))
|
||||
.unwrap();
|
||||
assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3"));
|
||||
|
||||
// The next check starts at the new address and keeps its validators
|
||||
// through the temporary hop.
|
||||
let checked = refresh(&f.services, &report.podcast_id, &moved, NOW + 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(checked.not_modified);
|
||||
assert_eq!(f.server.requests("/old.xml").len(), 1);
|
||||
let cdn = f.server.requests("/cdn/feed.xml");
|
||||
assert_eq!(cdn.len(), 2);
|
||||
assert_eq!(
|
||||
cdn[1].get("if-none-match").map(String::as_str),
|
||||
Some("\"v1\"")
|
||||
);
|
||||
|
||||
// A temporary hop first never moves the feed, whatever follows it.
|
||||
f.server.set(
|
||||
"/tracked.xml",
|
||||
Route::redirect(302, &f.server.url("/old.xml")),
|
||||
);
|
||||
let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap();
|
||||
let other = subscribe(&f.services, tracked.as_str(), NOW + 2)
|
||||
.await
|
||||
.unwrap();
|
||||
let summary = store.podcast(&other.podcast_id).unwrap().unwrap();
|
||||
assert_eq!(summary.meta.feed_url, tracked);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn temporary_redirect_keeps_the_feed_url() {
|
||||
let f = fixture();
|
||||
|
||||
Reference in new issue
Block a user