Bound a whole feed fetch and test the fetch limits

A slow server could hold one of the four refresh slots for 60 s per
redirect hop. A feed or cover fetch now has an overall deadline (120 s).
Tests cover feeds over the size cap with and without Content-Length
(error recorded with back-off, catalog kept), the redirect cap, a
redirect loop, a non-http redirect, a hanging server, and covers (not an
image, over the cap, written atomically). The test server can stall.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-19 21:53:44 +02:00
1 parent 268ea915da
commit 34c096a36b
4 files changed
+228 -9

No files matched your search

+4 -1
View File
@@ -320,7 +320,10 @@ registers (subscription, share, progress) and listening sessions in
The TUI owns what the contract leaves to clients:
- **Fetching.** Feeds are fetched with conditional requests (ETag,
Last-Modified), bounded size and time, and redirects followed by hand. The
Last-Modified), bounded size (32 MiB per feed, 10 MiB per cover) and time
(15 s to connect, 60 s per request, 120 s for a whole fetch with its
redirects), and at most 10 redirects (http and https only) followed by
hand. The
address that the leading permanent hops (301/308) lead to becomes a feed
alias and the fetch URL, even when a temporary hop (CDN, tracker) follows
it; a chain that starts with a temporary hop never moves the feed.
+38 -8
View File
@@ -29,6 +29,12 @@ pub const MAX_REDIRECTS: usize = 10;
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
/// Whole request, including the body, per hop.
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
/// A whole feed or cover fetch, all redirect hops and the body included, so
/// a slow server holds one of the few refresh slots for at most this long.
#[cfg(not(test))]
pub const FETCH_DEADLINE: Duration = Duration::from_secs(120);
#[cfg(test)]
pub const FETCH_DEADLINE: Duration = Duration::from_secs(3);
/// Sent with every podcast request.
pub const USER_AGENT: &str = concat!("furumi/", env!("CARGO_PKG_VERSION"), " (podcast reader)");
@@ -116,6 +122,14 @@ impl Fetcher {
&self,
url: &FeedUrl,
validators: Validators<'_>,
) -> Result<FeedResponse> {
within_deadline("feed", self.fetch_feed_unbounded(url, validators)).await
}
async fn fetch_feed_unbounded(
&self,
url: &FeedUrl,
validators: Validators<'_>,
) -> Result<FeedResponse> {
let Followed {
response,
@@ -167,6 +181,17 @@ impl Fetcher {
/// Downloads a cover image into the covers directory; returns its path.
pub async fn fetch_cover(&self, podcast_id: &PodcastId, image_url: &str) -> Result<PathBuf> {
let body = within_deadline("cover image", self.download_cover(image_url)).await?;
let format = image::guess_format(&body).context("the cover is not a supported image")?;
let extension = format.extensions_str().first().copied().unwrap_or("img");
let name = cover_file_name(podcast_id, image_url, extension);
let dir = self.covers_dir.clone();
tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body))
.await
.map_err(|err| anyhow!("cover writer task failed: {err}"))?
}
async fn download_cover(&self, image_url: &str) -> Result<Vec<u8>> {
let response = self
.get_following(image_url, None, Validators::default())
.await?
@@ -175,14 +200,7 @@ impl Fetcher {
if !status.is_success() {
return Err(status_error(status, response.headers()).into());
}
let body = read_capped(response, MAX_COVER_BYTES, "cover image").await?;
let format = image::guess_format(&body).context("the cover is not a supported image")?;
let extension = format.extensions_str().first().copied().unwrap_or("img");
let name = cover_file_name(podcast_id, image_url, extension);
let dir = self.covers_dir.clone();
tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body))
.await
.map_err(|err| anyhow!("cover writer task failed: {err}"))?
read_capped(response, MAX_COVER_BYTES, "cover image").await
}
/// GET following up to [`MAX_REDIRECTS`] http(s) redirects by hand.
@@ -259,6 +277,18 @@ impl Fetcher {
}
}
/// Fails `fetch` once [`FETCH_DEADLINE`] has passed.
async fn within_deadline<T>(what: &str, fetch: impl Future<Output = Result<T>>) -> Result<T> {
tokio::time::timeout(FETCH_DEADLINE, fetch)
.await
.unwrap_or_else(|_| {
bail!(
"the {what} took longer than {} seconds",
FETCH_DEADLINE.as_secs()
)
})
}
struct Followed {
response: reqwest::Response,
/// URL that answered.
+12
View File
@@ -28,6 +28,8 @@ pub(super) struct Route {
pub cut_after: Option<usize>,
/// ...for this many requests; later requests are served in full.
pub fail_times: usize,
/// Wait this long before answering (a hanging server).
pub stall: Option<Duration>,
}
impl Route {
@@ -44,9 +46,16 @@ impl Route {
send_length: true,
cut_after: None,
fail_times: 0,
stall: None,
}
}
/// Answer only after `delay`.
pub fn stall(mut self, delay: Duration) -> Route {
self.stall = Some(delay);
self
}
/// A server that ignores `Range` and always answers 200 with the file.
pub fn no_ranges(mut self) -> Route {
self.ranges = false;
@@ -206,6 +215,9 @@ fn handle(stream: TcpStream, state: &Mutex<ServerState>) {
(route, cut)
};
let mut out = stream;
if let Some(delay) = route.as_ref().and_then(|route| route.stall) {
std::thread::sleep(delay);
}
let Some(route) = route else {
let _ = out
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
+174
View File
@@ -12,6 +12,7 @@ use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::{FeedUrl, PodcastId};
use super::OpLog;
use super::fetch::{FETCH_DEADLINE, MAX_COVER_BYTES, MAX_FEED_BYTES, MAX_REDIRECTS, Validators};
use super::refresh::{
FAILURE_BACKOFF_BASE, FAILURE_BACKOFF_MAX, REFRESH_INTERVAL, Services,
next_check_after_failure, next_check_after_success, refresh, subscribe,
@@ -741,3 +742,176 @@ async fn imported_podcasts_get_their_seasons_on_the_first_check() {
Some("\"e1\"")
);
}
fn feed_url(f: &Fixture, path: &str) -> FeedUrl {
FeedUrl::parse(&f.server.url(path)).unwrap()
}
#[tokio::test]
async fn feeds_over_the_size_limit_are_refused_with_or_without_a_length() {
let f = fixture();
f.server.set(
"/feed.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
);
let url = feed_url(&f, "/feed.xml");
let id = subscribe(&f.services, url.as_str(), NOW)
.await
.unwrap()
.podcast_id;
let mut huge = rss(&[("a", "Pilot"), ("b", "Second")]);
huge.resize(MAX_FEED_BYTES + 1, b' ');
let store = &f.services.store;
for (attempt, route) in [
Route::new(huge.clone(), "application/rss+xml"),
Route::new(huge, "application/rss+xml").no_length(),
]
.into_iter()
.enumerate()
{
let failures = attempt as u32 + 1;
f.server.set("/feed.xml", route);
let now = NOW + i64::from(failures);
let err = refresh(&f.services, &id, &url, now).await.unwrap_err();
assert!(format!("{err:#}").contains("too large"), "{err:#}");
let state = store.fetch_state(&id).unwrap();
assert_eq!(state.consecutive_failures, failures);
assert!(state.last_error.unwrap().contains("too large"));
assert_eq!(
state.next_check_at_ms,
Some(next_check_after_failure(now, failures, None))
);
assert_eq!(episode_count(store, &id), 1, "the catalog is kept");
}
}
#[tokio::test]
async fn redirect_chains_are_bounded_and_lead_only_to_http() {
let f = fixture();
let fetch = |path: &'static str| {
let url = feed_url(&f, path);
let fetcher = std::sync::Arc::clone(&f.services.fetcher);
async move { fetcher.fetch_feed(&url, Validators::default()).await }
};
for n in 0..=MAX_REDIRECTS {
f.server.set(
&format!("/r{n}.xml"),
Route::redirect(302, &format!("/r{}.xml", n + 1)),
);
}
f.server.set(
&format!("/r{}.xml", MAX_REDIRECTS + 1),
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
);
// MAX_REDIRECTS hops are followed, one more is refused.
let followed = fetch("/r1.xml").await.unwrap();
assert_eq!(followed.redirects, MAX_REDIRECTS);
let err = fetch("/r0.xml").await.unwrap_err();
assert!(format!("{err:#}").contains("too many redirects"), "{err:#}");
// A loop ends the same way.
f.server.set("/a.xml", Route::redirect(301, "/b.xml"));
f.server.set("/b.xml", Route::redirect(301, "/a.xml"));
let err = fetch("/a.xml").await.unwrap_err();
assert!(format!("{err:#}").contains("too many redirects"), "{err:#}");
assert_eq!(f.server.requests("/a.xml").len(), MAX_REDIRECTS / 2 + 1);
// Only http(s) is followed.
f.server.set(
"/ftp.xml",
Route::redirect(302, "ftp://example.test/feed.xml"),
);
let err = fetch("/ftp.xml").await.unwrap_err();
assert!(
format!("{err:#}").contains("unsupported URL scheme"),
"{err:#}"
);
// Subscribing through any of them stores nothing.
for path in ["/r0.xml", "/a.xml", "/ftp.xml"] {
assert!(
subscribe(&f.services, &f.server.url(path), NOW)
.await
.is_err()
);
}
assert!(
f.services
.store
.list_podcasts(PodcastListFilter::All)
.unwrap()
.is_empty()
);
}
#[tokio::test]
async fn a_hanging_server_fails_the_check_at_the_deadline() {
let f = fixture();
f.server.set(
"/feed.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
);
let url = feed_url(&f, "/feed.xml");
let id = subscribe(&f.services, url.as_str(), NOW)
.await
.unwrap()
.podcast_id;
f.server.set(
"/feed.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml")
.stall(FETCH_DEADLINE + Duration::from_secs(5)),
);
let started = std::time::Instant::now();
let err = refresh(&f.services, &id, &url, NOW + 1).await.unwrap_err();
assert!(format!("{err:#}").contains("took longer than"), "{err:#}");
assert!(started.elapsed() < FETCH_DEADLINE + Duration::from_secs(2));
let state = f.services.store.fetch_state(&id).unwrap();
assert_eq!(state.consecutive_failures, 1);
assert_eq!(episode_count(&f.services.store, &id), 1);
}
#[tokio::test]
async fn covers_must_be_images_within_the_limit_and_are_written_whole() {
let f = fixture();
let png = {
let mut out = std::io::Cursor::new(Vec::new());
image::RgbImage::new(2, 2)
.write_to(&mut out, image::ImageFormat::Png)
.unwrap();
out.into_inner()
};
f.server
.set("/cover.png", Route::new(png.clone(), "image/png"));
f.server.set(
"/page.png",
Route::new("<html>not an image</html>", "image/png"),
);
f.server.set(
"/huge.png",
Route::new(vec![0u8; MAX_COVER_BYTES + 1], "image/png"),
);
let id = PodcastId::from_feed_url(&feed_url(&f, "/feed.xml"));
let fetcher = &f.services.fetcher;
let path = fetcher
.fetch_cover(&id, &f.server.url("/cover.png"))
.await
.unwrap();
assert_eq!(std::fs::read(&path).unwrap(), png);
assert_eq!(path.extension().unwrap(), "png");
let err = fetcher
.fetch_cover(&id, &f.server.url("/page.png"))
.await
.unwrap_err();
assert!(
format!("{err:#}").contains("not a supported image"),
"{err:#}"
);
let err = fetcher
.fetch_cover(&id, &f.server.url("/huge.png"))
.await
.unwrap_err();
assert!(format!("{err:#}").contains("too large"), "{err:#}");
// Only the one finished file: nothing partial, no temporary files.
let files: Vec<_> = std::fs::read_dir(path.parent().unwrap())
.unwrap()
.map(|entry| entry.unwrap().path())
.collect();
assert_eq!(files, [path]);
}