From 34c096a36bfd7cab78ef0a99ca5538f58370f30a Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 21:53:44 +0200 Subject: [PATCH] Bound a whole feed fetch and test the fetch limits A slow server could hold one of the four refresh slots for 60 s per redirect hop. A feed or cover fetch now has an overall deadline (120 s). Tests cover feeds over the size cap with and without Content-Length (error recorded with back-off, catalog kept), the redirect cap, a redirect loop, a non-http redirect, a hanging server, and covers (not an image, over the cap, written atomically). The test server can stall. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 5 +- src/podcasts/fetch.rs | 46 ++++++++-- src/podcasts/test_server.rs | 12 +++ src/podcasts/tests.rs | 174 ++++++++++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 9 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0621b9a..7cdd389 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -320,7 +320,10 @@ registers (subscription, share, progress) and listening sessions in The TUI owns what the contract leaves to clients: - **Fetching.** Feeds are fetched with conditional requests (ETag, - Last-Modified), bounded size and time, and redirects followed by hand. The + Last-Modified), bounded size (32 MiB per feed, 10 MiB per cover) and time + (15 s to connect, 60 s per request, 120 s for a whole fetch with its + redirects), and at most 10 redirects (http and https only) followed by + hand. The address that the leading permanent hops (301/308) lead to becomes a feed alias and the fetch URL, even when a temporary hop (CDN, tracker) follows it; a chain that starts with a temporary hop never moves the feed. diff --git a/src/podcasts/fetch.rs b/src/podcasts/fetch.rs index bf2078b..f6cbb3f 100644 --- a/src/podcasts/fetch.rs +++ b/src/podcasts/fetch.rs @@ -29,6 +29,12 @@ pub const MAX_REDIRECTS: usize = 10; pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); /// Whole request, including the body, per hop. pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60); +/// A whole feed or cover fetch, all redirect hops and the body included, so +/// a slow server holds one of the few refresh slots for at most this long. +#[cfg(not(test))] +pub const FETCH_DEADLINE: Duration = Duration::from_secs(120); +#[cfg(test)] +pub const FETCH_DEADLINE: Duration = Duration::from_secs(3); /// Sent with every podcast request. pub const USER_AGENT: &str = concat!("furumi/", env!("CARGO_PKG_VERSION"), " (podcast reader)"); @@ -116,6 +122,14 @@ impl Fetcher { &self, url: &FeedUrl, validators: Validators<'_>, + ) -> Result { + within_deadline("feed", self.fetch_feed_unbounded(url, validators)).await + } + + async fn fetch_feed_unbounded( + &self, + url: &FeedUrl, + validators: Validators<'_>, ) -> Result { let Followed { response, @@ -167,6 +181,17 @@ impl Fetcher { /// Downloads a cover image into the covers directory; returns its path. pub async fn fetch_cover(&self, podcast_id: &PodcastId, image_url: &str) -> Result { + let body = within_deadline("cover image", self.download_cover(image_url)).await?; + let format = image::guess_format(&body).context("the cover is not a supported image")?; + let extension = format.extensions_str().first().copied().unwrap_or("img"); + let name = cover_file_name(podcast_id, image_url, extension); + let dir = self.covers_dir.clone(); + tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body)) + .await + .map_err(|err| anyhow!("cover writer task failed: {err}"))? + } + + async fn download_cover(&self, image_url: &str) -> Result> { let response = self .get_following(image_url, None, Validators::default()) .await? @@ -175,14 +200,7 @@ impl Fetcher { if !status.is_success() { return Err(status_error(status, response.headers()).into()); } - let body = read_capped(response, MAX_COVER_BYTES, "cover image").await?; - let format = image::guess_format(&body).context("the cover is not a supported image")?; - let extension = format.extensions_str().first().copied().unwrap_or("img"); - let name = cover_file_name(podcast_id, image_url, extension); - let dir = self.covers_dir.clone(); - tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body)) - .await - .map_err(|err| anyhow!("cover writer task failed: {err}"))? + read_capped(response, MAX_COVER_BYTES, "cover image").await } /// GET following up to [`MAX_REDIRECTS`] http(s) redirects by hand. @@ -259,6 +277,18 @@ impl Fetcher { } } +/// Fails `fetch` once [`FETCH_DEADLINE`] has passed. +async fn within_deadline(what: &str, fetch: impl Future>) -> Result { + tokio::time::timeout(FETCH_DEADLINE, fetch) + .await + .unwrap_or_else(|_| { + bail!( + "the {what} took longer than {} seconds", + FETCH_DEADLINE.as_secs() + ) + }) +} + struct Followed { response: reqwest::Response, /// URL that answered. diff --git a/src/podcasts/test_server.rs b/src/podcasts/test_server.rs index 3f3ff74..65c395c 100644 --- a/src/podcasts/test_server.rs +++ b/src/podcasts/test_server.rs @@ -28,6 +28,8 @@ pub(super) struct Route { pub cut_after: Option, /// ...for this many requests; later requests are served in full. pub fail_times: usize, + /// Wait this long before answering (a hanging server). + pub stall: Option, } impl Route { @@ -44,9 +46,16 @@ impl Route { send_length: true, cut_after: None, fail_times: 0, + stall: None, } } + /// Answer only after `delay`. + pub fn stall(mut self, delay: Duration) -> Route { + self.stall = Some(delay); + self + } + /// A server that ignores `Range` and always answers 200 with the file. pub fn no_ranges(mut self) -> Route { self.ranges = false; @@ -206,6 +215,9 @@ fn handle(stream: TcpStream, state: &Mutex) { (route, cut) }; let mut out = stream; + if let Some(delay) = route.as_ref().and_then(|route| route.stall) { + std::thread::sleep(delay); + } let Some(route) = route else { let _ = out .write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); diff --git a/src/podcasts/tests.rs b/src/podcasts/tests.rs index 3758393..c0f3ba6 100644 --- a/src/podcasts/tests.rs +++ b/src/podcasts/tests.rs @@ -12,6 +12,7 @@ use music_dht::device_sync::{SyncOpPayload, SyncOpWire}; use music_dht::podcast::{FeedUrl, PodcastId}; use super::OpLog; +use super::fetch::{FETCH_DEADLINE, MAX_COVER_BYTES, MAX_FEED_BYTES, MAX_REDIRECTS, Validators}; use super::refresh::{ FAILURE_BACKOFF_BASE, FAILURE_BACKOFF_MAX, REFRESH_INTERVAL, Services, next_check_after_failure, next_check_after_success, refresh, subscribe, @@ -741,3 +742,176 @@ async fn imported_podcasts_get_their_seasons_on_the_first_check() { Some("\"e1\"") ); } + +fn feed_url(f: &Fixture, path: &str) -> FeedUrl { + FeedUrl::parse(&f.server.url(path)).unwrap() +} + +#[tokio::test] +async fn feeds_over_the_size_limit_are_refused_with_or_without_a_length() { + let f = fixture(); + f.server.set( + "/feed.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"), + ); + let url = feed_url(&f, "/feed.xml"); + let id = subscribe(&f.services, url.as_str(), NOW) + .await + .unwrap() + .podcast_id; + let mut huge = rss(&[("a", "Pilot"), ("b", "Second")]); + huge.resize(MAX_FEED_BYTES + 1, b' '); + let store = &f.services.store; + for (attempt, route) in [ + Route::new(huge.clone(), "application/rss+xml"), + Route::new(huge, "application/rss+xml").no_length(), + ] + .into_iter() + .enumerate() + { + let failures = attempt as u32 + 1; + f.server.set("/feed.xml", route); + let now = NOW + i64::from(failures); + let err = refresh(&f.services, &id, &url, now).await.unwrap_err(); + assert!(format!("{err:#}").contains("too large"), "{err:#}"); + let state = store.fetch_state(&id).unwrap(); + assert_eq!(state.consecutive_failures, failures); + assert!(state.last_error.unwrap().contains("too large")); + assert_eq!( + state.next_check_at_ms, + Some(next_check_after_failure(now, failures, None)) + ); + assert_eq!(episode_count(store, &id), 1, "the catalog is kept"); + } +} + +#[tokio::test] +async fn redirect_chains_are_bounded_and_lead_only_to_http() { + let f = fixture(); + let fetch = |path: &'static str| { + let url = feed_url(&f, path); + let fetcher = std::sync::Arc::clone(&f.services.fetcher); + async move { fetcher.fetch_feed(&url, Validators::default()).await } + }; + for n in 0..=MAX_REDIRECTS { + f.server.set( + &format!("/r{n}.xml"), + Route::redirect(302, &format!("/r{}.xml", n + 1)), + ); + } + f.server.set( + &format!("/r{}.xml", MAX_REDIRECTS + 1), + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"), + ); + // MAX_REDIRECTS hops are followed, one more is refused. + let followed = fetch("/r1.xml").await.unwrap(); + assert_eq!(followed.redirects, MAX_REDIRECTS); + let err = fetch("/r0.xml").await.unwrap_err(); + assert!(format!("{err:#}").contains("too many redirects"), "{err:#}"); + // A loop ends the same way. + f.server.set("/a.xml", Route::redirect(301, "/b.xml")); + f.server.set("/b.xml", Route::redirect(301, "/a.xml")); + let err = fetch("/a.xml").await.unwrap_err(); + assert!(format!("{err:#}").contains("too many redirects"), "{err:#}"); + assert_eq!(f.server.requests("/a.xml").len(), MAX_REDIRECTS / 2 + 1); + // Only http(s) is followed. + f.server.set( + "/ftp.xml", + Route::redirect(302, "ftp://example.test/feed.xml"), + ); + let err = fetch("/ftp.xml").await.unwrap_err(); + assert!( + format!("{err:#}").contains("unsupported URL scheme"), + "{err:#}" + ); + // Subscribing through any of them stores nothing. + for path in ["/r0.xml", "/a.xml", "/ftp.xml"] { + assert!( + subscribe(&f.services, &f.server.url(path), NOW) + .await + .is_err() + ); + } + assert!( + f.services + .store + .list_podcasts(PodcastListFilter::All) + .unwrap() + .is_empty() + ); +} + +#[tokio::test] +async fn a_hanging_server_fails_the_check_at_the_deadline() { + let f = fixture(); + f.server.set( + "/feed.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"), + ); + let url = feed_url(&f, "/feed.xml"); + let id = subscribe(&f.services, url.as_str(), NOW) + .await + .unwrap() + .podcast_id; + f.server.set( + "/feed.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml") + .stall(FETCH_DEADLINE + Duration::from_secs(5)), + ); + let started = std::time::Instant::now(); + let err = refresh(&f.services, &id, &url, NOW + 1).await.unwrap_err(); + assert!(format!("{err:#}").contains("took longer than"), "{err:#}"); + assert!(started.elapsed() < FETCH_DEADLINE + Duration::from_secs(2)); + let state = f.services.store.fetch_state(&id).unwrap(); + assert_eq!(state.consecutive_failures, 1); + assert_eq!(episode_count(&f.services.store, &id), 1); +} + +#[tokio::test] +async fn covers_must_be_images_within_the_limit_and_are_written_whole() { + let f = fixture(); + let png = { + let mut out = std::io::Cursor::new(Vec::new()); + image::RgbImage::new(2, 2) + .write_to(&mut out, image::ImageFormat::Png) + .unwrap(); + out.into_inner() + }; + f.server + .set("/cover.png", Route::new(png.clone(), "image/png")); + f.server.set( + "/page.png", + Route::new("not an image", "image/png"), + ); + f.server.set( + "/huge.png", + Route::new(vec![0u8; MAX_COVER_BYTES + 1], "image/png"), + ); + let id = PodcastId::from_feed_url(&feed_url(&f, "/feed.xml")); + let fetcher = &f.services.fetcher; + let path = fetcher + .fetch_cover(&id, &f.server.url("/cover.png")) + .await + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), png); + assert_eq!(path.extension().unwrap(), "png"); + let err = fetcher + .fetch_cover(&id, &f.server.url("/page.png")) + .await + .unwrap_err(); + assert!( + format!("{err:#}").contains("not a supported image"), + "{err:#}" + ); + let err = fetcher + .fetch_cover(&id, &f.server.url("/huge.png")) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("too large"), "{err:#}"); + // Only the one finished file: nothing partial, no temporary files. + let files: Vec<_> = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect(); + assert_eq!(files, [path]); +}