Bound a whole feed fetch and test the fetch limits
A slow server could hold one of the four refresh slots for 60 s per redirect hop. A feed or cover fetch now has an overall deadline (120 s). Tests cover feeds over the size cap with and without Content-Length (error recorded with back-off, catalog kept), the redirect cap, a redirect loop, a non-http redirect, a hanging server, and covers (not an image, over the cap, written atomically). The test server can stall. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
268ea915da
commit
34c096a36b
4 files changed
+228
-9
No files matched your search
+4
-1
@@ -320,7 +320,10 @@ registers (subscription, share, progress) and listening sessions in
|
||||
The TUI owns what the contract leaves to clients:
|
||||
|
||||
- **Fetching.** Feeds are fetched with conditional requests (ETag,
|
||||
Last-Modified), bounded size and time, and redirects followed by hand. The
|
||||
Last-Modified), bounded size (32 MiB per feed, 10 MiB per cover) and time
|
||||
(15 s to connect, 60 s per request, 120 s for a whole fetch with its
|
||||
redirects), and at most 10 redirects (http and https only) followed by
|
||||
hand. The
|
||||
address that the leading permanent hops (301/308) lead to becomes a feed
|
||||
alias and the fetch URL, even when a temporary hop (CDN, tracker) follows
|
||||
it; a chain that starts with a temporary hop never moves the feed.
|
||||
|
||||
+38
-8
@@ -29,6 +29,12 @@ pub const MAX_REDIRECTS: usize = 10;
|
||||
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
/// Whole request, including the body, per hop.
|
||||
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
/// A whole feed or cover fetch, all redirect hops and the body included, so
|
||||
/// a slow server holds one of the few refresh slots for at most this long.
|
||||
#[cfg(not(test))]
|
||||
pub const FETCH_DEADLINE: Duration = Duration::from_secs(120);
|
||||
#[cfg(test)]
|
||||
pub const FETCH_DEADLINE: Duration = Duration::from_secs(3);
|
||||
/// Sent with every podcast request.
|
||||
pub const USER_AGENT: &str = concat!("furumi/", env!("CARGO_PKG_VERSION"), " (podcast reader)");
|
||||
|
||||
@@ -116,6 +122,14 @@ impl Fetcher {
|
||||
&self,
|
||||
url: &FeedUrl,
|
||||
validators: Validators<'_>,
|
||||
) -> Result<FeedResponse> {
|
||||
within_deadline("feed", self.fetch_feed_unbounded(url, validators)).await
|
||||
}
|
||||
|
||||
async fn fetch_feed_unbounded(
|
||||
&self,
|
||||
url: &FeedUrl,
|
||||
validators: Validators<'_>,
|
||||
) -> Result<FeedResponse> {
|
||||
let Followed {
|
||||
response,
|
||||
@@ -167,6 +181,17 @@ impl Fetcher {
|
||||
|
||||
/// Downloads a cover image into the covers directory; returns its path.
|
||||
pub async fn fetch_cover(&self, podcast_id: &PodcastId, image_url: &str) -> Result<PathBuf> {
|
||||
let body = within_deadline("cover image", self.download_cover(image_url)).await?;
|
||||
let format = image::guess_format(&body).context("the cover is not a supported image")?;
|
||||
let extension = format.extensions_str().first().copied().unwrap_or("img");
|
||||
let name = cover_file_name(podcast_id, image_url, extension);
|
||||
let dir = self.covers_dir.clone();
|
||||
tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body))
|
||||
.await
|
||||
.map_err(|err| anyhow!("cover writer task failed: {err}"))?
|
||||
}
|
||||
|
||||
async fn download_cover(&self, image_url: &str) -> Result<Vec<u8>> {
|
||||
let response = self
|
||||
.get_following(image_url, None, Validators::default())
|
||||
.await?
|
||||
@@ -175,14 +200,7 @@ impl Fetcher {
|
||||
if !status.is_success() {
|
||||
return Err(status_error(status, response.headers()).into());
|
||||
}
|
||||
let body = read_capped(response, MAX_COVER_BYTES, "cover image").await?;
|
||||
let format = image::guess_format(&body).context("the cover is not a supported image")?;
|
||||
let extension = format.extensions_str().first().copied().unwrap_or("img");
|
||||
let name = cover_file_name(podcast_id, image_url, extension);
|
||||
let dir = self.covers_dir.clone();
|
||||
tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body))
|
||||
.await
|
||||
.map_err(|err| anyhow!("cover writer task failed: {err}"))?
|
||||
read_capped(response, MAX_COVER_BYTES, "cover image").await
|
||||
}
|
||||
|
||||
/// GET following up to [`MAX_REDIRECTS`] http(s) redirects by hand.
|
||||
@@ -259,6 +277,18 @@ impl Fetcher {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fails `fetch` once [`FETCH_DEADLINE`] has passed.
|
||||
async fn within_deadline<T>(what: &str, fetch: impl Future<Output = Result<T>>) -> Result<T> {
|
||||
tokio::time::timeout(FETCH_DEADLINE, fetch)
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
bail!(
|
||||
"the {what} took longer than {} seconds",
|
||||
FETCH_DEADLINE.as_secs()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
struct Followed {
|
||||
response: reqwest::Response,
|
||||
/// URL that answered.
|
||||
|
||||
@@ -28,6 +28,8 @@ pub(super) struct Route {
|
||||
pub cut_after: Option<usize>,
|
||||
/// ...for this many requests; later requests are served in full.
|
||||
pub fail_times: usize,
|
||||
/// Wait this long before answering (a hanging server).
|
||||
pub stall: Option<Duration>,
|
||||
}
|
||||
|
||||
impl Route {
|
||||
@@ -44,9 +46,16 @@ impl Route {
|
||||
send_length: true,
|
||||
cut_after: None,
|
||||
fail_times: 0,
|
||||
stall: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Answer only after `delay`.
|
||||
pub fn stall(mut self, delay: Duration) -> Route {
|
||||
self.stall = Some(delay);
|
||||
self
|
||||
}
|
||||
|
||||
/// A server that ignores `Range` and always answers 200 with the file.
|
||||
pub fn no_ranges(mut self) -> Route {
|
||||
self.ranges = false;
|
||||
@@ -206,6 +215,9 @@ fn handle(stream: TcpStream, state: &Mutex<ServerState>) {
|
||||
(route, cut)
|
||||
};
|
||||
let mut out = stream;
|
||||
if let Some(delay) = route.as_ref().and_then(|route| route.stall) {
|
||||
std::thread::sleep(delay);
|
||||
}
|
||||
let Some(route) = route else {
|
||||
let _ = out
|
||||
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
|
||||
|
||||
@@ -12,6 +12,7 @@ use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
|
||||
use super::OpLog;
|
||||
use super::fetch::{FETCH_DEADLINE, MAX_COVER_BYTES, MAX_FEED_BYTES, MAX_REDIRECTS, Validators};
|
||||
use super::refresh::{
|
||||
FAILURE_BACKOFF_BASE, FAILURE_BACKOFF_MAX, REFRESH_INTERVAL, Services,
|
||||
next_check_after_failure, next_check_after_success, refresh, subscribe,
|
||||
@@ -741,3 +742,176 @@ async fn imported_podcasts_get_their_seasons_on_the_first_check() {
|
||||
Some("\"e1\"")
|
||||
);
|
||||
}
|
||||
|
||||
fn feed_url(f: &Fixture, path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&f.server.url(path)).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feeds_over_the_size_limit_are_refused_with_or_without_a_length() {
|
||||
let f = fixture();
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
|
||||
);
|
||||
let url = feed_url(&f, "/feed.xml");
|
||||
let id = subscribe(&f.services, url.as_str(), NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.podcast_id;
|
||||
let mut huge = rss(&[("a", "Pilot"), ("b", "Second")]);
|
||||
huge.resize(MAX_FEED_BYTES + 1, b' ');
|
||||
let store = &f.services.store;
|
||||
for (attempt, route) in [
|
||||
Route::new(huge.clone(), "application/rss+xml"),
|
||||
Route::new(huge, "application/rss+xml").no_length(),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let failures = attempt as u32 + 1;
|
||||
f.server.set("/feed.xml", route);
|
||||
let now = NOW + i64::from(failures);
|
||||
let err = refresh(&f.services, &id, &url, now).await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("too large"), "{err:#}");
|
||||
let state = store.fetch_state(&id).unwrap();
|
||||
assert_eq!(state.consecutive_failures, failures);
|
||||
assert!(state.last_error.unwrap().contains("too large"));
|
||||
assert_eq!(
|
||||
state.next_check_at_ms,
|
||||
Some(next_check_after_failure(now, failures, None))
|
||||
);
|
||||
assert_eq!(episode_count(store, &id), 1, "the catalog is kept");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn redirect_chains_are_bounded_and_lead_only_to_http() {
|
||||
let f = fixture();
|
||||
let fetch = |path: &'static str| {
|
||||
let url = feed_url(&f, path);
|
||||
let fetcher = std::sync::Arc::clone(&f.services.fetcher);
|
||||
async move { fetcher.fetch_feed(&url, Validators::default()).await }
|
||||
};
|
||||
for n in 0..=MAX_REDIRECTS {
|
||||
f.server.set(
|
||||
&format!("/r{n}.xml"),
|
||||
Route::redirect(302, &format!("/r{}.xml", n + 1)),
|
||||
);
|
||||
}
|
||||
f.server.set(
|
||||
&format!("/r{}.xml", MAX_REDIRECTS + 1),
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
|
||||
);
|
||||
// MAX_REDIRECTS hops are followed, one more is refused.
|
||||
let followed = fetch("/r1.xml").await.unwrap();
|
||||
assert_eq!(followed.redirects, MAX_REDIRECTS);
|
||||
let err = fetch("/r0.xml").await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("too many redirects"), "{err:#}");
|
||||
// A loop ends the same way.
|
||||
f.server.set("/a.xml", Route::redirect(301, "/b.xml"));
|
||||
f.server.set("/b.xml", Route::redirect(301, "/a.xml"));
|
||||
let err = fetch("/a.xml").await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("too many redirects"), "{err:#}");
|
||||
assert_eq!(f.server.requests("/a.xml").len(), MAX_REDIRECTS / 2 + 1);
|
||||
// Only http(s) is followed.
|
||||
f.server.set(
|
||||
"/ftp.xml",
|
||||
Route::redirect(302, "ftp://example.test/feed.xml"),
|
||||
);
|
||||
let err = fetch("/ftp.xml").await.unwrap_err();
|
||||
assert!(
|
||||
format!("{err:#}").contains("unsupported URL scheme"),
|
||||
"{err:#}"
|
||||
);
|
||||
// Subscribing through any of them stores nothing.
|
||||
for path in ["/r0.xml", "/a.xml", "/ftp.xml"] {
|
||||
assert!(
|
||||
subscribe(&f.services, &f.server.url(path), NOW)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
f.services
|
||||
.store
|
||||
.list_podcasts(PodcastListFilter::All)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_hanging_server_fails_the_check_at_the_deadline() {
|
||||
let f = fixture();
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
|
||||
);
|
||||
let url = feed_url(&f, "/feed.xml");
|
||||
let id = subscribe(&f.services, url.as_str(), NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.podcast_id;
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml")
|
||||
.stall(FETCH_DEADLINE + Duration::from_secs(5)),
|
||||
);
|
||||
let started = std::time::Instant::now();
|
||||
let err = refresh(&f.services, &id, &url, NOW + 1).await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("took longer than"), "{err:#}");
|
||||
assert!(started.elapsed() < FETCH_DEADLINE + Duration::from_secs(2));
|
||||
let state = f.services.store.fetch_state(&id).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert_eq!(episode_count(&f.services.store, &id), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn covers_must_be_images_within_the_limit_and_are_written_whole() {
|
||||
let f = fixture();
|
||||
let png = {
|
||||
let mut out = std::io::Cursor::new(Vec::new());
|
||||
image::RgbImage::new(2, 2)
|
||||
.write_to(&mut out, image::ImageFormat::Png)
|
||||
.unwrap();
|
||||
out.into_inner()
|
||||
};
|
||||
f.server
|
||||
.set("/cover.png", Route::new(png.clone(), "image/png"));
|
||||
f.server.set(
|
||||
"/page.png",
|
||||
Route::new("<html>not an image</html>", "image/png"),
|
||||
);
|
||||
f.server.set(
|
||||
"/huge.png",
|
||||
Route::new(vec![0u8; MAX_COVER_BYTES + 1], "image/png"),
|
||||
);
|
||||
let id = PodcastId::from_feed_url(&feed_url(&f, "/feed.xml"));
|
||||
let fetcher = &f.services.fetcher;
|
||||
let path = fetcher
|
||||
.fetch_cover(&id, &f.server.url("/cover.png"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), png);
|
||||
assert_eq!(path.extension().unwrap(), "png");
|
||||
let err = fetcher
|
||||
.fetch_cover(&id, &f.server.url("/page.png"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
format!("{err:#}").contains("not a supported image"),
|
||||
"{err:#}"
|
||||
);
|
||||
let err = fetcher
|
||||
.fetch_cover(&id, &f.server.url("/huge.png"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("too large"), "{err:#}");
|
||||
// Only the one finished file: nothing partial, no temporary files.
|
||||
let files: Vec<_> = std::fs::read_dir(path.parent().unwrap())
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.collect();
|
||||
assert_eq!(files, [path]);
|
||||
}
|
||||
Reference in new issue
Block a user