//! Feed fetching (conditional requests, redirects, limits) and the refresh //! schedule, against the local test server. use std::time::Duration; use music_dht::podcast::{FeedUrl, PodcastId}; use super::fetch::{FeedFetch, FeedFetcher, FetchErrorKind, Validators, parse_retry_after}; use super::runner::{ BACKOFF_BASE_MS, BACKOFF_MAX_MS, REFRESH_INTERVAL_MS, failure_backoff_ms, next_regular_check_ms, }; use std::sync::Arc; use super::test_server::{Route, ScriptedLookup, TestServer, test_http}; // --------------------------------------------------------------------------- // Feed fetching // --------------------------------------------------------------------------- fn rss(items: &str) -> String { format!( r#"Show{items}"# ) } fn item(guid: &str) -> String { format!( r#"{guid}{guid}"# ) } fn fetcher(lookup: Arc) -> FeedFetcher { FeedFetcher::with_http(test_http(lookup)) } fn feed_url(text: &str) -> FeedUrl { FeedUrl::parse(text).unwrap() } #[tokio::test] async fn feeds_are_fetched_conditionally() { let server = TestServer::start(); server.set( "/feed.xml", Route::new(rss(&item("a")), "application/rss+xml") .etag("\"v1\"") .last_modified("Wed, 21 Oct 2026 07:28:00 GMT"), ); let fetcher = fetcher(ScriptedLookup::with(&[])); let feed = feed_url(&server.url("/feed.xml")); let FeedFetch::Fetched(first) = fetcher.fetch(&feed, &Validators::default()).await.unwrap() else { panic!("expected a document"); }; assert_eq!(first.feed.episodes.len(), 1); assert_eq!( first.feed.episodes[0].audio_url, server.url("/audio/a.mp3"), "relative enclosures resolve against the feed URL" ); assert_eq!(first.fetched_from, feed); assert!(!first.moved_permanently); assert_eq!(first.validators.etag.as_deref(), Some("\"v1\"")); let second = fetcher.fetch(&feed, &first.validators).await.unwrap(); assert!(matches!(second, FeedFetch::NotModified)); // Only Last-Modified known. let by_date = Validators { etag: None, last_modified: first.validators.last_modified.clone(), }; assert!(matches!( fetcher.fetch(&feed, &by_date).await.unwrap(), FeedFetch::NotModified )); let requests = server.requests("/feed.xml"); assert_eq!(requests.len(), 3); assert!(!requests[0].contains_key("if-none-match")); assert_eq!(requests[1]["if-none-match"], "\"v1\""); assert_eq!( requests[1]["if-modified-since"], "Wed, 21 Oct 2026 07:28:00 GMT" ); assert!(!requests[2].contains_key("if-none-match")); assert_eq!(requests[2]["accept-encoding"], "identity"); } #[tokio::test] async fn only_permanent_redirect_chains_move_a_feed() { let server = TestServer::start(); server.set( "/new.xml", Route::new(rss(&item("a")), "application/rss+xml"), ); server.set("/old.xml", Route::redirect(301, "/new.xml")); server.set("/moved.xml", Route::redirect(308, "/old.xml")); server.set("/temp.xml", Route::redirect(302, "/new.xml")); server.set("/mixed.xml", Route::redirect(307, "/old.xml")); let fetcher = fetcher(ScriptedLookup::with(&[])); for (path, moved) in [ ("/old.xml", true), ("/moved.xml", true), ("/temp.xml", false), ("/mixed.xml", false), ] { let requested = feed_url(&server.url(path)); let FeedFetch::Fetched(document) = fetcher .fetch(&requested, &Validators::default()) .await .unwrap() else { panic!("{path}"); }; assert_eq!(document.moved_permanently, moved, "{path}"); let expected = if moved { feed_url(&server.url("/new.xml")) } else { requested }; assert_eq!(document.fetched_from, expected, "{path}"); assert_eq!( document.feed.episodes[0].audio_url, server.url("/audio/a.mp3") ); } } #[tokio::test] async fn feed_fetch_failures_are_typed() { let server = TestServer::start(); let big = rss(&item("x").repeat(200)); server.set("/big.xml", Route::new(big.clone(), "application/rss+xml")); server.set( "/big-chunked.xml", Route::new(big.clone(), "application/rss+xml").no_length(), ); server.set( "/page.html", Route::new("hi", "text/html"), ); server.set( "/busy.xml", Route::new("busy", "text/plain") .status(503) .header("Retry-After", "120"), ); server.set("/gone.xml", Route::new("", "text/plain").status(410)); server.set( "/private.xml", Route::redirect(301, "http://192.168.0.1/feed"), ); let fetcher = fetcher(ScriptedLookup::with(&[])).with_max_bytes(big.len() - 1); let fetch = |path: &str| { let feed = feed_url(&server.url(path)); let fetcher = fetcher.clone(); async move { fetcher .fetch(&feed, &Validators::default()) .await .unwrap_err() } }; assert_eq!(fetch("/big.xml").await.kind, FetchErrorKind::TooLarge); assert_eq!( fetch("/big-chunked.xml").await.kind, FetchErrorKind::TooLarge ); assert_eq!(fetch("/page.html").await.kind, FetchErrorKind::Parse); let busy = fetch("/busy.xml").await; assert_eq!(busy.kind, FetchErrorKind::Status(503)); assert_eq!(busy.retry_after, Some(Duration::from_secs(120))); let gone = fetch("/gone.xml").await; assert_eq!(gone.kind, FetchErrorKind::Status(410)); assert_eq!(gone.retry_after, None); assert_eq!(fetch("/private.xml").await.kind, FetchErrorKind::Refused); assert_eq!( fetch("/missing.xml").await.kind, FetchErrorKind::Status(404) ); } #[test] fn retry_after_accepts_seconds_and_dates() { let now = chrono::DateTime::parse_from_rfc3339("2026-11-19T10:00:00Z") .unwrap() .with_timezone(&chrono::Utc); assert_eq!(parse_retry_after("90", now), Some(Duration::from_secs(90))); assert_eq!( parse_retry_after("Thu, 19 Nov 2026 10:05:00 GMT", now), Some(Duration::from_secs(300)) ); assert_eq!( parse_retry_after("Thu, 19 Nov 2026 09:00:00 GMT", now), Some(Duration::ZERO) ); assert_eq!( parse_retry_after("999999999", now), Some(Duration::from_secs(24 * 60 * 60)) ); assert_eq!(parse_retry_after("soon", now), None); } // --------------------------------------------------------------------------- // Refresh schedule // --------------------------------------------------------------------------- #[test] fn failures_back_off_exponentially_up_to_a_day() { assert_eq!(failure_backoff_ms(1, None), BACKOFF_BASE_MS); assert_eq!(failure_backoff_ms(2, None), 2 * BACKOFF_BASE_MS); assert_eq!(failure_backoff_ms(3, None), 4 * BACKOFF_BASE_MS); assert_eq!(failure_backoff_ms(7, None), 64 * BACKOFF_BASE_MS); assert_eq!(failure_backoff_ms(8, None), BACKOFF_MAX_MS); assert_eq!(failure_backoff_ms(u32::MAX, None), BACKOFF_MAX_MS); // A longer Retry-After wins, a shorter one does not shorten the backoff. assert_eq!( failure_backoff_ms(1, Some(Duration::from_secs(3 * 60 * 60))), 3 * 60 * 60 * 1000 ); assert_eq!( failure_backoff_ms(2, Some(Duration::from_secs(1))), 2 * BACKOFF_BASE_MS ); assert_eq!( failure_backoff_ms(1, Some(Duration::from_secs(u64::MAX))), BACKOFF_MAX_MS ); } #[test] fn regular_checks_are_hourly_with_a_stable_spread() { let podcast = PodcastId::from_feed_url(&feed_url("https://feeds.example/a.xml")); let other = PodcastId::from_feed_url(&feed_url("https://feeds.example/b.xml")); let now = 1_000_000; let next = next_regular_check_ms(now, 7, &podcast); assert!( (now + REFRESH_INTERVAL_MS..now + REFRESH_INTERVAL_MS + 10 * 60 * 1000).contains(&next) ); assert_eq!(next, next_regular_check_ms(now, 7, &podcast)); let spread: std::collections::BTreeSet = (0..20) .map(|user| next_regular_check_ms(now, user, &podcast)) .chain([next_regular_check_ms(now, 7, &other)]) .collect(); assert!(spread.len() > 10); } /// A port on which nothing listens. fn closed_port() -> u16 { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); listener.local_addr().unwrap().port() } #[tokio::test] async fn internal_names_cannot_be_told_from_unreachable_ones() { // A name that resolves to a private address, a name that does not // resolve and a server that refuses the connection look the same to // the user: otherwise "add RSS" would tell which names exist in the // server's internal DNS. Only what the URL itself says (an IP literal, // the port, the scheme) is refused as such. let port = closed_port(); let lookup = ScriptedLookup::with(&[ ("vault.internal.test", &[&["10.20.30.40"]]), ("closed.test", &[&["127.0.0.1"]]), ]); let fetcher = fetcher(lookup); let mut answers = Vec::new(); for text in [ format!("http://vault.internal.test:{port}/feed"), format!("http://nowhere.internal.test:{port}/feed"), format!("http://closed.test:{port}/feed"), ] { let error = fetcher .fetch(&feed_url(&text), &Validators::default()) .await .unwrap_err(); assert!(!error.public_message().contains("internal"), "{text}"); answers.push((error.public_message(), error.kind)); } assert_eq!(answers[0].1, FetchErrorKind::Transport, "{answers:?}"); assert!( answers.iter().all(|answer| *answer == answers[0]), "{answers:?}" ); let literal = fetcher .fetch(&feed_url("http://10.20.30.40/feed"), &Validators::default()) .await .unwrap_err(); assert_eq!(literal.kind, FetchErrorKind::Refused); assert_ne!(literal.public_message(), answers[0].0); }