diff --git a/src/jobs/mod.rs b/src/jobs/mod.rs index 3e997d4..f9e50de 100644 --- a/src/jobs/mod.rs +++ b/src/jobs/mod.rs @@ -6,6 +6,7 @@ pub mod lastfm_popularity; pub mod lastfm_scrobble; pub mod metadata_backfill; pub mod musicbrainz; +pub mod podcast_refresh; use std::path::{Component, Path, PathBuf}; diff --git a/src/jobs/podcast_refresh.rs b/src/jobs/podcast_refresh.rs new file mode 100644 index 0000000..a1fc850 --- /dev/null +++ b/src/jobs/podcast_refresh.rs @@ -0,0 +1,73 @@ +//! Background refresh of subscribed podcast feeds. +//! +//! Every run takes the podcasts whose next check is due (see +//! [`crate::podcasts::runner`]): healthy feeds are checked about hourly with +//! conditional requests, failing ones back off exponentially up to a day. +//! One request per feed URL serves every account subscribed to it, a +//! bounded number of requests run concurrently, and a failing feed never +//! blocks the others or clears the stored catalog. + +use crate::podcasts::{FeedFetcher, PodcastStore, now_ms, runner}; +use crate::scheduler::{Job, JobContext, JobLog}; + +pub struct PodcastRefreshJob; + +/// Session-level advisory lock: one run at a time across processes. +const ADVISORY_LOCK_ID: i64 = 0x504f_4443_5246_5348; // "PODCRFSH" + +/// Failures listed individually in the run log. +const LOGGED_FAILURES: usize = 20; + +#[async_trait::async_trait] +impl Job for PodcastRefreshJob { + fn name(&self) -> &'static str { + "podcast_refresh" + } + + fn description(&self) -> &'static str { + "Check subscribed podcast feeds for new episodes (metadata only, no audio)" + } + + fn default_cron(&self) -> &'static str { + // Every five minutes; each feed is checked only when due. + "0 */5 * * * *" + } + + async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> { + let mut conn = ctx.pool.acquire().await?; + // A session lock must never return to the pool, including on + // cancellation; the store below uses other pool connections. + conn.close_on_drop(); + let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(ADVISORY_LOCK_ID) + .fetch_one(&mut *conn) + .await?; + if !locked { + log.info("Podcast refresh is already running; skipping this run"); + return Ok(()); + } + let store = PodcastStore::new(ctx.pool.clone()); + let fetcher = FeedFetcher::new()?; + let result = runner::refresh_due( + &store, + &fetcher, + now_ms(), + runner::RUN_LIMIT, + runner::RUN_CONCURRENCY, + ) + .await; + conn.close().await?; + let summary = result?; + for (user_id, podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) { + log.warn(&format!("user {user_id}, podcast {podcast_id}: {error}")); + } + if summary.failures.len() > LOGGED_FAILURES { + log.warn(&format!( + "... and {} more failed feeds", + summary.failures.len() - LOGGED_FAILURES + )); + } + log.info(&format!("Podcast refresh: {summary}")); + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index 0c41cc4..20ab040 100644 --- a/src/main.rs +++ b/src/main.rs @@ -64,6 +64,7 @@ fn build_registry() -> Arc { registry.register(jobs::metadata_backfill::MetadataBackfillJob); registry.register(jobs::lastfm_popularity::LastfmPopularityJob); registry.register(jobs::lastfm_scrobble::LastfmScrobbleJob); + registry.register(jobs::podcast_refresh::PodcastRefreshJob); Arc::new(registry) } diff --git a/src/podcasts/fetch.rs b/src/podcasts/fetch.rs new file mode 100644 index 0000000..a510b33 --- /dev/null +++ b/src/podcasts/fetch.rs @@ -0,0 +1,303 @@ +//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`]. + +use std::fmt; +use std::time::Duration; + +use reqwest::header::{ + ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH, + LAST_MODIFIED, RETRY_AFTER, +}; +use reqwest::{StatusCode, Url}; + +use music_dht::podcast::FeedUrl; +use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; + +use super::net::{GuardError, SafeHttp, SafeHttpConfig}; + +/// Hard cap on a feed document (large catalogs are a few MiB). +pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024; +/// Whole fetch, from connect to the last body byte. +pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60); +/// Redirect hops followed for a feed. +pub(crate) const MAX_FEED_REDIRECTS: usize = 5; +/// `Retry-After` values above this are clamped. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60); + +const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \ + text/xml;q=0.9, */*;q=0.5"; + +/// HTTP client settings of the feed fetcher. +pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig { + connect_timeout: Duration::from_secs(10), + read_timeout: Duration::from_secs(30), + total_timeout: Some(FEED_FETCH_TIMEOUT), + max_redirects: MAX_FEED_REDIRECTS, +}; + +/// Validators of the previous successful fetch. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub(crate) struct Validators { + pub(crate) etag: Option, + pub(crate) last_modified: Option, +} + +/// A successfully fetched and parsed document. +#[derive(Debug, Clone)] +pub(crate) struct FetchedFeed { + pub(crate) feed: ParsedFeed, + /// Where the feed now lives: the final URL after a chain of permanent + /// redirects, otherwise the requested URL. + pub(crate) fetched_from: FeedUrl, + /// Whether the feed moved: at least one redirect, all permanent (see + /// [`super::net::Followed::moved_permanently`]), to a valid feed URL. + pub(crate) moved_permanently: bool, + /// New validators for the next conditional request. + pub(crate) validators: Validators, +} + +/// Outcome of [`FeedFetcher::fetch`]. +#[derive(Debug, Clone)] +pub(crate) enum FeedFetch { + /// `304 Not Modified`: the stored catalog is current. + NotModified, + /// A new document. + Fetched(Box), +} + +/// Kind of a failed fetch. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum FetchErrorKind { + /// Refused by the SSRF guard (blocked address, scheme, credentials). + Refused, + /// Non-success HTTP status. + Status(u16), + /// Body over [`MAX_FEED_BYTES`]. + TooLarge, + /// The document is not a feed. + Parse, + /// Timed out. + Timeout, + /// DNS, connect, TLS or a dropped connection. + Transport, +} + +/// A failed fetch. The stored catalog is never touched on failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FetchError { + pub(crate) kind: FetchErrorKind, + pub(crate) message: String, + /// Server-requested delay (`Retry-After` on 429/503), clamped to a day. + pub(crate) retry_after: Option, +} + +impl FetchError { + fn new(kind: FetchErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + retry_after: None, + } + } +} + +impl fmt::Display for FetchError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for FetchError {} + +impl From for FetchError { + fn from(error: GuardError) -> Self { + let kind = match &error { + GuardError::UnsupportedScheme(_) + | GuardError::Credentials + | GuardError::MissingHost + | GuardError::Blocked(_) => FetchErrorKind::Refused, + _ if error.is_timeout() => FetchErrorKind::Timeout, + _ => FetchErrorKind::Transport, + }; + Self::new(kind, error.to_string()) + } +} + +/// Parses `Retry-After` (delta seconds or an HTTP date). +pub(crate) fn parse_retry_after( + value: &str, + now: chrono::DateTime, +) -> Option { + let value = value.trim(); + let delay = if let Ok(seconds) = value.parse::() { + Duration::from_secs(seconds) + } else { + let at = chrono::DateTime::parse_from_rfc2822(value).ok()?; + (at.with_timezone(&chrono::Utc) - now) + .to_std() + .unwrap_or_default() + }; + Some(delay.min(MAX_RETRY_AFTER)) +} + +/// Downloads feeds with the SSRF guard, conditional requests, redirects +/// and hard size/time limits; parses them with the shared parser. +#[derive(Debug, Clone)] +pub(crate) struct FeedFetcher { + http: SafeHttp, + max_bytes: usize, +} + +impl FeedFetcher { + /// The production fetcher (public addresses only). + pub(crate) fn new() -> anyhow::Result { + Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?)) + } + + pub(crate) fn with_http(http: SafeHttp) -> Self { + Self { + http, + max_bytes: MAX_FEED_BYTES, + } + } + + #[cfg(test)] + pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self { + self.max_bytes = max_bytes; + self + } + + /// Fetches `url`, conditionally when `validators` are known. + pub(crate) async fn fetch( + &self, + url: &FeedUrl, + validators: &Validators, + ) -> Result { + match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await { + Ok(result) => result, + Err(_) => Err(FetchError::new( + FetchErrorKind::Timeout, + format!( + "feed fetch timed out after {} s", + FEED_FETCH_TIMEOUT.as_secs() + ), + )), + } + } + + async fn fetch_inner( + &self, + feed_url: &FeedUrl, + validators: &Validators, + ) -> Result { + let url = Url::parse(feed_url.as_str()) + .map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?; + let mut headers = HeaderMap::new(); + headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT)); + // Compressed bodies are not negotiated (no decoder is enabled), so + // the size cap applies to what is parsed. + headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + if let Some(etag) = validators + .etag + .as_deref() + .and_then(|value| HeaderValue::from_str(value).ok()) + { + headers.insert(IF_NONE_MATCH, etag); + } + if let Some(date) = validators + .last_modified + .as_deref() + .and_then(|value| HeaderValue::from_str(value).ok()) + { + headers.insert(IF_MODIFIED_SINCE, date); + } + let followed = self.http.get(url, &headers).await?; + let mut moved_permanently = followed.moved_permanently(); + let final_url = followed.final_url; + let response = followed.response; + let status = response.status(); + if status == StatusCode::NOT_MODIFIED { + return Ok(FeedFetch::NotModified); + } + if !status.is_success() { + let mut error = FetchError::new( + FetchErrorKind::Status(status.as_u16()), + format!("feed request failed: HTTP {status}"), + ); + if matches!(status.as_u16(), 429 | 503) { + error.retry_after = response + .headers() + .get(RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| parse_retry_after(value, chrono::Utc::now())); + } + return Err(error); + } + let header = |name| { + response + .headers() + .get(name) + .and_then(|value: &HeaderValue| value.to_str().ok()) + .map(str::to_string) + }; + let validators = Validators { + etag: header(ETAG), + last_modified: header(LAST_MODIFIED), + }; + let body = read_capped(response, self.max_bytes).await?; + // Relative enclosure URLs resolve against where the feed really is. + let base = final_url.to_string(); + let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base))) + .await + .map_err(|error| { + FetchError::new( + FetchErrorKind::Parse, + format!("feed parser task failed: {error}"), + ) + })? + .map_err(|error| { + FetchError::new( + FetchErrorKind::Parse, + format!("not a podcast feed: {error}"), + ) + })?; + // Only a permanent move changes the feed URL; a target that is not a + // valid feed URL (over-long, ...) is not learned. + let moved_to = moved_permanently + .then(|| FeedUrl::parse(final_url.as_str()).ok()) + .flatten(); + moved_permanently = moved_to.is_some(); + let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone()); + Ok(FeedFetch::Fetched(Box::new(FetchedFeed { + feed, + fetched_from, + moved_permanently, + validators, + }))) + } +} + +async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result, FetchError> { + let too_large = |detail: String| { + FetchError::new( + FetchErrorKind::TooLarge, + format!("feed is too large ({detail}, limit {cap} bytes)"), + ) + }; + if let Some(len) = response.content_length() + && len > cap as u64 + { + return Err(too_large(format!("{len} bytes"))); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|error| FetchError::from(GuardError::from(error)))? + { + if body.len() + chunk.len() > cap { + return Err(too_large("body exceeds the limit".to_string())); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} diff --git a/src/podcasts/fetch_tests.rs b/src/podcasts/fetch_tests.rs new file mode 100644 index 0000000..e612a2a --- /dev/null +++ b/src/podcasts/fetch_tests.rs @@ -0,0 +1,244 @@ +//! Feed fetching (conditional requests, redirects, limits) and the refresh +//! schedule, against the local test server. + +use std::time::Duration; + +use music_dht::podcast::{FeedUrl, PodcastId}; + +use super::fetch::{FeedFetch, FeedFetcher, FetchErrorKind, Validators, parse_retry_after}; +use super::runner::{ + BACKOFF_BASE_MS, BACKOFF_MAX_MS, REFRESH_INTERVAL_MS, failure_backoff_ms, next_regular_check_ms, +}; +use std::sync::Arc; + +use super::test_server::{Route, ScriptedLookup, TestServer, test_http}; + +// --------------------------------------------------------------------------- +// Feed fetching +// --------------------------------------------------------------------------- + +fn rss(items: &str) -> String { + format!( + r#"Show{items}"# + ) +} + +fn item(guid: &str) -> String { + format!( + r#"{guid}{guid}"# + ) +} + +fn fetcher(lookup: Arc) -> FeedFetcher { + FeedFetcher::with_http(test_http(lookup)) +} + +fn feed_url(text: &str) -> FeedUrl { + FeedUrl::parse(text).unwrap() +} + +#[tokio::test] +async fn feeds_are_fetched_conditionally() { + let server = TestServer::start(); + server.set( + "/feed.xml", + Route::new(rss(&item("a")), "application/rss+xml") + .etag("\"v1\"") + .last_modified("Wed, 21 Oct 2026 07:28:00 GMT"), + ); + let fetcher = fetcher(ScriptedLookup::with(&[])); + let feed = feed_url(&server.url("/feed.xml")); + let FeedFetch::Fetched(first) = fetcher.fetch(&feed, &Validators::default()).await.unwrap() + else { + panic!("expected a document"); + }; + assert_eq!(first.feed.episodes.len(), 1); + assert_eq!( + first.feed.episodes[0].audio_url, + server.url("/audio/a.mp3"), + "relative enclosures resolve against the feed URL" + ); + assert_eq!(first.fetched_from, feed); + assert!(!first.moved_permanently); + assert_eq!(first.validators.etag.as_deref(), Some("\"v1\"")); + let second = fetcher.fetch(&feed, &first.validators).await.unwrap(); + assert!(matches!(second, FeedFetch::NotModified)); + // Only Last-Modified known. + let by_date = Validators { + etag: None, + last_modified: first.validators.last_modified.clone(), + }; + assert!(matches!( + fetcher.fetch(&feed, &by_date).await.unwrap(), + FeedFetch::NotModified + )); + let requests = server.requests("/feed.xml"); + assert_eq!(requests.len(), 3); + assert!(!requests[0].contains_key("if-none-match")); + assert_eq!(requests[1]["if-none-match"], "\"v1\""); + assert_eq!( + requests[1]["if-modified-since"], + "Wed, 21 Oct 2026 07:28:00 GMT" + ); + assert!(!requests[2].contains_key("if-none-match")); + assert_eq!(requests[2]["accept-encoding"], "identity"); +} + +#[tokio::test] +async fn only_permanent_redirect_chains_move_a_feed() { + let server = TestServer::start(); + server.set( + "/new.xml", + Route::new(rss(&item("a")), "application/rss+xml"), + ); + server.set("/old.xml", Route::redirect(301, "/new.xml")); + server.set("/moved.xml", Route::redirect(308, "/old.xml")); + server.set("/temp.xml", Route::redirect(302, "/new.xml")); + server.set("/mixed.xml", Route::redirect(307, "/old.xml")); + let fetcher = fetcher(ScriptedLookup::with(&[])); + for (path, moved) in [ + ("/old.xml", true), + ("/moved.xml", true), + ("/temp.xml", false), + ("/mixed.xml", false), + ] { + let requested = feed_url(&server.url(path)); + let FeedFetch::Fetched(document) = fetcher + .fetch(&requested, &Validators::default()) + .await + .unwrap() + else { + panic!("{path}"); + }; + assert_eq!(document.moved_permanently, moved, "{path}"); + let expected = if moved { + feed_url(&server.url("/new.xml")) + } else { + requested + }; + assert_eq!(document.fetched_from, expected, "{path}"); + assert_eq!( + document.feed.episodes[0].audio_url, + server.url("/audio/a.mp3") + ); + } +} + +#[tokio::test] +async fn feed_fetch_failures_are_typed() { + let server = TestServer::start(); + let big = rss(&item("x").repeat(200)); + server.set("/big.xml", Route::new(big.clone(), "application/rss+xml")); + server.set( + "/big-chunked.xml", + Route::new(big.clone(), "application/rss+xml").no_length(), + ); + server.set( + "/page.html", + Route::new("hi", "text/html"), + ); + server.set( + "/busy.xml", + Route::new("busy", "text/plain") + .status(503) + .header("Retry-After", "120"), + ); + server.set("/gone.xml", Route::new("", "text/plain").status(410)); + server.set( + "/private.xml", + Route::redirect(301, "http://192.168.0.1/feed"), + ); + let fetcher = fetcher(ScriptedLookup::with(&[])).with_max_bytes(big.len() - 1); + let fetch = |path: &str| { + let feed = feed_url(&server.url(path)); + let fetcher = fetcher.clone(); + async move { + fetcher + .fetch(&feed, &Validators::default()) + .await + .unwrap_err() + } + }; + assert_eq!(fetch("/big.xml").await.kind, FetchErrorKind::TooLarge); + assert_eq!( + fetch("/big-chunked.xml").await.kind, + FetchErrorKind::TooLarge + ); + assert_eq!(fetch("/page.html").await.kind, FetchErrorKind::Parse); + let busy = fetch("/busy.xml").await; + assert_eq!(busy.kind, FetchErrorKind::Status(503)); + assert_eq!(busy.retry_after, Some(Duration::from_secs(120))); + let gone = fetch("/gone.xml").await; + assert_eq!(gone.kind, FetchErrorKind::Status(410)); + assert_eq!(gone.retry_after, None); + assert_eq!(fetch("/private.xml").await.kind, FetchErrorKind::Refused); + assert_eq!( + fetch("/missing.xml").await.kind, + FetchErrorKind::Status(404) + ); +} + +#[test] +fn retry_after_accepts_seconds_and_dates() { + let now = chrono::DateTime::parse_from_rfc3339("2026-11-19T10:00:00Z") + .unwrap() + .with_timezone(&chrono::Utc); + assert_eq!(parse_retry_after("90", now), Some(Duration::from_secs(90))); + assert_eq!( + parse_retry_after("Thu, 19 Nov 2026 10:05:00 GMT", now), + Some(Duration::from_secs(300)) + ); + assert_eq!( + parse_retry_after("Thu, 19 Nov 2026 09:00:00 GMT", now), + Some(Duration::ZERO) + ); + assert_eq!( + parse_retry_after("999999999", now), + Some(Duration::from_secs(24 * 60 * 60)) + ); + assert_eq!(parse_retry_after("soon", now), None); +} + +// --------------------------------------------------------------------------- +// Refresh schedule +// --------------------------------------------------------------------------- + +#[test] +fn failures_back_off_exponentially_up_to_a_day() { + assert_eq!(failure_backoff_ms(1, None), BACKOFF_BASE_MS); + assert_eq!(failure_backoff_ms(2, None), 2 * BACKOFF_BASE_MS); + assert_eq!(failure_backoff_ms(3, None), 4 * BACKOFF_BASE_MS); + assert_eq!(failure_backoff_ms(7, None), 64 * BACKOFF_BASE_MS); + assert_eq!(failure_backoff_ms(8, None), BACKOFF_MAX_MS); + assert_eq!(failure_backoff_ms(u32::MAX, None), BACKOFF_MAX_MS); + // A longer Retry-After wins, a shorter one does not shorten the backoff. + assert_eq!( + failure_backoff_ms(1, Some(Duration::from_secs(3 * 60 * 60))), + 3 * 60 * 60 * 1000 + ); + assert_eq!( + failure_backoff_ms(2, Some(Duration::from_secs(1))), + 2 * BACKOFF_BASE_MS + ); + assert_eq!( + failure_backoff_ms(1, Some(Duration::from_secs(u64::MAX))), + BACKOFF_MAX_MS + ); +} + +#[test] +fn regular_checks_are_hourly_with_a_stable_spread() { + let podcast = PodcastId::from_feed_url(&feed_url("https://feeds.example/a.xml")); + let other = PodcastId::from_feed_url(&feed_url("https://feeds.example/b.xml")); + let now = 1_000_000; + let next = next_regular_check_ms(now, 7, &podcast); + assert!( + (now + REFRESH_INTERVAL_MS..now + REFRESH_INTERVAL_MS + 10 * 60 * 1000).contains(&next) + ); + assert_eq!(next, next_regular_check_ms(now, 7, &podcast)); + let spread: std::collections::BTreeSet = (0..20) + .map(|user| next_regular_check_ms(now, user, &podcast)) + .chain([next_regular_check_ms(now, 7, &other)]) + .collect(); + assert!(spread.len() > 10); +} diff --git a/src/podcasts/mod.rs b/src/podcasts/mod.rs index cd4b469..8585c03 100644 --- a/src/podcasts/mod.rs +++ b/src/podcasts/mod.rs @@ -42,19 +42,23 @@ //! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the //! alias set only; they never write a register, progress or a session. -// The RSS refresh job and the podcast UI/HTTP API that call the local -// actions, queries and fetch state land in the following commits of this -// series; until then only sync and the tests use them. Remove this -// allowance with them. +// The podcast UI and its HTTP API, which call the local actions and most +// queries, land in the next commits of this series; until then only sync, +// the refresh job and the tests use the store. Remove this allowance with +// them. #![allow(dead_code, unused_imports)] mod actions; mod apply; +mod fetch; mod net; mod queries; mod refresh; +pub(crate) mod runner; pub(crate) mod schema; +#[cfg(test)] +mod fetch_tests; #[cfg(test)] mod net_tests; #[cfg(test)] @@ -80,6 +84,7 @@ pub(crate) use queries::{ EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter, PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary, }; +pub(crate) use fetch::FeedFetcher; pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome}; /// Typed errors of [`PodcastStore`]; other failures are database errors. diff --git a/src/podcasts/postgres_tests.rs b/src/podcasts/postgres_tests.rs index f626996..755c9c3 100644 --- a/src/podcasts/postgres_tests.rs +++ b/src/podcasts/postgres_tests.rs @@ -1465,3 +1465,256 @@ async fn postgres_podcast_migrations_import_the_relay_state() { .await .unwrap(); } + +// --------------------------------------------------------------------------- +// RSS refresh job +// --------------------------------------------------------------------------- + +fn refresh_item(guid: &str) -> String { + format!( + r#"Episode {guid}{guid} + "# + ) +} + +/// A fetcher that may reach the local test server (loopback is allowed +/// only by the test policy). +fn local_fetcher() -> FeedFetcher { + FeedFetcher::with_http( + net::SafeHttp::with_lookup( + fetch::FEED_HTTP, + net::AddressPolicy::test_allow_loopback(), + std::sync::Arc::new(net::SystemLookup), + ) + .unwrap(), + ) +} + +/// Due groups of the given accounts only (other tests share the database). +async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups { + let due = store + .podcasts_due_for_refresh(now, 100_000) + .await + .unwrap() + .into_iter() + .filter(|(user_id, _, _)| users.contains(user_id)) + .collect(); + runner::group_targets(due) +} + +async fn run_due( + store: &PodcastStore, + fetcher: &FeedFetcher, + users: &[i64], + now: i64, +) -> runner::RunSummary { + let groups = due_for(store, users, now).await; + runner::refresh_groups(store, fetcher, groups, now, 4) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_refresh_job_schedules_backs_off_and_moves_feeds() { + use super::runner::{BACKOFF_BASE_MS, ManualRefreshError, TargetResult}; + use super::test_server::{Route, TestServer}; + + let pool = test_pool().await; + let store = PodcastStore::new(pool.clone()); + let (a, b, c) = (random_user(), random_user(), random_user()); + cleanup(&pool, [a, b, c]).await; + let users = [a, b, c]; + let server = TestServer::start(); + server.set( + "/show.xml", + Route::new(rss(&refresh_item("one")), "application/rss+xml").etag("\"v1\""), + ); + let show = PodcastMeta { + feed_url: FeedUrl::parse(&server.url("/show.xml")).unwrap(), + title: "Local Show".into(), + author: None, + description: None, + link: None, + image_url: None, + }; + let podcast_id = store.subscribe(a, &show).await.unwrap().podcast_id; + assert_eq!( + store.subscribe(b, &show).await.unwrap().podcast_id, + podcast_id + ); + let fetcher = local_fetcher(); + + // First run: one request serves both accounts. + let t0 = super::now_ms(); + let summary = run_due(&store, &fetcher, &users, t0).await; + assert_eq!((summary.targets, summary.feeds), (2, 1), "{summary}"); + assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}"); + assert_eq!(server.requests("/show.xml").len(), 1); + for user in [a, b] { + let state = store.fetch_state(user, &podcast_id).await.unwrap(); + assert_eq!(state.etag.as_deref(), Some("\"v1\"")); + assert_eq!(state.consecutive_failures, 0); + let next = state.next_check_at_ms.unwrap(); + assert!((t0 + runner::REFRESH_INTERVAL_MS..t0 + 70 * 60 * 1000).contains(&next)); + } + let episodes = store + .episodes(a, &EpisodeQuery::new(podcast_id.clone())) + .await + .unwrap(); + assert_eq!(episodes.len(), 1); + let episode_id = episodes[0].episode.episode_id.clone(); + assert_eq!(episodes[0].episode.audio_url, server.url("/audio/one.mp3")); + + // Nothing is due before the interval; afterwards the request is + // conditional and the catalog stays as it is. + assert!(due_for(&store, &users, t0 + 1000).await.is_empty()); + let t1 = t0 + 2 * runner::REFRESH_INTERVAL_MS; + let summary = run_due(&store, &fetcher, &users, t1).await; + assert_eq!(summary.not_modified, 2, "{summary}"); + assert_eq!(server.requests("/show.xml")[1]["if-none-match"], "\"v1\""); + + // Failures back off exponentially and keep the catalog. + server.set("/show.xml", Route::new("boom", "text/plain").status(500)); + let t2 = t1 + 2 * runner::REFRESH_INTERVAL_MS; + let summary = run_due(&store, &fetcher, &users, t2).await; + assert_eq!(summary.failed, 2, "{summary}"); + assert_eq!(summary.failures.len(), 2); + let state = store.fetch_state(a, &podcast_id).await.unwrap(); + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.next_check_at_ms, Some(t2 + BACKOFF_BASE_MS)); + assert!(state.last_error.as_deref().unwrap().contains("500")); + assert_eq!(state.etag.as_deref(), Some("\"v1\""), "validators are kept"); + assert!( + due_for(&store, &users, t2 + BACKOFF_BASE_MS - 1) + .await + .is_empty() + ); + let t3 = t2 + BACKOFF_BASE_MS; + let summary = run_due(&store, &fetcher, &users, t3).await; + assert_eq!(summary.failed, 2, "{summary}"); + let state = store.fetch_state(a, &podcast_id).await.unwrap(); + assert_eq!(state.consecutive_failures, 2); + assert_eq!(state.next_check_at_ms, Some(t3 + 2 * BACKOFF_BASE_MS)); + assert_eq!( + store + .episodes(a, &EpisodeQuery::new(podcast_id.clone())) + .await + .unwrap() + .len(), + 1 + ); + + // Recovery resets the backoff. Accounts with different validators are + // fetched unconditionally. + server.set( + "/show.xml", + Route::new( + rss(&format!("{}{}", refresh_item("two"), refresh_item("one"))), + "application/rss+xml", + ) + .etag("\"v2\""), + ); + store + .record_fetch_success(b, &podcast_id, t3, Some("\"other\""), None, Some(t3)) + .await + .unwrap(); + let t4 = t3 + 2 * BACKOFF_BASE_MS; + let summary = run_due(&store, &fetcher, &users, t4).await; + assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}"); + let last = server.requests("/show.xml").pop().unwrap(); + assert!(!last.contains_key("if-none-match"), "{last:?}"); + let state = store.fetch_state(a, &podcast_id).await.unwrap(); + assert_eq!((state.consecutive_failures, state.last_error), (0, None)); + assert_eq!(state.etag.as_deref(), Some("\"v2\"")); + + // Manual refresh: not more often than every 30 s, only when subscribed. + let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 10_000).await; + assert_eq!( + manual.unwrap_err().downcast_ref::(), + Some(&ManualRefreshError::TooSoon { + retry_after_ms: 20_000 + }) + ); + let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 31_000) + .await + .unwrap(); + assert_eq!(manual, TargetResult::NotModified); + let unknown = runner::refresh_now(&store, &fetcher, c, &podcast_id, t4 + 31_000).await; + assert_eq!( + unknown.unwrap_err().downcast_ref::(), + Some(&ManualRefreshError::UnknownPodcast) + ); + store.unsubscribe(b, &podcast_id).await.unwrap(); + let unsubscribed = runner::refresh_now(&store, &fetcher, b, &podcast_id, t4 + 31_000).await; + assert_eq!( + unsubscribed + .unwrap_err() + .downcast_ref::(), + Some(&ManualRefreshError::NotSubscribed) + ); + let far = t4 + 10 * runner::REFRESH_INTERVAL_MS; + let due = due_for(&store, &users, far).await; + assert_eq!( + due.values().flatten().count(), + 1, + "unsubscribing stops monitoring" + ); + + // A permanent move becomes an alias (one sync operation) and the new + // fetch URL; later refreshes go to the new URL without new operations. + server.set("/show.xml", Route::redirect(301, "/moved.xml")); + server.set( + "/moved.xml", + Route::new(rss(&refresh_item("one")), "application/rss+xml"), + ); + let moved = FeedUrl::parse(&server.url("/moved.xml")).unwrap(); + let result = runner::refresh_now(&store, &fetcher, a, &podcast_id, far) + .await + .unwrap(); + let TargetResult::Updated(outcome) = result else { + panic!("{result:?}"); + }; + assert_eq!(outcome.feed_url.as_ref(), Some(&moved)); + assert_eq!(outcome.ops.len(), 1); + assert_eq!( + store + .podcast(a, &podcast_id) + .await + .unwrap() + .unwrap() + .meta + .feed_url, + moved + ); + let due = due_for(&store, &users, far + 10 * runner::REFRESH_INTERVAL_MS).await; + assert_eq!(due.keys().collect::>(), vec![&moved]); + let summary = run_due( + &store, + &fetcher, + &users, + far + 10 * runner::REFRESH_INTERVAL_MS, + ) + .await; + assert_eq!(summary.updated, 1, "{summary}"); + assert_eq!(server.requests("/moved.xml").len(), 2); + assert_eq!(server.requests("/show.xml").len(), 7); + let TargetResult::Updated(again) = runner::refresh_now( + &store, + &fetcher, + a, + &podcast_id, + far + 20 * runner::REFRESH_INTERVAL_MS, + ) + .await + .unwrap() else { + panic!("expected a document"); + }; + assert!(again.ops.is_empty()); + // The episode keeps its id; its audio now resolves against the new URL + // (same host here). + let after_move = store.episode(a, &episode_id).await.unwrap().unwrap(); + assert_eq!(after_move.episode.audio_url, server.url("/audio/one.mp3")); + + cleanup(&pool, [a, b, c]).await; +} diff --git a/src/podcasts/runner.rs b/src/podcasts/runner.rs new file mode 100644 index 0000000..6f2f837 --- /dev/null +++ b/src/podcasts/runner.rs @@ -0,0 +1,388 @@ +//! Scheduling of feed refreshes: which feeds are due, one request per feed +//! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local +//! fetch state (validators, next check, backoff). + +use std::collections::BTreeMap; +use std::fmt; +use std::time::Duration; + +use anyhow::Result; +use futures_util::StreamExt; +use music_dht::podcast::{FeedUrl, PodcastId}; + +use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators}; +use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome}; + +/// Regular interval between two checks of a healthy feed. +pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000; +/// Spread of the regular interval, so feeds added together drift apart. +const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000; +/// First retry delay after a failure; doubles per consecutive failure. +pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000; +/// Longest delay between two attempts of a failing feed. +pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000; +/// A manual refresh is refused when the feed was checked this recently. +pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000; +/// Podcasts considered per scheduled run. +pub(crate) const RUN_LIMIT: usize = 200; +/// Feed requests in flight at once. +pub(crate) const RUN_CONCURRENCY: usize = 4; + +/// Delay before the next attempt after `consecutive_failures` failures in a +/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a +/// day. A longer server-requested `Retry-After` wins (also capped). +pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option) -> i64 { + let exponent = consecutive_failures.saturating_sub(1).min(16); + let backoff = BACKOFF_BASE_MS + .saturating_mul(1_i64 << exponent) + .min(BACKOFF_MAX_MS); + let requested = retry_after + .map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0); + backoff.max(requested).min(BACKOFF_MAX_MS) +} + +/// Next regular check after a success: the interval plus a stable per +/// account and podcast jitter. +pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 { + let key = format!("{user_id}:{podcast_id}"); + let hash = blake3::hash(key.as_bytes()); + let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes")); + let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0); + now_ms + REFRESH_INTERVAL_MS + jitter +} + +/// One podcast of one account to refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RefreshTarget { + pub(crate) user_id: i64, + pub(crate) podcast_id: PodcastId, +} + +/// Result of refreshing one [`RefreshTarget`]. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum TargetResult { + /// A new document was ingested. + Updated(RefreshOutcome), + /// `304 Not Modified`. + NotModified, + /// The fetch or the ingestion failed; the catalog is unchanged. + Failed { + error: String, + consecutive_failures: u32, + next_check_at_ms: i64, + }, + /// Nothing to do any more (unsubscribed or removed meanwhile). + Skipped(String), +} + +/// Counters of a scheduled run. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct RunSummary { + /// Account podcasts that were due. + pub(crate) targets: usize, + /// Distinct feed URLs requested. + pub(crate) feeds: usize, + pub(crate) updated: usize, + pub(crate) not_modified: usize, + pub(crate) failed: usize, + pub(crate) skipped: usize, + pub(crate) new_episodes: usize, + pub(crate) updated_episodes: usize, + /// `(user_id, podcast_id, error)` of failed targets, for the job log. + pub(crate) failures: Vec<(i64, PodcastId, String)>, +} + +impl RunSummary { + fn add(&mut self, target: &RefreshTarget, result: &TargetResult) { + match result { + TargetResult::Updated(outcome) => { + self.updated += 1; + self.new_episodes += outcome.new_episodes; + self.updated_episodes += outcome.updated_episodes; + } + TargetResult::NotModified => self.not_modified += 1, + TargetResult::Failed { error, .. } => { + self.failed += 1; + self.failures + .push((target.user_id, target.podcast_id.clone(), error.clone())); + } + TargetResult::Skipped(_) => self.skipped += 1, + } + } +} + +impl fmt::Display for RunSummary { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \ + {} not modified, {} failed, {} skipped", + self.targets, + self.feeds, + self.updated, + self.new_episodes, + self.updated_episodes, + self.not_modified, + self.failed, + self.skipped + ) + } +} + +/// Due podcasts grouped by feed URL: accounts subscribed to the same URL +/// share one request. +pub(crate) type RefreshGroups = BTreeMap>; + +/// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`]. +pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups { + let mut groups = RefreshGroups::new(); + for (user_id, podcast_id, feed_url) in due { + groups.entry(feed_url).or_default().push(RefreshTarget { + user_id, + podcast_id, + }); + } + groups +} + +/// Refreshes every podcast due at `now_ms` (at most `limit`), with at most +/// `concurrency` feed requests in flight. Accounts subscribed to the same +/// feed URL share one request. A failing feed never stops the others. +pub(crate) async fn refresh_due( + store: &PodcastStore, + fetcher: &FeedFetcher, + now_ms: i64, + limit: usize, + concurrency: usize, +) -> Result { + let due = store.podcasts_due_for_refresh(now_ms, limit).await?; + refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await +} + +/// Refreshes `groups`, one request per feed URL, at most `concurrency` at +/// a time. +pub(crate) async fn refresh_groups( + store: &PodcastStore, + fetcher: &FeedFetcher, + groups: RefreshGroups, + now_ms: i64, + concurrency: usize, +) -> Result { + let mut summary = RunSummary { + targets: groups.values().map(Vec::len).sum(), + feeds: groups.len(), + ..RunSummary::default() + }; + let mut results = futures_util::stream::iter(groups) + .map(|(feed_url, targets)| async move { + refresh_group(store, fetcher, &feed_url, targets, now_ms).await + }) + .buffer_unordered(concurrency.max(1)); + while let Some(group) = results.next().await { + for (target, result) in group? { + summary.add(&target, &result); + } + } + Ok(summary) +} + +/// Fetches `feed_url` once for all `targets` and records the outcome for +/// each. The request is conditional only when every target holds the same +/// validators (the usual case: one account). +pub(crate) async fn refresh_group( + store: &PodcastStore, + fetcher: &FeedFetcher, + feed_url: &FeedUrl, + targets: Vec, + now_ms: i64, +) -> Result> { + let mut states = Vec::with_capacity(targets.len()); + for target in &targets { + states.push( + store + .fetch_state(target.user_id, &target.podcast_id) + .await?, + ); + } + let validators = states + .iter() + .map(|state| Validators { + etag: state.etag.clone(), + last_modified: state.last_modified.clone(), + }) + .reduce(|a, b| if a == b { a } else { Validators::default() }) + .unwrap_or_default(); + let fetched = fetcher.fetch(feed_url, &validators).await; + let mut out = Vec::with_capacity(targets.len()); + for (target, state) in targets.into_iter().zip(states) { + let result = + record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?; + out.push((target, result)); + } + Ok(out) +} + +/// Ingests (or records the failure of) one fetch for one target. +async fn record_result( + store: &PodcastStore, + target: &RefreshTarget, + fetched: &Result, + previous_failures: u32, + now_ms: i64, +) -> Result { + let (user_id, podcast_id) = (target.user_id, &target.podcast_id); + let failure = |error: String, retry_after: Option| { + let consecutive_failures = previous_failures.saturating_add(1); + let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after); + (error, consecutive_failures, next_check_at_ms) + }; + let (error, consecutive_failures, next_check_at_ms) = match fetched { + Ok(FeedFetch::NotModified) => { + let next = next_regular_check_ms(now_ms, user_id, podcast_id); + return match store + .record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next)) + .await + { + Ok(()) => Ok(TargetResult::NotModified), + Err(error) => skipped_or(error), + }; + } + Ok(FeedFetch::Fetched(document)) => { + match ingest(store, target, document, now_ms).await { + Ok(outcome) => return Ok(TargetResult::Updated(outcome)), + Err(error) if is_gone(&error) => return skipped_or(error), + // A store failure is recorded like a fetch failure, so the + // feed backs off instead of being retried every run. + Err(error) => failure(format!("storing the feed failed: {error:#}"), None), + } + } + Err(error) => failure(error.message.clone(), error.retry_after), + }; + match store + .record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms)) + .await + { + Ok(_) => Ok(TargetResult::Failed { + error, + consecutive_failures, + next_check_at_ms, + }), + Err(error) => skipped_or(error), + } +} + +async fn ingest( + store: &PodcastStore, + target: &RefreshTarget, + document: &FetchedFeed, + now_ms: i64, +) -> Result { + let outcome = store + .apply_feed( + target.user_id, + &FeedRefresh { + podcast_id: &target.podcast_id, + feed: &document.feed, + fetched_from: &document.fetched_from, + moved_permanently: document.moved_permanently, + }, + ) + .await?; + store + .record_fetch_success( + target.user_id, + &target.podcast_id, + now_ms, + document.validators.etag.as_deref(), + document.validators.last_modified.as_deref(), + Some(next_regular_check_ms( + now_ms, + target.user_id, + &target.podcast_id, + )), + ) + .await?; + Ok(outcome) +} + +fn is_gone(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(PodcastStoreError::UnknownPodcast(_)) + ) +} + +/// A podcast removed from the account meanwhile is skipped; any other +/// store error is returned. +fn skipped_or(error: anyhow::Error) -> Result { + if is_gone(&error) { + Ok(TargetResult::Skipped(error.to_string())) + } else { + Err(error) + } +} + +/// Why a manual refresh was not started. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ManualRefreshError { + /// The account does not know the podcast. + UnknownPodcast, + /// The account is not subscribed (unsubscribing stops monitoring). + NotSubscribed, + /// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago. + TooSoon { retry_after_ms: i64 }, +} + +impl fmt::Display for ManualRefreshError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnknownPodcast => f.write_str("unknown podcast"), + Self::NotSubscribed => f.write_str("not subscribed to this podcast"), + Self::TooSoon { retry_after_ms } => write!( + f, + "the feed was checked moments ago; retry in {} s", + (retry_after_ms + 999) / 1000 + ), + } + } +} + +impl std::error::Error for ManualRefreshError {} + +/// Refreshes one subscribed podcast of one account now, ignoring the +/// failure backoff but not more often than every +/// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a +/// [`ManualRefreshError`] when the refresh was not attempted. +pub(crate) async fn refresh_now( + store: &PodcastStore, + fetcher: &FeedFetcher, + user_id: i64, + podcast_id: &PodcastId, + now_ms: i64, +) -> Result { + let feed_url = match store.podcast(user_id, podcast_id).await? { + Some(summary) if summary.subscribed => summary.meta.feed_url, + Some(_) => return Err(ManualRefreshError::NotSubscribed.into()), + None => match store.subscription(user_id, podcast_id).await? { + Some(register) if register.subscribed => register.podcast.feed_url, + Some(_) => return Err(ManualRefreshError::NotSubscribed.into()), + None => return Err(ManualRefreshError::UnknownPodcast.into()), + }, + }; + if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms { + let elapsed = now_ms - checked; + if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) { + return Err(ManualRefreshError::TooSoon { + retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed, + } + .into()); + } + } + let target = RefreshTarget { + user_id, + podcast_id: podcast_id.clone(), + }; + let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?; + Ok(results.pop().map(|(_, result)| result).expect("one target")) +} diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 253c019..5294eef 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -1311,6 +1311,7 @@ async fn run_scheduled_job( && job_name != "lastfm_scrobble" && job_name != "archive_cleanup" && job_name != "artwork_backfill" + && job_name != "podcast_refresh" { tracing::warn!(job = job_name, "Skipping: agent_enabled=false"); return;