Refresh subscribed podcast feeds in a background job

The podcast_refresh job runs every five minutes and refreshes the podcasts
whose next check is due: conditional requests (ETag, Last-Modified), 32 MiB
and 60 s limits, the shared music-dht parser and PodcastStore::apply_feed.
A chain of permanent redirects moves the feed and becomes an alias.
Healthy feeds are checked hourly with a stable spread; failures back off
from 15 minutes up to a day, honouring Retry-After. Accounts subscribed to
the same feed URL share one request, at most four run at once, and one
run at a time holds an advisory lock. runner::refresh_now is the manual
refresh for one subscribed podcast, at most every 30 seconds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent 04ae7d5bd1
commit bf9a6abc3f
9 files changed
+1273 -4

No files matched your search

+1
View File
@@ -6,6 +6,7 @@ pub mod lastfm_popularity;
pub mod lastfm_scrobble;
pub mod metadata_backfill;
pub mod musicbrainz;
pub mod podcast_refresh;
use std::path::{Component, Path, PathBuf};
+73
View File
@@ -0,0 +1,73 @@
//! Background refresh of subscribed podcast feeds.
//!
//! Every run takes the podcasts whose next check is due (see
//! [`crate::podcasts::runner`]): healthy feeds are checked about hourly with
//! conditional requests, failing ones back off exponentially up to a day.
//! One request per feed URL serves every account subscribed to it, a
//! bounded number of requests run concurrently, and a failing feed never
//! blocks the others or clears the stored catalog.
use crate::podcasts::{FeedFetcher, PodcastStore, now_ms, runner};
use crate::scheduler::{Job, JobContext, JobLog};
pub struct PodcastRefreshJob;
/// Session-level advisory lock: one run at a time across processes.
const ADVISORY_LOCK_ID: i64 = 0x504f_4443_5246_5348; // "PODCRFSH"
/// Failures listed individually in the run log.
const LOGGED_FAILURES: usize = 20;
#[async_trait::async_trait]
impl Job for PodcastRefreshJob {
fn name(&self) -> &'static str {
"podcast_refresh"
}
fn description(&self) -> &'static str {
"Check subscribed podcast feeds for new episodes (metadata only, no audio)"
}
fn default_cron(&self) -> &'static str {
// Every five minutes; each feed is checked only when due.
"0 */5 * * * *"
}
async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> {
let mut conn = ctx.pool.acquire().await?;
// A session lock must never return to the pool, including on
// cancellation; the store below uses other pool connections.
conn.close_on_drop();
let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)")
.bind(ADVISORY_LOCK_ID)
.fetch_one(&mut *conn)
.await?;
if !locked {
log.info("Podcast refresh is already running; skipping this run");
return Ok(());
}
let store = PodcastStore::new(ctx.pool.clone());
let fetcher = FeedFetcher::new()?;
let result = runner::refresh_due(
&store,
&fetcher,
now_ms(),
runner::RUN_LIMIT,
runner::RUN_CONCURRENCY,
)
.await;
conn.close().await?;
let summary = result?;
for (user_id, podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
log.warn(&format!("user {user_id}, podcast {podcast_id}: {error}"));
}
if summary.failures.len() > LOGGED_FAILURES {
log.warn(&format!(
"... and {} more failed feeds",
summary.failures.len() - LOGGED_FAILURES
));
}
log.info(&format!("Podcast refresh: {summary}"));
Ok(())
}
}
+1
View File
@@ -64,6 +64,7 @@ fn build_registry() -> Arc<JobRegistry> {
registry.register(jobs::metadata_backfill::MetadataBackfillJob);
registry.register(jobs::lastfm_popularity::LastfmPopularityJob);
registry.register(jobs::lastfm_scrobble::LastfmScrobbleJob);
registry.register(jobs::podcast_refresh::PodcastRefreshJob);
Arc::new(registry)
}
+303
View File
@@ -0,0 +1,303 @@
//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`].
use std::fmt;
use std::time::Duration;
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH,
LAST_MODIFIED, RETRY_AFTER,
};
use reqwest::{StatusCode, Url};
use music_dht::podcast::FeedUrl;
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
/// Hard cap on a feed document (large catalogs are a few MiB).
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
/// Whole fetch, from connect to the last body byte.
pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60);
/// Redirect hops followed for a feed.
pub(crate) const MAX_FEED_REDIRECTS: usize = 5;
/// `Retry-After` values above this are clamped.
const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
text/xml;q=0.9, */*;q=0.5";
/// HTTP client settings of the feed fetcher.
pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig {
connect_timeout: Duration::from_secs(10),
read_timeout: Duration::from_secs(30),
total_timeout: Some(FEED_FETCH_TIMEOUT),
max_redirects: MAX_FEED_REDIRECTS,
};
/// Validators of the previous successful fetch.
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
pub(crate) struct Validators {
pub(crate) etag: Option<String>,
pub(crate) last_modified: Option<String>,
}
/// A successfully fetched and parsed document.
#[derive(Debug, Clone)]
pub(crate) struct FetchedFeed {
pub(crate) feed: ParsedFeed,
/// Where the feed now lives: the final URL after a chain of permanent
/// redirects, otherwise the requested URL.
pub(crate) fetched_from: FeedUrl,
/// Whether the feed moved: at least one redirect, all permanent (see
/// [`super::net::Followed::moved_permanently`]), to a valid feed URL.
pub(crate) moved_permanently: bool,
/// New validators for the next conditional request.
pub(crate) validators: Validators,
}
/// Outcome of [`FeedFetcher::fetch`].
#[derive(Debug, Clone)]
pub(crate) enum FeedFetch {
/// `304 Not Modified`: the stored catalog is current.
NotModified,
/// A new document.
Fetched(Box<FetchedFeed>),
}
/// Kind of a failed fetch.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum FetchErrorKind {
/// Refused by the SSRF guard (blocked address, scheme, credentials).
Refused,
/// Non-success HTTP status.
Status(u16),
/// Body over [`MAX_FEED_BYTES`].
TooLarge,
/// The document is not a feed.
Parse,
/// Timed out.
Timeout,
/// DNS, connect, TLS or a dropped connection.
Transport,
}
/// A failed fetch. The stored catalog is never touched on failure.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct FetchError {
pub(crate) kind: FetchErrorKind,
pub(crate) message: String,
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
pub(crate) retry_after: Option<Duration>,
}
impl FetchError {
fn new(kind: FetchErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
retry_after: None,
}
}
}
impl fmt::Display for FetchError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for FetchError {}
impl From<GuardError> for FetchError {
fn from(error: GuardError) -> Self {
let kind = match &error {
GuardError::UnsupportedScheme(_)
| GuardError::Credentials
| GuardError::MissingHost
| GuardError::Blocked(_) => FetchErrorKind::Refused,
_ if error.is_timeout() => FetchErrorKind::Timeout,
_ => FetchErrorKind::Transport,
};
Self::new(kind, error.to_string())
}
}
/// Parses `Retry-After` (delta seconds or an HTTP date).
pub(crate) fn parse_retry_after(
value: &str,
now: chrono::DateTime<chrono::Utc>,
) -> Option<Duration> {
let value = value.trim();
let delay = if let Ok(seconds) = value.parse::<u64>() {
Duration::from_secs(seconds)
} else {
let at = chrono::DateTime::parse_from_rfc2822(value).ok()?;
(at.with_timezone(&chrono::Utc) - now)
.to_std()
.unwrap_or_default()
};
Some(delay.min(MAX_RETRY_AFTER))
}
/// Downloads feeds with the SSRF guard, conditional requests, redirects
/// and hard size/time limits; parses them with the shared parser.
#[derive(Debug, Clone)]
pub(crate) struct FeedFetcher {
http: SafeHttp,
max_bytes: usize,
}
impl FeedFetcher {
/// The production fetcher (public addresses only).
pub(crate) fn new() -> anyhow::Result<Self> {
Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?))
}
pub(crate) fn with_http(http: SafeHttp) -> Self {
Self {
http,
max_bytes: MAX_FEED_BYTES,
}
}
#[cfg(test)]
pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self {
self.max_bytes = max_bytes;
self
}
/// Fetches `url`, conditionally when `validators` are known.
pub(crate) async fn fetch(
&self,
url: &FeedUrl,
validators: &Validators,
) -> Result<FeedFetch, FetchError> {
match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await {
Ok(result) => result,
Err(_) => Err(FetchError::new(
FetchErrorKind::Timeout,
format!(
"feed fetch timed out after {} s",
FEED_FETCH_TIMEOUT.as_secs()
),
)),
}
}
async fn fetch_inner(
&self,
feed_url: &FeedUrl,
validators: &Validators,
) -> Result<FeedFetch, FetchError> {
let url = Url::parse(feed_url.as_str())
.map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?;
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT));
// Compressed bodies are not negotiated (no decoder is enabled), so
// the size cap applies to what is parsed.
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
if let Some(etag) = validators
.etag
.as_deref()
.and_then(|value| HeaderValue::from_str(value).ok())
{
headers.insert(IF_NONE_MATCH, etag);
}
if let Some(date) = validators
.last_modified
.as_deref()
.and_then(|value| HeaderValue::from_str(value).ok())
{
headers.insert(IF_MODIFIED_SINCE, date);
}
let followed = self.http.get(url, &headers).await?;
let mut moved_permanently = followed.moved_permanently();
let final_url = followed.final_url;
let response = followed.response;
let status = response.status();
if status == StatusCode::NOT_MODIFIED {
return Ok(FeedFetch::NotModified);
}
if !status.is_success() {
let mut error = FetchError::new(
FetchErrorKind::Status(status.as_u16()),
format!("feed request failed: HTTP {status}"),
);
if matches!(status.as_u16(), 429 | 503) {
error.retry_after = response
.headers()
.get(RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(|value| parse_retry_after(value, chrono::Utc::now()));
}
return Err(error);
}
let header = |name| {
response
.headers()
.get(name)
.and_then(|value: &HeaderValue| value.to_str().ok())
.map(str::to_string)
};
let validators = Validators {
etag: header(ETAG),
last_modified: header(LAST_MODIFIED),
};
let body = read_capped(response, self.max_bytes).await?;
// Relative enclosure URLs resolve against where the feed really is.
let base = final_url.to_string();
let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base)))
.await
.map_err(|error| {
FetchError::new(
FetchErrorKind::Parse,
format!("feed parser task failed: {error}"),
)
})?
.map_err(|error| {
FetchError::new(
FetchErrorKind::Parse,
format!("not a podcast feed: {error}"),
)
})?;
// Only a permanent move changes the feed URL; a target that is not a
// valid feed URL (over-long, ...) is not learned.
let moved_to = moved_permanently
.then(|| FeedUrl::parse(final_url.as_str()).ok())
.flatten();
moved_permanently = moved_to.is_some();
let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone());
Ok(FeedFetch::Fetched(Box::new(FetchedFeed {
feed,
fetched_from,
moved_permanently,
validators,
})))
}
}
async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result<Vec<u8>, FetchError> {
let too_large = |detail: String| {
FetchError::new(
FetchErrorKind::TooLarge,
format!("feed is too large ({detail}, limit {cap} bytes)"),
)
};
if let Some(len) = response.content_length()
&& len > cap as u64
{
return Err(too_large(format!("{len} bytes")));
}
let mut body = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(|error| FetchError::from(GuardError::from(error)))?
{
if body.len() + chunk.len() > cap {
return Err(too_large("body exceeds the limit".to_string()));
}
body.extend_from_slice(&chunk);
}
Ok(body)
}
+244
View File
@@ -0,0 +1,244 @@
//! Feed fetching (conditional requests, redirects, limits) and the refresh
//! schedule, against the local test server.
use std::time::Duration;
use music_dht::podcast::{FeedUrl, PodcastId};
use super::fetch::{FeedFetch, FeedFetcher, FetchErrorKind, Validators, parse_retry_after};
use super::runner::{
BACKOFF_BASE_MS, BACKOFF_MAX_MS, REFRESH_INTERVAL_MS, failure_backoff_ms, next_regular_check_ms,
};
use std::sync::Arc;
use super::test_server::{Route, ScriptedLookup, TestServer, test_http};
// ---------------------------------------------------------------------------
// Feed fetching
// ---------------------------------------------------------------------------
fn rss(items: &str) -> String {
format!(
r#"<?xml version="1.0"?><rss version="2.0"><channel><title>Show</title>{items}</channel></rss>"#
)
}
fn item(guid: &str) -> String {
format!(
r#"<item><title>{guid}</title><guid>{guid}</guid><enclosure url="/audio/{guid}.mp3" type="audio/mpeg"/></item>"#
)
}
fn fetcher(lookup: Arc<ScriptedLookup>) -> FeedFetcher {
FeedFetcher::with_http(test_http(lookup))
}
fn feed_url(text: &str) -> FeedUrl {
FeedUrl::parse(text).unwrap()
}
#[tokio::test]
async fn feeds_are_fetched_conditionally() {
let server = TestServer::start();
server.set(
"/feed.xml",
Route::new(rss(&item("a")), "application/rss+xml")
.etag("\"v1\"")
.last_modified("Wed, 21 Oct 2026 07:28:00 GMT"),
);
let fetcher = fetcher(ScriptedLookup::with(&[]));
let feed = feed_url(&server.url("/feed.xml"));
let FeedFetch::Fetched(first) = fetcher.fetch(&feed, &Validators::default()).await.unwrap()
else {
panic!("expected a document");
};
assert_eq!(first.feed.episodes.len(), 1);
assert_eq!(
first.feed.episodes[0].audio_url,
server.url("/audio/a.mp3"),
"relative enclosures resolve against the feed URL"
);
assert_eq!(first.fetched_from, feed);
assert!(!first.moved_permanently);
assert_eq!(first.validators.etag.as_deref(), Some("\"v1\""));
let second = fetcher.fetch(&feed, &first.validators).await.unwrap();
assert!(matches!(second, FeedFetch::NotModified));
// Only Last-Modified known.
let by_date = Validators {
etag: None,
last_modified: first.validators.last_modified.clone(),
};
assert!(matches!(
fetcher.fetch(&feed, &by_date).await.unwrap(),
FeedFetch::NotModified
));
let requests = server.requests("/feed.xml");
assert_eq!(requests.len(), 3);
assert!(!requests[0].contains_key("if-none-match"));
assert_eq!(requests[1]["if-none-match"], "\"v1\"");
assert_eq!(
requests[1]["if-modified-since"],
"Wed, 21 Oct 2026 07:28:00 GMT"
);
assert!(!requests[2].contains_key("if-none-match"));
assert_eq!(requests[2]["accept-encoding"], "identity");
}
#[tokio::test]
async fn only_permanent_redirect_chains_move_a_feed() {
let server = TestServer::start();
server.set(
"/new.xml",
Route::new(rss(&item("a")), "application/rss+xml"),
);
server.set("/old.xml", Route::redirect(301, "/new.xml"));
server.set("/moved.xml", Route::redirect(308, "/old.xml"));
server.set("/temp.xml", Route::redirect(302, "/new.xml"));
server.set("/mixed.xml", Route::redirect(307, "/old.xml"));
let fetcher = fetcher(ScriptedLookup::with(&[]));
for (path, moved) in [
("/old.xml", true),
("/moved.xml", true),
("/temp.xml", false),
("/mixed.xml", false),
] {
let requested = feed_url(&server.url(path));
let FeedFetch::Fetched(document) = fetcher
.fetch(&requested, &Validators::default())
.await
.unwrap()
else {
panic!("{path}");
};
assert_eq!(document.moved_permanently, moved, "{path}");
let expected = if moved {
feed_url(&server.url("/new.xml"))
} else {
requested
};
assert_eq!(document.fetched_from, expected, "{path}");
assert_eq!(
document.feed.episodes[0].audio_url,
server.url("/audio/a.mp3")
);
}
}
#[tokio::test]
async fn feed_fetch_failures_are_typed() {
let server = TestServer::start();
let big = rss(&item("x").repeat(200));
server.set("/big.xml", Route::new(big.clone(), "application/rss+xml"));
server.set(
"/big-chunked.xml",
Route::new(big.clone(), "application/rss+xml").no_length(),
);
server.set(
"/page.html",
Route::new("<html><body>hi</body></html>", "text/html"),
);
server.set(
"/busy.xml",
Route::new("busy", "text/plain")
.status(503)
.header("Retry-After", "120"),
);
server.set("/gone.xml", Route::new("", "text/plain").status(410));
server.set(
"/private.xml",
Route::redirect(301, "http://192.168.0.1/feed"),
);
let fetcher = fetcher(ScriptedLookup::with(&[])).with_max_bytes(big.len() - 1);
let fetch = |path: &str| {
let feed = feed_url(&server.url(path));
let fetcher = fetcher.clone();
async move {
fetcher
.fetch(&feed, &Validators::default())
.await
.unwrap_err()
}
};
assert_eq!(fetch("/big.xml").await.kind, FetchErrorKind::TooLarge);
assert_eq!(
fetch("/big-chunked.xml").await.kind,
FetchErrorKind::TooLarge
);
assert_eq!(fetch("/page.html").await.kind, FetchErrorKind::Parse);
let busy = fetch("/busy.xml").await;
assert_eq!(busy.kind, FetchErrorKind::Status(503));
assert_eq!(busy.retry_after, Some(Duration::from_secs(120)));
let gone = fetch("/gone.xml").await;
assert_eq!(gone.kind, FetchErrorKind::Status(410));
assert_eq!(gone.retry_after, None);
assert_eq!(fetch("/private.xml").await.kind, FetchErrorKind::Refused);
assert_eq!(
fetch("/missing.xml").await.kind,
FetchErrorKind::Status(404)
);
}
#[test]
fn retry_after_accepts_seconds_and_dates() {
let now = chrono::DateTime::parse_from_rfc3339("2026-11-19T10:00:00Z")
.unwrap()
.with_timezone(&chrono::Utc);
assert_eq!(parse_retry_after("90", now), Some(Duration::from_secs(90)));
assert_eq!(
parse_retry_after("Thu, 19 Nov 2026 10:05:00 GMT", now),
Some(Duration::from_secs(300))
);
assert_eq!(
parse_retry_after("Thu, 19 Nov 2026 09:00:00 GMT", now),
Some(Duration::ZERO)
);
assert_eq!(
parse_retry_after("999999999", now),
Some(Duration::from_secs(24 * 60 * 60))
);
assert_eq!(parse_retry_after("soon", now), None);
}
// ---------------------------------------------------------------------------
// Refresh schedule
// ---------------------------------------------------------------------------
#[test]
fn failures_back_off_exponentially_up_to_a_day() {
assert_eq!(failure_backoff_ms(1, None), BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(2, None), 2 * BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(3, None), 4 * BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(7, None), 64 * BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(8, None), BACKOFF_MAX_MS);
assert_eq!(failure_backoff_ms(u32::MAX, None), BACKOFF_MAX_MS);
// A longer Retry-After wins, a shorter one does not shorten the backoff.
assert_eq!(
failure_backoff_ms(1, Some(Duration::from_secs(3 * 60 * 60))),
3 * 60 * 60 * 1000
);
assert_eq!(
failure_backoff_ms(2, Some(Duration::from_secs(1))),
2 * BACKOFF_BASE_MS
);
assert_eq!(
failure_backoff_ms(1, Some(Duration::from_secs(u64::MAX))),
BACKOFF_MAX_MS
);
}
#[test]
fn regular_checks_are_hourly_with_a_stable_spread() {
let podcast = PodcastId::from_feed_url(&feed_url("https://feeds.example/a.xml"));
let other = PodcastId::from_feed_url(&feed_url("https://feeds.example/b.xml"));
let now = 1_000_000;
let next = next_regular_check_ms(now, 7, &podcast);
assert!(
(now + REFRESH_INTERVAL_MS..now + REFRESH_INTERVAL_MS + 10 * 60 * 1000).contains(&next)
);
assert_eq!(next, next_regular_check_ms(now, 7, &podcast));
let spread: std::collections::BTreeSet<i64> = (0..20)
.map(|user| next_regular_check_ms(now, user, &podcast))
.chain([next_regular_check_ms(now, 7, &other)])
.collect();
assert!(spread.len() > 10);
}
+9 -4
View File
@@ -42,19 +42,23 @@
//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the
//! alias set only; they never write a register, progress or a session.
// The RSS refresh job and the podcast UI/HTTP API that call the local
// actions, queries and fetch state land in the following commits of this
// series; until then only sync and the tests use them. Remove this
// allowance with them.
// The podcast UI and its HTTP API, which call the local actions and most
// queries, land in the next commits of this series; until then only sync,
// the refresh job and the tests use the store. Remove this allowance with
// them.
#![allow(dead_code, unused_imports)]
mod actions;
mod apply;
mod fetch;
mod net;
mod queries;
mod refresh;
pub(crate) mod runner;
pub(crate) mod schema;
#[cfg(test)]
mod fetch_tests;
#[cfg(test)]
mod net_tests;
#[cfg(test)]
@@ -80,6 +84,7 @@ pub(crate) use queries::{
EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter,
PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary,
};
pub(crate) use fetch::FeedFetcher;
pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome};
/// Typed errors of [`PodcastStore`]; other failures are database errors.
+253
View File
@@ -1465,3 +1465,256 @@ async fn postgres_podcast_migrations_import_the_relay_state() {
.await
.unwrap();
}
// ---------------------------------------------------------------------------
// RSS refresh job
// ---------------------------------------------------------------------------
fn refresh_item(guid: &str) -> String {
format!(
r#"<item><title>Episode {guid}</title><guid>{guid}</guid>
<enclosure url="/audio/{guid}.mp3" type="audio/mpeg" length="1000"/></item>"#
)
}
/// A fetcher that may reach the local test server (loopback is allowed
/// only by the test policy).
fn local_fetcher() -> FeedFetcher {
FeedFetcher::with_http(
net::SafeHttp::with_lookup(
fetch::FEED_HTTP,
net::AddressPolicy::test_allow_loopback(),
std::sync::Arc::new(net::SystemLookup),
)
.unwrap(),
)
}
/// Due groups of the given accounts only (other tests share the database).
async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups {
let due = store
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
.filter(|(user_id, _, _)| users.contains(user_id))
.collect();
runner::group_targets(due)
}
async fn run_due(
store: &PodcastStore,
fetcher: &FeedFetcher,
users: &[i64],
now: i64,
) -> runner::RunSummary {
let groups = due_for(store, users, now).await;
runner::refresh_groups(store, fetcher, groups, now, 4)
.await
.unwrap()
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_refresh_job_schedules_backs_off_and_moves_feeds() {
use super::runner::{BACKOFF_BASE_MS, ManualRefreshError, TargetResult};
use super::test_server::{Route, TestServer};
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let (a, b, c) = (random_user(), random_user(), random_user());
cleanup(&pool, [a, b, c]).await;
let users = [a, b, c];
let server = TestServer::start();
server.set(
"/show.xml",
Route::new(rss(&refresh_item("one")), "application/rss+xml").etag("\"v1\""),
);
let show = PodcastMeta {
feed_url: FeedUrl::parse(&server.url("/show.xml")).unwrap(),
title: "Local Show".into(),
author: None,
description: None,
link: None,
image_url: None,
};
let podcast_id = store.subscribe(a, &show).await.unwrap().podcast_id;
assert_eq!(
store.subscribe(b, &show).await.unwrap().podcast_id,
podcast_id
);
let fetcher = local_fetcher();
// First run: one request serves both accounts.
let t0 = super::now_ms();
let summary = run_due(&store, &fetcher, &users, t0).await;
assert_eq!((summary.targets, summary.feeds), (2, 1), "{summary}");
assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}");
assert_eq!(server.requests("/show.xml").len(), 1);
for user in [a, b] {
let state = store.fetch_state(user, &podcast_id).await.unwrap();
assert_eq!(state.etag.as_deref(), Some("\"v1\""));
assert_eq!(state.consecutive_failures, 0);
let next = state.next_check_at_ms.unwrap();
assert!((t0 + runner::REFRESH_INTERVAL_MS..t0 + 70 * 60 * 1000).contains(&next));
}
let episodes = store
.episodes(a, &EpisodeQuery::new(podcast_id.clone()))
.await
.unwrap();
assert_eq!(episodes.len(), 1);
let episode_id = episodes[0].episode.episode_id.clone();
assert_eq!(episodes[0].episode.audio_url, server.url("/audio/one.mp3"));
// Nothing is due before the interval; afterwards the request is
// conditional and the catalog stays as it is.
assert!(due_for(&store, &users, t0 + 1000).await.is_empty());
let t1 = t0 + 2 * runner::REFRESH_INTERVAL_MS;
let summary = run_due(&store, &fetcher, &users, t1).await;
assert_eq!(summary.not_modified, 2, "{summary}");
assert_eq!(server.requests("/show.xml")[1]["if-none-match"], "\"v1\"");
// Failures back off exponentially and keep the catalog.
server.set("/show.xml", Route::new("boom", "text/plain").status(500));
let t2 = t1 + 2 * runner::REFRESH_INTERVAL_MS;
let summary = run_due(&store, &fetcher, &users, t2).await;
assert_eq!(summary.failed, 2, "{summary}");
assert_eq!(summary.failures.len(), 2);
let state = store.fetch_state(a, &podcast_id).await.unwrap();
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_check_at_ms, Some(t2 + BACKOFF_BASE_MS));
assert!(state.last_error.as_deref().unwrap().contains("500"));
assert_eq!(state.etag.as_deref(), Some("\"v1\""), "validators are kept");
assert!(
due_for(&store, &users, t2 + BACKOFF_BASE_MS - 1)
.await
.is_empty()
);
let t3 = t2 + BACKOFF_BASE_MS;
let summary = run_due(&store, &fetcher, &users, t3).await;
assert_eq!(summary.failed, 2, "{summary}");
let state = store.fetch_state(a, &podcast_id).await.unwrap();
assert_eq!(state.consecutive_failures, 2);
assert_eq!(state.next_check_at_ms, Some(t3 + 2 * BACKOFF_BASE_MS));
assert_eq!(
store
.episodes(a, &EpisodeQuery::new(podcast_id.clone()))
.await
.unwrap()
.len(),
1
);
// Recovery resets the backoff. Accounts with different validators are
// fetched unconditionally.
server.set(
"/show.xml",
Route::new(
rss(&format!("{}{}", refresh_item("two"), refresh_item("one"))),
"application/rss+xml",
)
.etag("\"v2\""),
);
store
.record_fetch_success(b, &podcast_id, t3, Some("\"other\""), None, Some(t3))
.await
.unwrap();
let t4 = t3 + 2 * BACKOFF_BASE_MS;
let summary = run_due(&store, &fetcher, &users, t4).await;
assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}");
let last = server.requests("/show.xml").pop().unwrap();
assert!(!last.contains_key("if-none-match"), "{last:?}");
let state = store.fetch_state(a, &podcast_id).await.unwrap();
assert_eq!((state.consecutive_failures, state.last_error), (0, None));
assert_eq!(state.etag.as_deref(), Some("\"v2\""));
// Manual refresh: not more often than every 30 s, only when subscribed.
let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 10_000).await;
assert_eq!(
manual.unwrap_err().downcast_ref::<ManualRefreshError>(),
Some(&ManualRefreshError::TooSoon {
retry_after_ms: 20_000
})
);
let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 31_000)
.await
.unwrap();
assert_eq!(manual, TargetResult::NotModified);
let unknown = runner::refresh_now(&store, &fetcher, c, &podcast_id, t4 + 31_000).await;
assert_eq!(
unknown.unwrap_err().downcast_ref::<ManualRefreshError>(),
Some(&ManualRefreshError::UnknownPodcast)
);
store.unsubscribe(b, &podcast_id).await.unwrap();
let unsubscribed = runner::refresh_now(&store, &fetcher, b, &podcast_id, t4 + 31_000).await;
assert_eq!(
unsubscribed
.unwrap_err()
.downcast_ref::<ManualRefreshError>(),
Some(&ManualRefreshError::NotSubscribed)
);
let far = t4 + 10 * runner::REFRESH_INTERVAL_MS;
let due = due_for(&store, &users, far).await;
assert_eq!(
due.values().flatten().count(),
1,
"unsubscribing stops monitoring"
);
// A permanent move becomes an alias (one sync operation) and the new
// fetch URL; later refreshes go to the new URL without new operations.
server.set("/show.xml", Route::redirect(301, "/moved.xml"));
server.set(
"/moved.xml",
Route::new(rss(&refresh_item("one")), "application/rss+xml"),
);
let moved = FeedUrl::parse(&server.url("/moved.xml")).unwrap();
let result = runner::refresh_now(&store, &fetcher, a, &podcast_id, far)
.await
.unwrap();
let TargetResult::Updated(outcome) = result else {
panic!("{result:?}");
};
assert_eq!(outcome.feed_url.as_ref(), Some(&moved));
assert_eq!(outcome.ops.len(), 1);
assert_eq!(
store
.podcast(a, &podcast_id)
.await
.unwrap()
.unwrap()
.meta
.feed_url,
moved
);
let due = due_for(&store, &users, far + 10 * runner::REFRESH_INTERVAL_MS).await;
assert_eq!(due.keys().collect::<Vec<_>>(), vec![&moved]);
let summary = run_due(
&store,
&fetcher,
&users,
far + 10 * runner::REFRESH_INTERVAL_MS,
)
.await;
assert_eq!(summary.updated, 1, "{summary}");
assert_eq!(server.requests("/moved.xml").len(), 2);
assert_eq!(server.requests("/show.xml").len(), 7);
let TargetResult::Updated(again) = runner::refresh_now(
&store,
&fetcher,
a,
&podcast_id,
far + 20 * runner::REFRESH_INTERVAL_MS,
)
.await
.unwrap() else {
panic!("expected a document");
};
assert!(again.ops.is_empty());
// The episode keeps its id; its audio now resolves against the new URL
// (same host here).
let after_move = store.episode(a, &episode_id).await.unwrap().unwrap();
assert_eq!(after_move.episode.audio_url, server.url("/audio/one.mp3"));
cleanup(&pool, [a, b, c]).await;
}
+388
View File
@@ -0,0 +1,388 @@
//! Scheduling of feed refreshes: which feeds are due, one request per feed
//! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local
//! fetch state (validators, next check, backoff).
use std::collections::BTreeMap;
use std::fmt;
use std::time::Duration;
use anyhow::Result;
use futures_util::StreamExt;
use music_dht::podcast::{FeedUrl, PodcastId};
use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators};
use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome};
/// Regular interval between two checks of a healthy feed.
pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000;
/// Spread of the regular interval, so feeds added together drift apart.
const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000;
/// First retry delay after a failure; doubles per consecutive failure.
pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
/// Longest delay between two attempts of a failing feed.
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
/// A manual refresh is refused when the feed was checked this recently.
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
/// Podcasts considered per scheduled run.
pub(crate) const RUN_LIMIT: usize = 200;
/// Feed requests in flight at once.
pub(crate) const RUN_CONCURRENCY: usize = 4;
/// Delay before the next attempt after `consecutive_failures` failures in a
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
/// day. A longer server-requested `Retry-After` wins (also capped).
pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option<Duration>) -> i64 {
let exponent = consecutive_failures.saturating_sub(1).min(16);
let backoff = BACKOFF_BASE_MS
.saturating_mul(1_i64 << exponent)
.min(BACKOFF_MAX_MS);
let requested = retry_after
.map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX))
.unwrap_or(0);
backoff.max(requested).min(BACKOFF_MAX_MS)
}
/// Next regular check after a success: the interval plus a stable per
/// account and podcast jitter.
pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 {
let key = format!("{user_id}:{podcast_id}");
let hash = blake3::hash(key.as_bytes());
let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes"));
let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0);
now_ms + REFRESH_INTERVAL_MS + jitter
}
/// One podcast of one account to refresh.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RefreshTarget {
pub(crate) user_id: i64,
pub(crate) podcast_id: PodcastId,
}
/// Result of refreshing one [`RefreshTarget`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum TargetResult {
/// A new document was ingested.
Updated(RefreshOutcome),
/// `304 Not Modified`.
NotModified,
/// The fetch or the ingestion failed; the catalog is unchanged.
Failed {
error: String,
consecutive_failures: u32,
next_check_at_ms: i64,
},
/// Nothing to do any more (unsubscribed or removed meanwhile).
Skipped(String),
}
/// Counters of a scheduled run.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct RunSummary {
/// Account podcasts that were due.
pub(crate) targets: usize,
/// Distinct feed URLs requested.
pub(crate) feeds: usize,
pub(crate) updated: usize,
pub(crate) not_modified: usize,
pub(crate) failed: usize,
pub(crate) skipped: usize,
pub(crate) new_episodes: usize,
pub(crate) updated_episodes: usize,
/// `(user_id, podcast_id, error)` of failed targets, for the job log.
pub(crate) failures: Vec<(i64, PodcastId, String)>,
}
impl RunSummary {
fn add(&mut self, target: &RefreshTarget, result: &TargetResult) {
match result {
TargetResult::Updated(outcome) => {
self.updated += 1;
self.new_episodes += outcome.new_episodes;
self.updated_episodes += outcome.updated_episodes;
}
TargetResult::NotModified => self.not_modified += 1,
TargetResult::Failed { error, .. } => {
self.failed += 1;
self.failures
.push((target.user_id, target.podcast_id.clone(), error.clone()));
}
TargetResult::Skipped(_) => self.skipped += 1,
}
}
}
impl fmt::Display for RunSummary {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \
{} not modified, {} failed, {} skipped",
self.targets,
self.feeds,
self.updated,
self.new_episodes,
self.updated_episodes,
self.not_modified,
self.failed,
self.skipped
)
}
}
/// Due podcasts grouped by feed URL: accounts subscribed to the same URL
/// share one request.
pub(crate) type RefreshGroups = BTreeMap<FeedUrl, Vec<RefreshTarget>>;
/// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`].
pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups {
let mut groups = RefreshGroups::new();
for (user_id, podcast_id, feed_url) in due {
groups.entry(feed_url).or_default().push(RefreshTarget {
user_id,
podcast_id,
});
}
groups
}
/// Refreshes every podcast due at `now_ms` (at most `limit`), with at most
/// `concurrency` feed requests in flight. Accounts subscribed to the same
/// feed URL share one request. A failing feed never stops the others.
pub(crate) async fn refresh_due(
store: &PodcastStore,
fetcher: &FeedFetcher,
now_ms: i64,
limit: usize,
concurrency: usize,
) -> Result<RunSummary> {
let due = store.podcasts_due_for_refresh(now_ms, limit).await?;
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
}
/// Refreshes `groups`, one request per feed URL, at most `concurrency` at
/// a time.
pub(crate) async fn refresh_groups(
store: &PodcastStore,
fetcher: &FeedFetcher,
groups: RefreshGroups,
now_ms: i64,
concurrency: usize,
) -> Result<RunSummary> {
let mut summary = RunSummary {
targets: groups.values().map(Vec::len).sum(),
feeds: groups.len(),
..RunSummary::default()
};
let mut results = futures_util::stream::iter(groups)
.map(|(feed_url, targets)| async move {
refresh_group(store, fetcher, &feed_url, targets, now_ms).await
})
.buffer_unordered(concurrency.max(1));
while let Some(group) = results.next().await {
for (target, result) in group? {
summary.add(&target, &result);
}
}
Ok(summary)
}
/// Fetches `feed_url` once for all `targets` and records the outcome for
/// each. The request is conditional only when every target holds the same
/// validators (the usual case: one account).
pub(crate) async fn refresh_group(
store: &PodcastStore,
fetcher: &FeedFetcher,
feed_url: &FeedUrl,
targets: Vec<RefreshTarget>,
now_ms: i64,
) -> Result<Vec<(RefreshTarget, TargetResult)>> {
let mut states = Vec::with_capacity(targets.len());
for target in &targets {
states.push(
store
.fetch_state(target.user_id, &target.podcast_id)
.await?,
);
}
let validators = states
.iter()
.map(|state| Validators {
etag: state.etag.clone(),
last_modified: state.last_modified.clone(),
})
.reduce(|a, b| if a == b { a } else { Validators::default() })
.unwrap_or_default();
let fetched = fetcher.fetch(feed_url, &validators).await;
let mut out = Vec::with_capacity(targets.len());
for (target, state) in targets.into_iter().zip(states) {
let result =
record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?;
out.push((target, result));
}
Ok(out)
}
/// Ingests (or records the failure of) one fetch for one target.
async fn record_result(
store: &PodcastStore,
target: &RefreshTarget,
fetched: &Result<FeedFetch, FetchError>,
previous_failures: u32,
now_ms: i64,
) -> Result<TargetResult> {
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
let failure = |error: String, retry_after: Option<Duration>| {
let consecutive_failures = previous_failures.saturating_add(1);
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
(error, consecutive_failures, next_check_at_ms)
};
let (error, consecutive_failures, next_check_at_ms) = match fetched {
Ok(FeedFetch::NotModified) => {
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
return match store
.record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next))
.await
{
Ok(()) => Ok(TargetResult::NotModified),
Err(error) => skipped_or(error),
};
}
Ok(FeedFetch::Fetched(document)) => {
match ingest(store, target, document, now_ms).await {
Ok(outcome) => return Ok(TargetResult::Updated(outcome)),
Err(error) if is_gone(&error) => return skipped_or(error),
// A store failure is recorded like a fetch failure, so the
// feed backs off instead of being retried every run.
Err(error) => failure(format!("storing the feed failed: {error:#}"), None),
}
}
Err(error) => failure(error.message.clone(), error.retry_after),
};
match store
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
.await
{
Ok(_) => Ok(TargetResult::Failed {
error,
consecutive_failures,
next_check_at_ms,
}),
Err(error) => skipped_or(error),
}
}
async fn ingest(
store: &PodcastStore,
target: &RefreshTarget,
document: &FetchedFeed,
now_ms: i64,
) -> Result<RefreshOutcome> {
let outcome = store
.apply_feed(
target.user_id,
&FeedRefresh {
podcast_id: &target.podcast_id,
feed: &document.feed,
fetched_from: &document.fetched_from,
moved_permanently: document.moved_permanently,
},
)
.await?;
store
.record_fetch_success(
target.user_id,
&target.podcast_id,
now_ms,
document.validators.etag.as_deref(),
document.validators.last_modified.as_deref(),
Some(next_regular_check_ms(
now_ms,
target.user_id,
&target.podcast_id,
)),
)
.await?;
Ok(outcome)
}
fn is_gone(error: &anyhow::Error) -> bool {
matches!(
error.downcast_ref::<PodcastStoreError>(),
Some(PodcastStoreError::UnknownPodcast(_))
)
}
/// A podcast removed from the account meanwhile is skipped; any other
/// store error is returned.
fn skipped_or(error: anyhow::Error) -> Result<TargetResult> {
if is_gone(&error) {
Ok(TargetResult::Skipped(error.to_string()))
} else {
Err(error)
}
}
/// Why a manual refresh was not started.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ManualRefreshError {
/// The account does not know the podcast.
UnknownPodcast,
/// The account is not subscribed (unsubscribing stops monitoring).
NotSubscribed,
/// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago.
TooSoon { retry_after_ms: i64 },
}
impl fmt::Display for ManualRefreshError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownPodcast => f.write_str("unknown podcast"),
Self::NotSubscribed => f.write_str("not subscribed to this podcast"),
Self::TooSoon { retry_after_ms } => write!(
f,
"the feed was checked moments ago; retry in {} s",
(retry_after_ms + 999) / 1000
),
}
}
}
impl std::error::Error for ManualRefreshError {}
/// Refreshes one subscribed podcast of one account now, ignoring the
/// failure backoff but not more often than every
/// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a
/// [`ManualRefreshError`] when the refresh was not attempted.
pub(crate) async fn refresh_now(
store: &PodcastStore,
fetcher: &FeedFetcher,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
) -> Result<TargetResult> {
let feed_url = match store.podcast(user_id, podcast_id).await? {
Some(summary) if summary.subscribed => summary.meta.feed_url,
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
None => match store.subscription(user_id, podcast_id).await? {
Some(register) if register.subscribed => register.podcast.feed_url,
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
None => return Err(ManualRefreshError::UnknownPodcast.into()),
},
};
if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms {
let elapsed = now_ms - checked;
if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) {
return Err(ManualRefreshError::TooSoon {
retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed,
}
.into());
}
}
let target = RefreshTarget {
user_id,
podcast_id: podcast_id.clone(),
};
let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?;
Ok(results.pop().map(|(_, result)| result).expect("one target"))
}
+1
View File
@@ -1311,6 +1311,7 @@ async fn run_scheduled_job(
&& job_name != "lastfm_scrobble"
&& job_name != "archive_cleanup"
&& job_name != "artwork_backfill"
&& job_name != "podcast_refresh"
{
tracing::warn!(job = job_name, "Skipping: agent_enabled=false");
return;