Refresh subscribed podcast feeds in a background job
The podcast_refresh job runs every five minutes and refreshes the podcasts whose next check is due: conditional requests (ETag, Last-Modified), 32 MiB and 60 s limits, the shared music-dht parser and PodcastStore::apply_feed. A chain of permanent redirects moves the feed and becomes an alias. Healthy feeds are checked hourly with a stable spread; failures back off from 15 minutes up to a day, honouring Retry-After. Accounts subscribed to the same feed URL share one request, at most four run at once, and one run at a time holds an advisory lock. runner::refresh_now is the manual refresh for one subscribed podcast, at most every 30 seconds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
04ae7d5bd1
commit
bf9a6abc3f
9 files changed
+1273
-4
No files matched your search
@@ -6,6 +6,7 @@ pub mod lastfm_popularity;
|
||||
pub mod lastfm_scrobble;
|
||||
pub mod metadata_backfill;
|
||||
pub mod musicbrainz;
|
||||
pub mod podcast_refresh;
|
||||
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
//! Background refresh of subscribed podcast feeds.
|
||||
//!
|
||||
//! Every run takes the podcasts whose next check is due (see
|
||||
//! [`crate::podcasts::runner`]): healthy feeds are checked about hourly with
|
||||
//! conditional requests, failing ones back off exponentially up to a day.
|
||||
//! One request per feed URL serves every account subscribed to it, a
|
||||
//! bounded number of requests run concurrently, and a failing feed never
|
||||
//! blocks the others or clears the stored catalog.
|
||||
|
||||
use crate::podcasts::{FeedFetcher, PodcastStore, now_ms, runner};
|
||||
use crate::scheduler::{Job, JobContext, JobLog};
|
||||
|
||||
pub struct PodcastRefreshJob;
|
||||
|
||||
/// Session-level advisory lock: one run at a time across processes.
|
||||
const ADVISORY_LOCK_ID: i64 = 0x504f_4443_5246_5348; // "PODCRFSH"
|
||||
|
||||
/// Failures listed individually in the run log.
|
||||
const LOGGED_FAILURES: usize = 20;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Job for PodcastRefreshJob {
|
||||
fn name(&self) -> &'static str {
|
||||
"podcast_refresh"
|
||||
}
|
||||
|
||||
fn description(&self) -> &'static str {
|
||||
"Check subscribed podcast feeds for new episodes (metadata only, no audio)"
|
||||
}
|
||||
|
||||
fn default_cron(&self) -> &'static str {
|
||||
// Every five minutes; each feed is checked only when due.
|
||||
"0 */5 * * * *"
|
||||
}
|
||||
|
||||
async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> {
|
||||
let mut conn = ctx.pool.acquire().await?;
|
||||
// A session lock must never return to the pool, including on
|
||||
// cancellation; the store below uses other pool connections.
|
||||
conn.close_on_drop();
|
||||
let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)")
|
||||
.bind(ADVISORY_LOCK_ID)
|
||||
.fetch_one(&mut *conn)
|
||||
.await?;
|
||||
if !locked {
|
||||
log.info("Podcast refresh is already running; skipping this run");
|
||||
return Ok(());
|
||||
}
|
||||
let store = PodcastStore::new(ctx.pool.clone());
|
||||
let fetcher = FeedFetcher::new()?;
|
||||
let result = runner::refresh_due(
|
||||
&store,
|
||||
&fetcher,
|
||||
now_ms(),
|
||||
runner::RUN_LIMIT,
|
||||
runner::RUN_CONCURRENCY,
|
||||
)
|
||||
.await;
|
||||
conn.close().await?;
|
||||
let summary = result?;
|
||||
for (user_id, podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
|
||||
log.warn(&format!("user {user_id}, podcast {podcast_id}: {error}"));
|
||||
}
|
||||
if summary.failures.len() > LOGGED_FAILURES {
|
||||
log.warn(&format!(
|
||||
"... and {} more failed feeds",
|
||||
summary.failures.len() - LOGGED_FAILURES
|
||||
));
|
||||
}
|
||||
log.info(&format!("Podcast refresh: {summary}"));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,7 @@ fn build_registry() -> Arc<JobRegistry> {
|
||||
registry.register(jobs::metadata_backfill::MetadataBackfillJob);
|
||||
registry.register(jobs::lastfm_popularity::LastfmPopularityJob);
|
||||
registry.register(jobs::lastfm_scrobble::LastfmScrobbleJob);
|
||||
registry.register(jobs::podcast_refresh::PodcastRefreshJob);
|
||||
Arc::new(registry)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`].
|
||||
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::header::{
|
||||
ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH,
|
||||
LAST_MODIFIED, RETRY_AFTER,
|
||||
};
|
||||
use reqwest::{StatusCode, Url};
|
||||
|
||||
use music_dht::podcast::FeedUrl;
|
||||
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
|
||||
|
||||
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
|
||||
|
||||
/// Hard cap on a feed document (large catalogs are a few MiB).
|
||||
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// Whole fetch, from connect to the last body byte.
|
||||
pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
/// Redirect hops followed for a feed.
|
||||
pub(crate) const MAX_FEED_REDIRECTS: usize = 5;
|
||||
/// `Retry-After` values above this are clamped.
|
||||
const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
|
||||
text/xml;q=0.9, */*;q=0.5";
|
||||
|
||||
/// HTTP client settings of the feed fetcher.
|
||||
pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig {
|
||||
connect_timeout: Duration::from_secs(10),
|
||||
read_timeout: Duration::from_secs(30),
|
||||
total_timeout: Some(FEED_FETCH_TIMEOUT),
|
||||
max_redirects: MAX_FEED_REDIRECTS,
|
||||
};
|
||||
|
||||
/// Validators of the previous successful fetch.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
|
||||
pub(crate) struct Validators {
|
||||
pub(crate) etag: Option<String>,
|
||||
pub(crate) last_modified: Option<String>,
|
||||
}
|
||||
|
||||
/// A successfully fetched and parsed document.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct FetchedFeed {
|
||||
pub(crate) feed: ParsedFeed,
|
||||
/// Where the feed now lives: the final URL after a chain of permanent
|
||||
/// redirects, otherwise the requested URL.
|
||||
pub(crate) fetched_from: FeedUrl,
|
||||
/// Whether the feed moved: at least one redirect, all permanent (see
|
||||
/// [`super::net::Followed::moved_permanently`]), to a valid feed URL.
|
||||
pub(crate) moved_permanently: bool,
|
||||
/// New validators for the next conditional request.
|
||||
pub(crate) validators: Validators,
|
||||
}
|
||||
|
||||
/// Outcome of [`FeedFetcher::fetch`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum FeedFetch {
|
||||
/// `304 Not Modified`: the stored catalog is current.
|
||||
NotModified,
|
||||
/// A new document.
|
||||
Fetched(Box<FetchedFeed>),
|
||||
}
|
||||
|
||||
/// Kind of a failed fetch.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum FetchErrorKind {
|
||||
/// Refused by the SSRF guard (blocked address, scheme, credentials).
|
||||
Refused,
|
||||
/// Non-success HTTP status.
|
||||
Status(u16),
|
||||
/// Body over [`MAX_FEED_BYTES`].
|
||||
TooLarge,
|
||||
/// The document is not a feed.
|
||||
Parse,
|
||||
/// Timed out.
|
||||
Timeout,
|
||||
/// DNS, connect, TLS or a dropped connection.
|
||||
Transport,
|
||||
}
|
||||
|
||||
/// A failed fetch. The stored catalog is never touched on failure.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct FetchError {
|
||||
pub(crate) kind: FetchErrorKind,
|
||||
pub(crate) message: String,
|
||||
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
|
||||
pub(crate) retry_after: Option<Duration>,
|
||||
}
|
||||
|
||||
impl FetchError {
|
||||
fn new(kind: FetchErrorKind, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
retry_after: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FetchError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FetchError {}
|
||||
|
||||
impl From<GuardError> for FetchError {
|
||||
fn from(error: GuardError) -> Self {
|
||||
let kind = match &error {
|
||||
GuardError::UnsupportedScheme(_)
|
||||
| GuardError::Credentials
|
||||
| GuardError::MissingHost
|
||||
| GuardError::Blocked(_) => FetchErrorKind::Refused,
|
||||
_ if error.is_timeout() => FetchErrorKind::Timeout,
|
||||
_ => FetchErrorKind::Transport,
|
||||
};
|
||||
Self::new(kind, error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses `Retry-After` (delta seconds or an HTTP date).
|
||||
pub(crate) fn parse_retry_after(
|
||||
value: &str,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> Option<Duration> {
|
||||
let value = value.trim();
|
||||
let delay = if let Ok(seconds) = value.parse::<u64>() {
|
||||
Duration::from_secs(seconds)
|
||||
} else {
|
||||
let at = chrono::DateTime::parse_from_rfc2822(value).ok()?;
|
||||
(at.with_timezone(&chrono::Utc) - now)
|
||||
.to_std()
|
||||
.unwrap_or_default()
|
||||
};
|
||||
Some(delay.min(MAX_RETRY_AFTER))
|
||||
}
|
||||
|
||||
/// Downloads feeds with the SSRF guard, conditional requests, redirects
|
||||
/// and hard size/time limits; parses them with the shared parser.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct FeedFetcher {
|
||||
http: SafeHttp,
|
||||
max_bytes: usize,
|
||||
}
|
||||
|
||||
impl FeedFetcher {
|
||||
/// The production fetcher (public addresses only).
|
||||
pub(crate) fn new() -> anyhow::Result<Self> {
|
||||
Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?))
|
||||
}
|
||||
|
||||
pub(crate) fn with_http(http: SafeHttp) -> Self {
|
||||
Self {
|
||||
http,
|
||||
max_bytes: MAX_FEED_BYTES,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self {
|
||||
self.max_bytes = max_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Fetches `url`, conditionally when `validators` are known.
|
||||
pub(crate) async fn fetch(
|
||||
&self,
|
||||
url: &FeedUrl,
|
||||
validators: &Validators,
|
||||
) -> Result<FeedFetch, FetchError> {
|
||||
match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(FetchError::new(
|
||||
FetchErrorKind::Timeout,
|
||||
format!(
|
||||
"feed fetch timed out after {} s",
|
||||
FEED_FETCH_TIMEOUT.as_secs()
|
||||
),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_inner(
|
||||
&self,
|
||||
feed_url: &FeedUrl,
|
||||
validators: &Validators,
|
||||
) -> Result<FeedFetch, FetchError> {
|
||||
let url = Url::parse(feed_url.as_str())
|
||||
.map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT));
|
||||
// Compressed bodies are not negotiated (no decoder is enabled), so
|
||||
// the size cap applies to what is parsed.
|
||||
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
|
||||
if let Some(etag) = validators
|
||||
.etag
|
||||
.as_deref()
|
||||
.and_then(|value| HeaderValue::from_str(value).ok())
|
||||
{
|
||||
headers.insert(IF_NONE_MATCH, etag);
|
||||
}
|
||||
if let Some(date) = validators
|
||||
.last_modified
|
||||
.as_deref()
|
||||
.and_then(|value| HeaderValue::from_str(value).ok())
|
||||
{
|
||||
headers.insert(IF_MODIFIED_SINCE, date);
|
||||
}
|
||||
let followed = self.http.get(url, &headers).await?;
|
||||
let mut moved_permanently = followed.moved_permanently();
|
||||
let final_url = followed.final_url;
|
||||
let response = followed.response;
|
||||
let status = response.status();
|
||||
if status == StatusCode::NOT_MODIFIED {
|
||||
return Ok(FeedFetch::NotModified);
|
||||
}
|
||||
if !status.is_success() {
|
||||
let mut error = FetchError::new(
|
||||
FetchErrorKind::Status(status.as_u16()),
|
||||
format!("feed request failed: HTTP {status}"),
|
||||
);
|
||||
if matches!(status.as_u16(), 429 | 503) {
|
||||
error.retry_after = response
|
||||
.headers()
|
||||
.get(RETRY_AFTER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| parse_retry_after(value, chrono::Utc::now()));
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
let header = |name| {
|
||||
response
|
||||
.headers()
|
||||
.get(name)
|
||||
.and_then(|value: &HeaderValue| value.to_str().ok())
|
||||
.map(str::to_string)
|
||||
};
|
||||
let validators = Validators {
|
||||
etag: header(ETAG),
|
||||
last_modified: header(LAST_MODIFIED),
|
||||
};
|
||||
let body = read_capped(response, self.max_bytes).await?;
|
||||
// Relative enclosure URLs resolve against where the feed really is.
|
||||
let base = final_url.to_string();
|
||||
let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base)))
|
||||
.await
|
||||
.map_err(|error| {
|
||||
FetchError::new(
|
||||
FetchErrorKind::Parse,
|
||||
format!("feed parser task failed: {error}"),
|
||||
)
|
||||
})?
|
||||
.map_err(|error| {
|
||||
FetchError::new(
|
||||
FetchErrorKind::Parse,
|
||||
format!("not a podcast feed: {error}"),
|
||||
)
|
||||
})?;
|
||||
// Only a permanent move changes the feed URL; a target that is not a
|
||||
// valid feed URL (over-long, ...) is not learned.
|
||||
let moved_to = moved_permanently
|
||||
.then(|| FeedUrl::parse(final_url.as_str()).ok())
|
||||
.flatten();
|
||||
moved_permanently = moved_to.is_some();
|
||||
let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone());
|
||||
Ok(FeedFetch::Fetched(Box::new(FetchedFeed {
|
||||
feed,
|
||||
fetched_from,
|
||||
moved_permanently,
|
||||
validators,
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result<Vec<u8>, FetchError> {
|
||||
let too_large = |detail: String| {
|
||||
FetchError::new(
|
||||
FetchErrorKind::TooLarge,
|
||||
format!("feed is too large ({detail}, limit {cap} bytes)"),
|
||||
)
|
||||
};
|
||||
if let Some(len) = response.content_length()
|
||||
&& len > cap as u64
|
||||
{
|
||||
return Err(too_large(format!("{len} bytes")));
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response
|
||||
.chunk()
|
||||
.await
|
||||
.map_err(|error| FetchError::from(GuardError::from(error)))?
|
||||
{
|
||||
if body.len() + chunk.len() > cap {
|
||||
return Err(too_large("body exceeds the limit".to_string()));
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
//! Feed fetching (conditional requests, redirects, limits) and the refresh
|
||||
//! schedule, against the local test server.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
|
||||
use super::fetch::{FeedFetch, FeedFetcher, FetchErrorKind, Validators, parse_retry_after};
|
||||
use super::runner::{
|
||||
BACKOFF_BASE_MS, BACKOFF_MAX_MS, REFRESH_INTERVAL_MS, failure_backoff_ms, next_regular_check_ms,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::test_server::{Route, ScriptedLookup, TestServer, test_http};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Feed fetching
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn rss(items: &str) -> String {
|
||||
format!(
|
||||
r#"<?xml version="1.0"?><rss version="2.0"><channel><title>Show</title>{items}</channel></rss>"#
|
||||
)
|
||||
}
|
||||
|
||||
fn item(guid: &str) -> String {
|
||||
format!(
|
||||
r#"<item><title>{guid}</title><guid>{guid}</guid><enclosure url="/audio/{guid}.mp3" type="audio/mpeg"/></item>"#
|
||||
)
|
||||
}
|
||||
|
||||
fn fetcher(lookup: Arc<ScriptedLookup>) -> FeedFetcher {
|
||||
FeedFetcher::with_http(test_http(lookup))
|
||||
}
|
||||
|
||||
fn feed_url(text: &str) -> FeedUrl {
|
||||
FeedUrl::parse(text).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feeds_are_fetched_conditionally() {
|
||||
let server = TestServer::start();
|
||||
server.set(
|
||||
"/feed.xml",
|
||||
Route::new(rss(&item("a")), "application/rss+xml")
|
||||
.etag("\"v1\"")
|
||||
.last_modified("Wed, 21 Oct 2026 07:28:00 GMT"),
|
||||
);
|
||||
let fetcher = fetcher(ScriptedLookup::with(&[]));
|
||||
let feed = feed_url(&server.url("/feed.xml"));
|
||||
let FeedFetch::Fetched(first) = fetcher.fetch(&feed, &Validators::default()).await.unwrap()
|
||||
else {
|
||||
panic!("expected a document");
|
||||
};
|
||||
assert_eq!(first.feed.episodes.len(), 1);
|
||||
assert_eq!(
|
||||
first.feed.episodes[0].audio_url,
|
||||
server.url("/audio/a.mp3"),
|
||||
"relative enclosures resolve against the feed URL"
|
||||
);
|
||||
assert_eq!(first.fetched_from, feed);
|
||||
assert!(!first.moved_permanently);
|
||||
assert_eq!(first.validators.etag.as_deref(), Some("\"v1\""));
|
||||
let second = fetcher.fetch(&feed, &first.validators).await.unwrap();
|
||||
assert!(matches!(second, FeedFetch::NotModified));
|
||||
// Only Last-Modified known.
|
||||
let by_date = Validators {
|
||||
etag: None,
|
||||
last_modified: first.validators.last_modified.clone(),
|
||||
};
|
||||
assert!(matches!(
|
||||
fetcher.fetch(&feed, &by_date).await.unwrap(),
|
||||
FeedFetch::NotModified
|
||||
));
|
||||
let requests = server.requests("/feed.xml");
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert!(!requests[0].contains_key("if-none-match"));
|
||||
assert_eq!(requests[1]["if-none-match"], "\"v1\"");
|
||||
assert_eq!(
|
||||
requests[1]["if-modified-since"],
|
||||
"Wed, 21 Oct 2026 07:28:00 GMT"
|
||||
);
|
||||
assert!(!requests[2].contains_key("if-none-match"));
|
||||
assert_eq!(requests[2]["accept-encoding"], "identity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn only_permanent_redirect_chains_move_a_feed() {
|
||||
let server = TestServer::start();
|
||||
server.set(
|
||||
"/new.xml",
|
||||
Route::new(rss(&item("a")), "application/rss+xml"),
|
||||
);
|
||||
server.set("/old.xml", Route::redirect(301, "/new.xml"));
|
||||
server.set("/moved.xml", Route::redirect(308, "/old.xml"));
|
||||
server.set("/temp.xml", Route::redirect(302, "/new.xml"));
|
||||
server.set("/mixed.xml", Route::redirect(307, "/old.xml"));
|
||||
let fetcher = fetcher(ScriptedLookup::with(&[]));
|
||||
for (path, moved) in [
|
||||
("/old.xml", true),
|
||||
("/moved.xml", true),
|
||||
("/temp.xml", false),
|
||||
("/mixed.xml", false),
|
||||
] {
|
||||
let requested = feed_url(&server.url(path));
|
||||
let FeedFetch::Fetched(document) = fetcher
|
||||
.fetch(&requested, &Validators::default())
|
||||
.await
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("{path}");
|
||||
};
|
||||
assert_eq!(document.moved_permanently, moved, "{path}");
|
||||
let expected = if moved {
|
||||
feed_url(&server.url("/new.xml"))
|
||||
} else {
|
||||
requested
|
||||
};
|
||||
assert_eq!(document.fetched_from, expected, "{path}");
|
||||
assert_eq!(
|
||||
document.feed.episodes[0].audio_url,
|
||||
server.url("/audio/a.mp3")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feed_fetch_failures_are_typed() {
|
||||
let server = TestServer::start();
|
||||
let big = rss(&item("x").repeat(200));
|
||||
server.set("/big.xml", Route::new(big.clone(), "application/rss+xml"));
|
||||
server.set(
|
||||
"/big-chunked.xml",
|
||||
Route::new(big.clone(), "application/rss+xml").no_length(),
|
||||
);
|
||||
server.set(
|
||||
"/page.html",
|
||||
Route::new("<html><body>hi</body></html>", "text/html"),
|
||||
);
|
||||
server.set(
|
||||
"/busy.xml",
|
||||
Route::new("busy", "text/plain")
|
||||
.status(503)
|
||||
.header("Retry-After", "120"),
|
||||
);
|
||||
server.set("/gone.xml", Route::new("", "text/plain").status(410));
|
||||
server.set(
|
||||
"/private.xml",
|
||||
Route::redirect(301, "http://192.168.0.1/feed"),
|
||||
);
|
||||
let fetcher = fetcher(ScriptedLookup::with(&[])).with_max_bytes(big.len() - 1);
|
||||
let fetch = |path: &str| {
|
||||
let feed = feed_url(&server.url(path));
|
||||
let fetcher = fetcher.clone();
|
||||
async move {
|
||||
fetcher
|
||||
.fetch(&feed, &Validators::default())
|
||||
.await
|
||||
.unwrap_err()
|
||||
}
|
||||
};
|
||||
assert_eq!(fetch("/big.xml").await.kind, FetchErrorKind::TooLarge);
|
||||
assert_eq!(
|
||||
fetch("/big-chunked.xml").await.kind,
|
||||
FetchErrorKind::TooLarge
|
||||
);
|
||||
assert_eq!(fetch("/page.html").await.kind, FetchErrorKind::Parse);
|
||||
let busy = fetch("/busy.xml").await;
|
||||
assert_eq!(busy.kind, FetchErrorKind::Status(503));
|
||||
assert_eq!(busy.retry_after, Some(Duration::from_secs(120)));
|
||||
let gone = fetch("/gone.xml").await;
|
||||
assert_eq!(gone.kind, FetchErrorKind::Status(410));
|
||||
assert_eq!(gone.retry_after, None);
|
||||
assert_eq!(fetch("/private.xml").await.kind, FetchErrorKind::Refused);
|
||||
assert_eq!(
|
||||
fetch("/missing.xml").await.kind,
|
||||
FetchErrorKind::Status(404)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_accepts_seconds_and_dates() {
|
||||
let now = chrono::DateTime::parse_from_rfc3339("2026-11-19T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&chrono::Utc);
|
||||
assert_eq!(parse_retry_after("90", now), Some(Duration::from_secs(90)));
|
||||
assert_eq!(
|
||||
parse_retry_after("Thu, 19 Nov 2026 10:05:00 GMT", now),
|
||||
Some(Duration::from_secs(300))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_retry_after("Thu, 19 Nov 2026 09:00:00 GMT", now),
|
||||
Some(Duration::ZERO)
|
||||
);
|
||||
assert_eq!(
|
||||
parse_retry_after("999999999", now),
|
||||
Some(Duration::from_secs(24 * 60 * 60))
|
||||
);
|
||||
assert_eq!(parse_retry_after("soon", now), None);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Refresh schedule
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn failures_back_off_exponentially_up_to_a_day() {
|
||||
assert_eq!(failure_backoff_ms(1, None), BACKOFF_BASE_MS);
|
||||
assert_eq!(failure_backoff_ms(2, None), 2 * BACKOFF_BASE_MS);
|
||||
assert_eq!(failure_backoff_ms(3, None), 4 * BACKOFF_BASE_MS);
|
||||
assert_eq!(failure_backoff_ms(7, None), 64 * BACKOFF_BASE_MS);
|
||||
assert_eq!(failure_backoff_ms(8, None), BACKOFF_MAX_MS);
|
||||
assert_eq!(failure_backoff_ms(u32::MAX, None), BACKOFF_MAX_MS);
|
||||
// A longer Retry-After wins, a shorter one does not shorten the backoff.
|
||||
assert_eq!(
|
||||
failure_backoff_ms(1, Some(Duration::from_secs(3 * 60 * 60))),
|
||||
3 * 60 * 60 * 1000
|
||||
);
|
||||
assert_eq!(
|
||||
failure_backoff_ms(2, Some(Duration::from_secs(1))),
|
||||
2 * BACKOFF_BASE_MS
|
||||
);
|
||||
assert_eq!(
|
||||
failure_backoff_ms(1, Some(Duration::from_secs(u64::MAX))),
|
||||
BACKOFF_MAX_MS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_checks_are_hourly_with_a_stable_spread() {
|
||||
let podcast = PodcastId::from_feed_url(&feed_url("https://feeds.example/a.xml"));
|
||||
let other = PodcastId::from_feed_url(&feed_url("https://feeds.example/b.xml"));
|
||||
let now = 1_000_000;
|
||||
let next = next_regular_check_ms(now, 7, &podcast);
|
||||
assert!(
|
||||
(now + REFRESH_INTERVAL_MS..now + REFRESH_INTERVAL_MS + 10 * 60 * 1000).contains(&next)
|
||||
);
|
||||
assert_eq!(next, next_regular_check_ms(now, 7, &podcast));
|
||||
let spread: std::collections::BTreeSet<i64> = (0..20)
|
||||
.map(|user| next_regular_check_ms(now, user, &podcast))
|
||||
.chain([next_regular_check_ms(now, 7, &other)])
|
||||
.collect();
|
||||
assert!(spread.len() > 10);
|
||||
}
|
||||
+9
-4
@@ -42,19 +42,23 @@
|
||||
//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the
|
||||
//! alias set only; they never write a register, progress or a session.
|
||||
|
||||
// The RSS refresh job and the podcast UI/HTTP API that call the local
|
||||
// actions, queries and fetch state land in the following commits of this
|
||||
// series; until then only sync and the tests use them. Remove this
|
||||
// allowance with them.
|
||||
// The podcast UI and its HTTP API, which call the local actions and most
|
||||
// queries, land in the next commits of this series; until then only sync,
|
||||
// the refresh job and the tests use the store. Remove this allowance with
|
||||
// them.
|
||||
#![allow(dead_code, unused_imports)]
|
||||
|
||||
mod actions;
|
||||
mod apply;
|
||||
mod fetch;
|
||||
mod net;
|
||||
mod queries;
|
||||
mod refresh;
|
||||
pub(crate) mod runner;
|
||||
pub(crate) mod schema;
|
||||
|
||||
#[cfg(test)]
|
||||
mod fetch_tests;
|
||||
#[cfg(test)]
|
||||
mod net_tests;
|
||||
#[cfg(test)]
|
||||
@@ -80,6 +84,7 @@ pub(crate) use queries::{
|
||||
EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter,
|
||||
PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary,
|
||||
};
|
||||
pub(crate) use fetch::FeedFetcher;
|
||||
pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome};
|
||||
|
||||
/// Typed errors of [`PodcastStore`]; other failures are database errors.
|
||||
|
||||
@@ -1465,3 +1465,256 @@ async fn postgres_podcast_migrations_import_the_relay_state() {
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RSS refresh job
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn refresh_item(guid: &str) -> String {
|
||||
format!(
|
||||
r#"<item><title>Episode {guid}</title><guid>{guid}</guid>
|
||||
<enclosure url="/audio/{guid}.mp3" type="audio/mpeg" length="1000"/></item>"#
|
||||
)
|
||||
}
|
||||
|
||||
/// A fetcher that may reach the local test server (loopback is allowed
|
||||
/// only by the test policy).
|
||||
fn local_fetcher() -> FeedFetcher {
|
||||
FeedFetcher::with_http(
|
||||
net::SafeHttp::with_lookup(
|
||||
fetch::FEED_HTTP,
|
||||
net::AddressPolicy::test_allow_loopback(),
|
||||
std::sync::Arc::new(net::SystemLookup),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Due groups of the given accounts only (other tests share the database).
|
||||
async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups {
|
||||
let due = store
|
||||
.podcasts_due_for_refresh(now, 100_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|(user_id, _, _)| users.contains(user_id))
|
||||
.collect();
|
||||
runner::group_targets(due)
|
||||
}
|
||||
|
||||
async fn run_due(
|
||||
store: &PodcastStore,
|
||||
fetcher: &FeedFetcher,
|
||||
users: &[i64],
|
||||
now: i64,
|
||||
) -> runner::RunSummary {
|
||||
let groups = due_for(store, users, now).await;
|
||||
runner::refresh_groups(store, fetcher, groups, now, 4)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
async fn postgres_podcast_refresh_job_schedules_backs_off_and_moves_feeds() {
|
||||
use super::runner::{BACKOFF_BASE_MS, ManualRefreshError, TargetResult};
|
||||
use super::test_server::{Route, TestServer};
|
||||
|
||||
let pool = test_pool().await;
|
||||
let store = PodcastStore::new(pool.clone());
|
||||
let (a, b, c) = (random_user(), random_user(), random_user());
|
||||
cleanup(&pool, [a, b, c]).await;
|
||||
let users = [a, b, c];
|
||||
let server = TestServer::start();
|
||||
server.set(
|
||||
"/show.xml",
|
||||
Route::new(rss(&refresh_item("one")), "application/rss+xml").etag("\"v1\""),
|
||||
);
|
||||
let show = PodcastMeta {
|
||||
feed_url: FeedUrl::parse(&server.url("/show.xml")).unwrap(),
|
||||
title: "Local Show".into(),
|
||||
author: None,
|
||||
description: None,
|
||||
link: None,
|
||||
image_url: None,
|
||||
};
|
||||
let podcast_id = store.subscribe(a, &show).await.unwrap().podcast_id;
|
||||
assert_eq!(
|
||||
store.subscribe(b, &show).await.unwrap().podcast_id,
|
||||
podcast_id
|
||||
);
|
||||
let fetcher = local_fetcher();
|
||||
|
||||
// First run: one request serves both accounts.
|
||||
let t0 = super::now_ms();
|
||||
let summary = run_due(&store, &fetcher, &users, t0).await;
|
||||
assert_eq!((summary.targets, summary.feeds), (2, 1), "{summary}");
|
||||
assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}");
|
||||
assert_eq!(server.requests("/show.xml").len(), 1);
|
||||
for user in [a, b] {
|
||||
let state = store.fetch_state(user, &podcast_id).await.unwrap();
|
||||
assert_eq!(state.etag.as_deref(), Some("\"v1\""));
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
let next = state.next_check_at_ms.unwrap();
|
||||
assert!((t0 + runner::REFRESH_INTERVAL_MS..t0 + 70 * 60 * 1000).contains(&next));
|
||||
}
|
||||
let episodes = store
|
||||
.episodes(a, &EpisodeQuery::new(podcast_id.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(episodes.len(), 1);
|
||||
let episode_id = episodes[0].episode.episode_id.clone();
|
||||
assert_eq!(episodes[0].episode.audio_url, server.url("/audio/one.mp3"));
|
||||
|
||||
// Nothing is due before the interval; afterwards the request is
|
||||
// conditional and the catalog stays as it is.
|
||||
assert!(due_for(&store, &users, t0 + 1000).await.is_empty());
|
||||
let t1 = t0 + 2 * runner::REFRESH_INTERVAL_MS;
|
||||
let summary = run_due(&store, &fetcher, &users, t1).await;
|
||||
assert_eq!(summary.not_modified, 2, "{summary}");
|
||||
assert_eq!(server.requests("/show.xml")[1]["if-none-match"], "\"v1\"");
|
||||
|
||||
// Failures back off exponentially and keep the catalog.
|
||||
server.set("/show.xml", Route::new("boom", "text/plain").status(500));
|
||||
let t2 = t1 + 2 * runner::REFRESH_INTERVAL_MS;
|
||||
let summary = run_due(&store, &fetcher, &users, t2).await;
|
||||
assert_eq!(summary.failed, 2, "{summary}");
|
||||
assert_eq!(summary.failures.len(), 2);
|
||||
let state = store.fetch_state(a, &podcast_id).await.unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert_eq!(state.next_check_at_ms, Some(t2 + BACKOFF_BASE_MS));
|
||||
assert!(state.last_error.as_deref().unwrap().contains("500"));
|
||||
assert_eq!(state.etag.as_deref(), Some("\"v1\""), "validators are kept");
|
||||
assert!(
|
||||
due_for(&store, &users, t2 + BACKOFF_BASE_MS - 1)
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
let t3 = t2 + BACKOFF_BASE_MS;
|
||||
let summary = run_due(&store, &fetcher, &users, t3).await;
|
||||
assert_eq!(summary.failed, 2, "{summary}");
|
||||
let state = store.fetch_state(a, &podcast_id).await.unwrap();
|
||||
assert_eq!(state.consecutive_failures, 2);
|
||||
assert_eq!(state.next_check_at_ms, Some(t3 + 2 * BACKOFF_BASE_MS));
|
||||
assert_eq!(
|
||||
store
|
||||
.episodes(a, &EpisodeQuery::new(podcast_id.clone()))
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
// Recovery resets the backoff. Accounts with different validators are
|
||||
// fetched unconditionally.
|
||||
server.set(
|
||||
"/show.xml",
|
||||
Route::new(
|
||||
rss(&format!("{}{}", refresh_item("two"), refresh_item("one"))),
|
||||
"application/rss+xml",
|
||||
)
|
||||
.etag("\"v2\""),
|
||||
);
|
||||
store
|
||||
.record_fetch_success(b, &podcast_id, t3, Some("\"other\""), None, Some(t3))
|
||||
.await
|
||||
.unwrap();
|
||||
let t4 = t3 + 2 * BACKOFF_BASE_MS;
|
||||
let summary = run_due(&store, &fetcher, &users, t4).await;
|
||||
assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}");
|
||||
let last = server.requests("/show.xml").pop().unwrap();
|
||||
assert!(!last.contains_key("if-none-match"), "{last:?}");
|
||||
let state = store.fetch_state(a, &podcast_id).await.unwrap();
|
||||
assert_eq!((state.consecutive_failures, state.last_error), (0, None));
|
||||
assert_eq!(state.etag.as_deref(), Some("\"v2\""));
|
||||
|
||||
// Manual refresh: not more often than every 30 s, only when subscribed.
|
||||
let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 10_000).await;
|
||||
assert_eq!(
|
||||
manual.unwrap_err().downcast_ref::<ManualRefreshError>(),
|
||||
Some(&ManualRefreshError::TooSoon {
|
||||
retry_after_ms: 20_000
|
||||
})
|
||||
);
|
||||
let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 31_000)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(manual, TargetResult::NotModified);
|
||||
let unknown = runner::refresh_now(&store, &fetcher, c, &podcast_id, t4 + 31_000).await;
|
||||
assert_eq!(
|
||||
unknown.unwrap_err().downcast_ref::<ManualRefreshError>(),
|
||||
Some(&ManualRefreshError::UnknownPodcast)
|
||||
);
|
||||
store.unsubscribe(b, &podcast_id).await.unwrap();
|
||||
let unsubscribed = runner::refresh_now(&store, &fetcher, b, &podcast_id, t4 + 31_000).await;
|
||||
assert_eq!(
|
||||
unsubscribed
|
||||
.unwrap_err()
|
||||
.downcast_ref::<ManualRefreshError>(),
|
||||
Some(&ManualRefreshError::NotSubscribed)
|
||||
);
|
||||
let far = t4 + 10 * runner::REFRESH_INTERVAL_MS;
|
||||
let due = due_for(&store, &users, far).await;
|
||||
assert_eq!(
|
||||
due.values().flatten().count(),
|
||||
1,
|
||||
"unsubscribing stops monitoring"
|
||||
);
|
||||
|
||||
// A permanent move becomes an alias (one sync operation) and the new
|
||||
// fetch URL; later refreshes go to the new URL without new operations.
|
||||
server.set("/show.xml", Route::redirect(301, "/moved.xml"));
|
||||
server.set(
|
||||
"/moved.xml",
|
||||
Route::new(rss(&refresh_item("one")), "application/rss+xml"),
|
||||
);
|
||||
let moved = FeedUrl::parse(&server.url("/moved.xml")).unwrap();
|
||||
let result = runner::refresh_now(&store, &fetcher, a, &podcast_id, far)
|
||||
.await
|
||||
.unwrap();
|
||||
let TargetResult::Updated(outcome) = result else {
|
||||
panic!("{result:?}");
|
||||
};
|
||||
assert_eq!(outcome.feed_url.as_ref(), Some(&moved));
|
||||
assert_eq!(outcome.ops.len(), 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.podcast(a, &podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.meta
|
||||
.feed_url,
|
||||
moved
|
||||
);
|
||||
let due = due_for(&store, &users, far + 10 * runner::REFRESH_INTERVAL_MS).await;
|
||||
assert_eq!(due.keys().collect::<Vec<_>>(), vec![&moved]);
|
||||
let summary = run_due(
|
||||
&store,
|
||||
&fetcher,
|
||||
&users,
|
||||
far + 10 * runner::REFRESH_INTERVAL_MS,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(summary.updated, 1, "{summary}");
|
||||
assert_eq!(server.requests("/moved.xml").len(), 2);
|
||||
assert_eq!(server.requests("/show.xml").len(), 7);
|
||||
let TargetResult::Updated(again) = runner::refresh_now(
|
||||
&store,
|
||||
&fetcher,
|
||||
a,
|
||||
&podcast_id,
|
||||
far + 20 * runner::REFRESH_INTERVAL_MS,
|
||||
)
|
||||
.await
|
||||
.unwrap() else {
|
||||
panic!("expected a document");
|
||||
};
|
||||
assert!(again.ops.is_empty());
|
||||
// The episode keeps its id; its audio now resolves against the new URL
|
||||
// (same host here).
|
||||
let after_move = store.episode(a, &episode_id).await.unwrap().unwrap();
|
||||
assert_eq!(after_move.episode.audio_url, server.url("/audio/one.mp3"));
|
||||
|
||||
cleanup(&pool, [a, b, c]).await;
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
//! Scheduling of feed refreshes: which feeds are due, one request per feed
|
||||
//! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local
|
||||
//! fetch state (validators, next check, backoff).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use futures_util::StreamExt;
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
|
||||
use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators};
|
||||
use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome};
|
||||
|
||||
/// Regular interval between two checks of a healthy feed.
|
||||
pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000;
|
||||
/// Spread of the regular interval, so feeds added together drift apart.
|
||||
const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000;
|
||||
/// First retry delay after a failure; doubles per consecutive failure.
|
||||
pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
|
||||
/// Longest delay between two attempts of a failing feed.
|
||||
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
|
||||
/// A manual refresh is refused when the feed was checked this recently.
|
||||
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
|
||||
/// Podcasts considered per scheduled run.
|
||||
pub(crate) const RUN_LIMIT: usize = 200;
|
||||
/// Feed requests in flight at once.
|
||||
pub(crate) const RUN_CONCURRENCY: usize = 4;
|
||||
|
||||
/// Delay before the next attempt after `consecutive_failures` failures in a
|
||||
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
|
||||
/// day. A longer server-requested `Retry-After` wins (also capped).
|
||||
pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option<Duration>) -> i64 {
|
||||
let exponent = consecutive_failures.saturating_sub(1).min(16);
|
||||
let backoff = BACKOFF_BASE_MS
|
||||
.saturating_mul(1_i64 << exponent)
|
||||
.min(BACKOFF_MAX_MS);
|
||||
let requested = retry_after
|
||||
.map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX))
|
||||
.unwrap_or(0);
|
||||
backoff.max(requested).min(BACKOFF_MAX_MS)
|
||||
}
|
||||
|
||||
/// Next regular check after a success: the interval plus a stable per
|
||||
/// account and podcast jitter.
|
||||
pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 {
|
||||
let key = format!("{user_id}:{podcast_id}");
|
||||
let hash = blake3::hash(key.as_bytes());
|
||||
let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes"));
|
||||
let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0);
|
||||
now_ms + REFRESH_INTERVAL_MS + jitter
|
||||
}
|
||||
|
||||
/// One podcast of one account to refresh.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RefreshTarget {
|
||||
pub(crate) user_id: i64,
|
||||
pub(crate) podcast_id: PodcastId,
|
||||
}
|
||||
|
||||
/// Result of refreshing one [`RefreshTarget`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) enum TargetResult {
|
||||
/// A new document was ingested.
|
||||
Updated(RefreshOutcome),
|
||||
/// `304 Not Modified`.
|
||||
NotModified,
|
||||
/// The fetch or the ingestion failed; the catalog is unchanged.
|
||||
Failed {
|
||||
error: String,
|
||||
consecutive_failures: u32,
|
||||
next_check_at_ms: i64,
|
||||
},
|
||||
/// Nothing to do any more (unsubscribed or removed meanwhile).
|
||||
Skipped(String),
|
||||
}
|
||||
|
||||
/// Counters of a scheduled run.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub(crate) struct RunSummary {
|
||||
/// Account podcasts that were due.
|
||||
pub(crate) targets: usize,
|
||||
/// Distinct feed URLs requested.
|
||||
pub(crate) feeds: usize,
|
||||
pub(crate) updated: usize,
|
||||
pub(crate) not_modified: usize,
|
||||
pub(crate) failed: usize,
|
||||
pub(crate) skipped: usize,
|
||||
pub(crate) new_episodes: usize,
|
||||
pub(crate) updated_episodes: usize,
|
||||
/// `(user_id, podcast_id, error)` of failed targets, for the job log.
|
||||
pub(crate) failures: Vec<(i64, PodcastId, String)>,
|
||||
}
|
||||
|
||||
impl RunSummary {
|
||||
fn add(&mut self, target: &RefreshTarget, result: &TargetResult) {
|
||||
match result {
|
||||
TargetResult::Updated(outcome) => {
|
||||
self.updated += 1;
|
||||
self.new_episodes += outcome.new_episodes;
|
||||
self.updated_episodes += outcome.updated_episodes;
|
||||
}
|
||||
TargetResult::NotModified => self.not_modified += 1,
|
||||
TargetResult::Failed { error, .. } => {
|
||||
self.failed += 1;
|
||||
self.failures
|
||||
.push((target.user_id, target.podcast_id.clone(), error.clone()));
|
||||
}
|
||||
TargetResult::Skipped(_) => self.skipped += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for RunSummary {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \
|
||||
{} not modified, {} failed, {} skipped",
|
||||
self.targets,
|
||||
self.feeds,
|
||||
self.updated,
|
||||
self.new_episodes,
|
||||
self.updated_episodes,
|
||||
self.not_modified,
|
||||
self.failed,
|
||||
self.skipped
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Due podcasts grouped by feed URL: accounts subscribed to the same URL
|
||||
/// share one request.
|
||||
pub(crate) type RefreshGroups = BTreeMap<FeedUrl, Vec<RefreshTarget>>;
|
||||
|
||||
/// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`].
|
||||
pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups {
|
||||
let mut groups = RefreshGroups::new();
|
||||
for (user_id, podcast_id, feed_url) in due {
|
||||
groups.entry(feed_url).or_default().push(RefreshTarget {
|
||||
user_id,
|
||||
podcast_id,
|
||||
});
|
||||
}
|
||||
groups
|
||||
}
|
||||
|
||||
/// Refreshes every podcast due at `now_ms` (at most `limit`), with at most
|
||||
/// `concurrency` feed requests in flight. Accounts subscribed to the same
|
||||
/// feed URL share one request. A failing feed never stops the others.
|
||||
pub(crate) async fn refresh_due(
|
||||
store: &PodcastStore,
|
||||
fetcher: &FeedFetcher,
|
||||
now_ms: i64,
|
||||
limit: usize,
|
||||
concurrency: usize,
|
||||
) -> Result<RunSummary> {
|
||||
let due = store.podcasts_due_for_refresh(now_ms, limit).await?;
|
||||
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
|
||||
}
|
||||
|
||||
/// Refreshes `groups`, one request per feed URL, at most `concurrency` at
|
||||
/// a time.
|
||||
pub(crate) async fn refresh_groups(
|
||||
store: &PodcastStore,
|
||||
fetcher: &FeedFetcher,
|
||||
groups: RefreshGroups,
|
||||
now_ms: i64,
|
||||
concurrency: usize,
|
||||
) -> Result<RunSummary> {
|
||||
let mut summary = RunSummary {
|
||||
targets: groups.values().map(Vec::len).sum(),
|
||||
feeds: groups.len(),
|
||||
..RunSummary::default()
|
||||
};
|
||||
let mut results = futures_util::stream::iter(groups)
|
||||
.map(|(feed_url, targets)| async move {
|
||||
refresh_group(store, fetcher, &feed_url, targets, now_ms).await
|
||||
})
|
||||
.buffer_unordered(concurrency.max(1));
|
||||
while let Some(group) = results.next().await {
|
||||
for (target, result) in group? {
|
||||
summary.add(&target, &result);
|
||||
}
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Fetches `feed_url` once for all `targets` and records the outcome for
|
||||
/// each. The request is conditional only when every target holds the same
|
||||
/// validators (the usual case: one account).
|
||||
pub(crate) async fn refresh_group(
|
||||
store: &PodcastStore,
|
||||
fetcher: &FeedFetcher,
|
||||
feed_url: &FeedUrl,
|
||||
targets: Vec<RefreshTarget>,
|
||||
now_ms: i64,
|
||||
) -> Result<Vec<(RefreshTarget, TargetResult)>> {
|
||||
let mut states = Vec::with_capacity(targets.len());
|
||||
for target in &targets {
|
||||
states.push(
|
||||
store
|
||||
.fetch_state(target.user_id, &target.podcast_id)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let validators = states
|
||||
.iter()
|
||||
.map(|state| Validators {
|
||||
etag: state.etag.clone(),
|
||||
last_modified: state.last_modified.clone(),
|
||||
})
|
||||
.reduce(|a, b| if a == b { a } else { Validators::default() })
|
||||
.unwrap_or_default();
|
||||
let fetched = fetcher.fetch(feed_url, &validators).await;
|
||||
let mut out = Vec::with_capacity(targets.len());
|
||||
for (target, state) in targets.into_iter().zip(states) {
|
||||
let result =
|
||||
record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?;
|
||||
out.push((target, result));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Ingests (or records the failure of) one fetch for one target.
|
||||
async fn record_result(
|
||||
store: &PodcastStore,
|
||||
target: &RefreshTarget,
|
||||
fetched: &Result<FeedFetch, FetchError>,
|
||||
previous_failures: u32,
|
||||
now_ms: i64,
|
||||
) -> Result<TargetResult> {
|
||||
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
|
||||
let failure = |error: String, retry_after: Option<Duration>| {
|
||||
let consecutive_failures = previous_failures.saturating_add(1);
|
||||
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
|
||||
(error, consecutive_failures, next_check_at_ms)
|
||||
};
|
||||
let (error, consecutive_failures, next_check_at_ms) = match fetched {
|
||||
Ok(FeedFetch::NotModified) => {
|
||||
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
|
||||
return match store
|
||||
.record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next))
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(TargetResult::NotModified),
|
||||
Err(error) => skipped_or(error),
|
||||
};
|
||||
}
|
||||
Ok(FeedFetch::Fetched(document)) => {
|
||||
match ingest(store, target, document, now_ms).await {
|
||||
Ok(outcome) => return Ok(TargetResult::Updated(outcome)),
|
||||
Err(error) if is_gone(&error) => return skipped_or(error),
|
||||
// A store failure is recorded like a fetch failure, so the
|
||||
// feed backs off instead of being retried every run.
|
||||
Err(error) => failure(format!("storing the feed failed: {error:#}"), None),
|
||||
}
|
||||
}
|
||||
Err(error) => failure(error.message.clone(), error.retry_after),
|
||||
};
|
||||
match store
|
||||
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(TargetResult::Failed {
|
||||
error,
|
||||
consecutive_failures,
|
||||
next_check_at_ms,
|
||||
}),
|
||||
Err(error) => skipped_or(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ingest(
|
||||
store: &PodcastStore,
|
||||
target: &RefreshTarget,
|
||||
document: &FetchedFeed,
|
||||
now_ms: i64,
|
||||
) -> Result<RefreshOutcome> {
|
||||
let outcome = store
|
||||
.apply_feed(
|
||||
target.user_id,
|
||||
&FeedRefresh {
|
||||
podcast_id: &target.podcast_id,
|
||||
feed: &document.feed,
|
||||
fetched_from: &document.fetched_from,
|
||||
moved_permanently: document.moved_permanently,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
store
|
||||
.record_fetch_success(
|
||||
target.user_id,
|
||||
&target.podcast_id,
|
||||
now_ms,
|
||||
document.validators.etag.as_deref(),
|
||||
document.validators.last_modified.as_deref(),
|
||||
Some(next_regular_check_ms(
|
||||
now_ms,
|
||||
target.user_id,
|
||||
&target.podcast_id,
|
||||
)),
|
||||
)
|
||||
.await?;
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
fn is_gone(error: &anyhow::Error) -> bool {
|
||||
matches!(
|
||||
error.downcast_ref::<PodcastStoreError>(),
|
||||
Some(PodcastStoreError::UnknownPodcast(_))
|
||||
)
|
||||
}
|
||||
|
||||
/// A podcast removed from the account meanwhile is skipped; any other
|
||||
/// store error is returned.
|
||||
fn skipped_or(error: anyhow::Error) -> Result<TargetResult> {
|
||||
if is_gone(&error) {
|
||||
Ok(TargetResult::Skipped(error.to_string()))
|
||||
} else {
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a manual refresh was not started.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum ManualRefreshError {
|
||||
/// The account does not know the podcast.
|
||||
UnknownPodcast,
|
||||
/// The account is not subscribed (unsubscribing stops monitoring).
|
||||
NotSubscribed,
|
||||
/// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago.
|
||||
TooSoon { retry_after_ms: i64 },
|
||||
}
|
||||
|
||||
impl fmt::Display for ManualRefreshError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::UnknownPodcast => f.write_str("unknown podcast"),
|
||||
Self::NotSubscribed => f.write_str("not subscribed to this podcast"),
|
||||
Self::TooSoon { retry_after_ms } => write!(
|
||||
f,
|
||||
"the feed was checked moments ago; retry in {} s",
|
||||
(retry_after_ms + 999) / 1000
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ManualRefreshError {}
|
||||
|
||||
/// Refreshes one subscribed podcast of one account now, ignoring the
|
||||
/// failure backoff but not more often than every
|
||||
/// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a
|
||||
/// [`ManualRefreshError`] when the refresh was not attempted.
|
||||
pub(crate) async fn refresh_now(
|
||||
store: &PodcastStore,
|
||||
fetcher: &FeedFetcher,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
) -> Result<TargetResult> {
|
||||
let feed_url = match store.podcast(user_id, podcast_id).await? {
|
||||
Some(summary) if summary.subscribed => summary.meta.feed_url,
|
||||
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
|
||||
None => match store.subscription(user_id, podcast_id).await? {
|
||||
Some(register) if register.subscribed => register.podcast.feed_url,
|
||||
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
|
||||
None => return Err(ManualRefreshError::UnknownPodcast.into()),
|
||||
},
|
||||
};
|
||||
if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms {
|
||||
let elapsed = now_ms - checked;
|
||||
if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) {
|
||||
return Err(ManualRefreshError::TooSoon {
|
||||
retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
let target = RefreshTarget {
|
||||
user_id,
|
||||
podcast_id: podcast_id.clone(),
|
||||
};
|
||||
let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?;
|
||||
Ok(results.pop().map(|(_, result)| result).expect("one target"))
|
||||
}
|
||||
@@ -1311,6 +1311,7 @@ async fn run_scheduled_job(
|
||||
&& job_name != "lastfm_scrobble"
|
||||
&& job_name != "archive_cleanup"
|
||||
&& job_name != "artwork_backfill"
|
||||
&& job_name != "podcast_refresh"
|
||||
{
|
||||
tracing::warn!(job = job_name, "Skipping: agent_enabled=false");
|
||||
return;
|
||||
|
||||
Reference in new issue
Block a user