Files
furumi-web/src/podcasts/runner.rs
T

488 lines
17 KiB
Rust

//! Scheduling of feed refreshes: which feeds are due, one request per feed
//! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local
//! fetch state (validators, next check, backoff).
use std::collections::BTreeMap;
use std::fmt;
use std::time::Duration;
use anyhow::Result;
use futures_util::StreamExt;
use music_dht::podcast::{FeedUrl, PodcastId};
use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators};
use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome};
/// Regular interval between two checks of a healthy feed.
pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000;
/// Spread of the regular interval, so feeds added together drift apart.
const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000;
/// First retry delay after a failure; doubles per consecutive failure.
pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
/// Longest delay between two attempts of a failing feed.
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
/// A manual refresh is refused when the feed was checked this recently.
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
/// Podcasts considered per scheduled run. Due podcasts are taken in turns
/// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an
/// account with many (or slow) feeds cannot hold back everyone else's
/// refreshes, but can use a run nobody else needs.
pub(crate) const RUN_LIMIT: usize = 200;
/// Feed requests in flight at once.
pub(crate) const RUN_CONCURRENCY: usize = 4;
/// Delay before the next attempt after `consecutive_failures` failures in a
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
/// day. A longer server-requested `Retry-After` wins (also capped).
pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option<Duration>) -> i64 {
let exponent = consecutive_failures.saturating_sub(1).min(16);
let backoff = BACKOFF_BASE_MS
.saturating_mul(1_i64 << exponent)
.min(BACKOFF_MAX_MS);
let requested = retry_after
.map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX))
.unwrap_or(0);
backoff.max(requested).min(BACKOFF_MAX_MS)
}
/// Next regular check after a success: the interval plus a stable per
/// account and podcast jitter.
pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 {
let key = format!("{user_id}:{podcast_id}");
let hash = blake3::hash(key.as_bytes());
let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes"));
let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0);
now_ms + REFRESH_INTERVAL_MS + jitter
}
/// One podcast of one account to refresh.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RefreshTarget {
pub(crate) user_id: i64,
pub(crate) podcast_id: PodcastId,
}
/// Result of refreshing one [`RefreshTarget`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum TargetResult {
/// A new document was ingested.
Updated(RefreshOutcome),
/// `304 Not Modified`.
NotModified,
/// The fetch or the ingestion failed; the catalog is unchanged.
Failed {
/// What the user sees (stored as the last error).
error: String,
/// Details for the server log (no URLs).
detail: String,
consecutive_failures: u32,
next_check_at_ms: i64,
},
/// Nothing to do any more (unsubscribed or removed meanwhile).
Skipped(String),
}
/// Counters of a scheduled run.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct RunSummary {
/// Account podcasts that were due.
pub(crate) targets: usize,
/// Distinct feed URLs requested.
pub(crate) feeds: usize,
pub(crate) updated: usize,
pub(crate) not_modified: usize,
pub(crate) failed: usize,
pub(crate) skipped: usize,
pub(crate) new_episodes: usize,
pub(crate) updated_episodes: usize,
/// `(podcast_id, details)` of failed targets, for the job log (no
/// account ids or URLs: feed URLs often carry access tokens).
pub(crate) failures: Vec<(PodcastId, String)>,
}
impl RunSummary {
fn add(&mut self, target: &RefreshTarget, result: &TargetResult) {
match result {
TargetResult::Updated(outcome) => {
self.updated += 1;
self.new_episodes += outcome.new_episodes;
self.updated_episodes += outcome.updated_episodes;
}
TargetResult::NotModified => self.not_modified += 1,
TargetResult::Failed { detail, .. } => {
self.failed += 1;
self.failures
.push((target.podcast_id.clone(), detail.clone()));
}
TargetResult::Skipped(_) => self.skipped += 1,
}
}
}
impl fmt::Display for RunSummary {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \
{} not modified, {} failed, {} skipped",
self.targets,
self.feeds,
self.updated,
self.new_episodes,
self.updated_episodes,
self.not_modified,
self.failed,
self.skipped
)
}
}
/// Due podcasts grouped by feed URL: accounts subscribed to the same URL
/// share one request.
pub(crate) type RefreshGroups = BTreeMap<FeedUrl, Vec<RefreshTarget>>;
/// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`].
pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups {
let mut groups = RefreshGroups::new();
for (user_id, podcast_id, feed_url) in due {
groups.entry(feed_url).or_default().push(RefreshTarget {
user_id,
podcast_id,
});
}
groups
}
/// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in
/// turns), with at most
/// `concurrency` feed requests in flight. Accounts subscribed to the same
/// feed URL share one request. A failing feed never stops the others.
pub(crate) async fn refresh_due(
store: &PodcastStore,
fetcher: &FeedFetcher,
now_ms: i64,
limit: usize,
concurrency: usize,
) -> Result<RunSummary> {
let due = store
.podcasts_due_for_refresh(now_ms, limit)
.await?;
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
}
/// Refreshes `groups`, one request per feed URL, at most `concurrency` at
/// a time.
pub(crate) async fn refresh_groups(
store: &PodcastStore,
fetcher: &FeedFetcher,
groups: RefreshGroups,
now_ms: i64,
concurrency: usize,
) -> Result<RunSummary> {
let mut summary = RunSummary {
targets: groups.values().map(Vec::len).sum(),
feeds: groups.len(),
..RunSummary::default()
};
let mut results = futures_util::stream::iter(groups)
.map(|(feed_url, targets)| async move {
refresh_group(store, fetcher, &feed_url, targets, now_ms).await
})
.buffer_unordered(concurrency.max(1));
while let Some(group) = results.next().await {
for (target, result) in group? {
summary.add(&target, &result);
}
}
Ok(summary)
}
/// Fetches `feed_url` once for all `targets` and records the outcome for
/// each. The request is conditional only when every target holds the same
/// validators (the usual case: one account).
pub(crate) async fn refresh_group(
store: &PodcastStore,
fetcher: &FeedFetcher,
feed_url: &FeedUrl,
targets: Vec<RefreshTarget>,
now_ms: i64,
) -> Result<Vec<(RefreshTarget, TargetResult)>> {
let mut states = Vec::with_capacity(targets.len());
for target in &targets {
states.push(
store
.fetch_state(target.user_id, &target.podcast_id)
.await?,
);
}
let validators = states
.iter()
.map(|state| Validators {
etag: state.etag.clone(),
last_modified: state.last_modified.clone(),
})
.reduce(|a, b| if a == b { a } else { Validators::default() })
.unwrap_or_default();
let fetched = fetcher.fetch(feed_url, &validators).await;
let mut out = Vec::with_capacity(targets.len());
for (target, state) in targets.into_iter().zip(states) {
let result =
record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?;
out.push((target, result));
}
Ok(out)
}
/// Ingests (or records the failure of) one fetch for one target.
async fn record_result(
store: &PodcastStore,
target: &RefreshTarget,
fetched: &Result<FeedFetch, FetchError>,
previous_failures: u32,
now_ms: i64,
) -> Result<TargetResult> {
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
let failure = |error: String, detail: String, retry_after: Option<Duration>| {
let consecutive_failures = previous_failures.saturating_add(1);
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
(error, detail, consecutive_failures, next_check_at_ms)
};
let (error, detail, consecutive_failures, next_check_at_ms) = match fetched {
Ok(FeedFetch::NotModified) => {
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
return match store
.record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next))
.await
{
Ok(()) => Ok(TargetResult::NotModified),
Err(error) => skipped_or(error),
};
}
Ok(FeedFetch::Fetched(document)) => {
match ingest(store, target, document, now_ms).await {
Ok(outcome) => return Ok(TargetResult::Updated(outcome)),
Err(error) if is_gone(&error) => return skipped_or(error),
// A store failure is recorded like a fetch failure, so the
// feed backs off instead of being retried every run.
Err(error) => failure(
"the feed could not be stored".to_string(),
format!("storing the feed failed: {error:#}"),
None,
),
}
}
Err(error) => failure(
error.public_message(),
error.message.clone(),
error.retry_after,
),
};
match store
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
.await
{
Ok(_) => Ok(TargetResult::Failed {
error,
detail,
consecutive_failures,
next_check_at_ms,
}),
Err(error) => skipped_or(error),
}
}
async fn ingest(
store: &PodcastStore,
target: &RefreshTarget,
document: &FetchedFeed,
now_ms: i64,
) -> Result<RefreshOutcome> {
let outcome = store
.apply_feed(
target.user_id,
&FeedRefresh {
podcast_id: &target.podcast_id,
feed: &document.feed,
fetched_from: &document.fetched_from,
moved_permanently: document.moved_permanently,
},
)
.await?;
store
.record_fetch_success(
target.user_id,
&target.podcast_id,
now_ms,
document.validators.etag.as_deref(),
document.validators.last_modified.as_deref(),
Some(next_regular_check_ms(
now_ms,
target.user_id,
&target.podcast_id,
)),
)
.await?;
Ok(outcome)
}
fn is_gone(error: &anyhow::Error) -> bool {
matches!(
error.downcast_ref::<PodcastStoreError>(),
Some(PodcastStoreError::UnknownPodcast(_))
)
}
/// A podcast removed from the account meanwhile is skipped; any other
/// store error is returned.
fn skipped_or(error: anyhow::Error) -> Result<TargetResult> {
if is_gone(&error) {
Ok(TargetResult::Skipped(error.to_string()))
} else {
Err(error)
}
}
/// Why a manual refresh was not started.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ManualRefreshError {
/// The account does not know the podcast.
UnknownPodcast,
/// The account is not subscribed (unsubscribing stops monitoring).
NotSubscribed,
/// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago.
TooSoon { retry_after_ms: i64 },
}
impl fmt::Display for ManualRefreshError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownPodcast => f.write_str("unknown podcast"),
Self::NotSubscribed => f.write_str("not subscribed to this podcast"),
Self::TooSoon { retry_after_ms } => write!(
f,
"the feed was checked moments ago; retry in {} s",
(retry_after_ms + 999) / 1000
),
}
}
}
impl std::error::Error for ManualRefreshError {}
/// Refreshes one subscribed podcast of one account now, ignoring the
/// failure backoff but not more often than every
/// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a
/// [`ManualRefreshError`] when the refresh was not attempted.
pub(crate) async fn refresh_now(
store: &PodcastStore,
fetcher: &FeedFetcher,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
) -> Result<TargetResult> {
let feed_url = match store.podcast(user_id, podcast_id).await? {
Some(summary) if summary.subscribed => summary.meta.feed_url,
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
None => match store.subscription(user_id, podcast_id).await? {
Some(register) if register.subscribed => register.podcast.feed_url,
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
None => return Err(ManualRefreshError::UnknownPodcast.into()),
},
};
if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms {
let elapsed = now_ms - checked;
if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) {
return Err(ManualRefreshError::TooSoon {
retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed,
}
.into());
}
}
let target = RefreshTarget {
user_id,
podcast_id: podcast_id.clone(),
};
let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?;
Ok(results.pop().map(|(_, result)| result).expect("one target"))
}
/// Result of [`subscribe_url`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct UrlSubscription {
/// Podcast the URL resolved to (an existing one for a known URL or alias).
pub(crate) podcast_id: PodcastId,
/// The account was already subscribed; no subscription was written.
pub(crate) already_subscribed: bool,
/// Ingestion of the document fetched for the subscription.
pub(crate) refresh: RefreshOutcome,
}
/// Why [`subscribe_url`] did not subscribe.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum SubscribeUrlError {
/// The feed could not be fetched or parsed; nothing was stored.
Fetch(FetchError),
}
impl fmt::Display for SubscribeUrlError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Fetch(error) => {
write!(
f,
"the feed could not be loaded: {}",
error.public_message()
)
}
}
}
}
impl std::error::Error for SubscribeUrlError {}
/// Subscribes an account to the feed at `url` (the "add RSS" action).
///
/// The feed is fetched first: an address that is not public, a document
/// that is not a feed or a failed request subscribes nothing
/// ([`SubscribeUrlError`] inside the error). Then the subscription is
/// written with the channel metadata of the document
/// ([`PodcastStore::subscribe`]: a known URL or alias reuses the existing
/// podcast, its progress and history; an active subscription writes
/// nothing) and the document is ingested like a scheduled refresh, which
/// also stores the validators and the next check.
pub(crate) async fn subscribe_url(
store: &PodcastStore,
fetcher: &FeedFetcher,
user_id: i64,
url: &FeedUrl,
now_ms: i64,
) -> Result<UrlSubscription> {
let document = match fetcher.fetch(url, &Validators::default()).await {
Ok(FeedFetch::Fetched(document)) => document,
Ok(FeedFetch::NotModified) => {
// Unconditional requests are never answered with 304 by a
// conforming server; refuse instead of subscribing blindly.
return Err(SubscribeUrlError::Fetch(FetchError {
kind: super::fetch::FetchErrorKind::Status(304),
message: "unexpected 304 Not Modified".to_string(),
retry_after: None,
})
.into());
}
Err(error) => return Err(SubscribeUrlError::Fetch(error).into()),
};
let subscription = store
.subscribe(user_id, &document.feed.to_podcast_meta(url.clone()))
.await?;
let target = RefreshTarget {
user_id,
podcast_id: subscription.podcast_id.clone(),
};
let refresh = ingest(store, &target, &document, now_ms).await?;
Ok(UrlSubscription {
podcast_id: subscription.podcast_id,
already_subscribed: subscription.already_subscribed,
refresh,
})
}