Feed downloads and proxy streams have per-account and server-wide limits, new subscriptions stop at 1000 per account, and scheduled refreshes take accounts in turns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
489 lines
17 KiB
Rust
489 lines
17 KiB
Rust
//! Scheduling of feed refreshes: which feeds are due, one request per feed
|
|
//! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local
|
|
//! fetch state (validators, next check, backoff).
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::fmt;
|
|
use std::time::Duration;
|
|
|
|
use anyhow::Result;
|
|
use futures_util::StreamExt;
|
|
use music_dht::podcast::{FeedUrl, PodcastId};
|
|
|
|
use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators};
|
|
use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome};
|
|
|
|
/// Regular interval between two checks of a healthy feed.
|
|
pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000;
|
|
/// Spread of the regular interval, so feeds added together drift apart.
|
|
const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000;
|
|
/// First retry delay after a failure; doubles per consecutive failure.
|
|
pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
|
|
/// Longest delay between two attempts of a failing feed.
|
|
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
|
|
/// A manual refresh is refused when the feed was checked this recently.
|
|
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
|
|
/// Podcasts considered per scheduled run.
|
|
pub(crate) const RUN_LIMIT: usize = 200;
|
|
/// Feed requests in flight at once.
|
|
pub(crate) const RUN_CONCURRENCY: usize = 4;
|
|
/// Podcasts of one account considered per scheduled run. Due podcasts are
|
|
/// taken round-robin across accounts, so an account with many (or slow)
|
|
/// feeds cannot hold back everyone else's refreshes.
|
|
pub(crate) const RUN_PER_ACCOUNT: usize = 25;
|
|
|
|
/// Delay before the next attempt after `consecutive_failures` failures in a
|
|
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
|
|
/// day. A longer server-requested `Retry-After` wins (also capped).
|
|
pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option<Duration>) -> i64 {
|
|
let exponent = consecutive_failures.saturating_sub(1).min(16);
|
|
let backoff = BACKOFF_BASE_MS
|
|
.saturating_mul(1_i64 << exponent)
|
|
.min(BACKOFF_MAX_MS);
|
|
let requested = retry_after
|
|
.map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX))
|
|
.unwrap_or(0);
|
|
backoff.max(requested).min(BACKOFF_MAX_MS)
|
|
}
|
|
|
|
/// Next regular check after a success: the interval plus a stable per
|
|
/// account and podcast jitter.
|
|
pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 {
|
|
let key = format!("{user_id}:{podcast_id}");
|
|
let hash = blake3::hash(key.as_bytes());
|
|
let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes"));
|
|
let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0);
|
|
now_ms + REFRESH_INTERVAL_MS + jitter
|
|
}
|
|
|
|
/// One podcast of one account to refresh.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) struct RefreshTarget {
|
|
pub(crate) user_id: i64,
|
|
pub(crate) podcast_id: PodcastId,
|
|
}
|
|
|
|
/// Result of refreshing one [`RefreshTarget`].
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub(crate) enum TargetResult {
|
|
/// A new document was ingested.
|
|
Updated(RefreshOutcome),
|
|
/// `304 Not Modified`.
|
|
NotModified,
|
|
/// The fetch or the ingestion failed; the catalog is unchanged.
|
|
Failed {
|
|
/// What the user sees (stored as the last error).
|
|
error: String,
|
|
/// Details for the server log (no URLs).
|
|
detail: String,
|
|
consecutive_failures: u32,
|
|
next_check_at_ms: i64,
|
|
},
|
|
/// Nothing to do any more (unsubscribed or removed meanwhile).
|
|
Skipped(String),
|
|
}
|
|
|
|
/// Counters of a scheduled run.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub(crate) struct RunSummary {
|
|
/// Account podcasts that were due.
|
|
pub(crate) targets: usize,
|
|
/// Distinct feed URLs requested.
|
|
pub(crate) feeds: usize,
|
|
pub(crate) updated: usize,
|
|
pub(crate) not_modified: usize,
|
|
pub(crate) failed: usize,
|
|
pub(crate) skipped: usize,
|
|
pub(crate) new_episodes: usize,
|
|
pub(crate) updated_episodes: usize,
|
|
/// `(podcast_id, details)` of failed targets, for the job log (no
|
|
/// account ids or URLs: feed URLs often carry access tokens).
|
|
pub(crate) failures: Vec<(PodcastId, String)>,
|
|
}
|
|
|
|
impl RunSummary {
|
|
fn add(&mut self, target: &RefreshTarget, result: &TargetResult) {
|
|
match result {
|
|
TargetResult::Updated(outcome) => {
|
|
self.updated += 1;
|
|
self.new_episodes += outcome.new_episodes;
|
|
self.updated_episodes += outcome.updated_episodes;
|
|
}
|
|
TargetResult::NotModified => self.not_modified += 1,
|
|
TargetResult::Failed { detail, .. } => {
|
|
self.failed += 1;
|
|
self.failures
|
|
.push((target.podcast_id.clone(), detail.clone()));
|
|
}
|
|
TargetResult::Skipped(_) => self.skipped += 1,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl fmt::Display for RunSummary {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(
|
|
f,
|
|
"{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \
|
|
{} not modified, {} failed, {} skipped",
|
|
self.targets,
|
|
self.feeds,
|
|
self.updated,
|
|
self.new_episodes,
|
|
self.updated_episodes,
|
|
self.not_modified,
|
|
self.failed,
|
|
self.skipped
|
|
)
|
|
}
|
|
}
|
|
|
|
/// Due podcasts grouped by feed URL: accounts subscribed to the same URL
|
|
/// share one request.
|
|
pub(crate) type RefreshGroups = BTreeMap<FeedUrl, Vec<RefreshTarget>>;
|
|
|
|
/// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`].
|
|
pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups {
|
|
let mut groups = RefreshGroups::new();
|
|
for (user_id, podcast_id, feed_url) in due {
|
|
groups.entry(feed_url).or_default().push(RefreshTarget {
|
|
user_id,
|
|
podcast_id,
|
|
});
|
|
}
|
|
groups
|
|
}
|
|
|
|
/// Refreshes podcasts due at `now_ms` (at most `limit`, at most
|
|
/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most
|
|
/// `concurrency` feed requests in flight. Accounts subscribed to the same
|
|
/// feed URL share one request. A failing feed never stops the others.
|
|
pub(crate) async fn refresh_due(
|
|
store: &PodcastStore,
|
|
fetcher: &FeedFetcher,
|
|
now_ms: i64,
|
|
limit: usize,
|
|
concurrency: usize,
|
|
) -> Result<RunSummary> {
|
|
let due = store
|
|
.podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT)
|
|
.await?;
|
|
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
|
|
}
|
|
|
|
/// Refreshes `groups`, one request per feed URL, at most `concurrency` at
|
|
/// a time.
|
|
pub(crate) async fn refresh_groups(
|
|
store: &PodcastStore,
|
|
fetcher: &FeedFetcher,
|
|
groups: RefreshGroups,
|
|
now_ms: i64,
|
|
concurrency: usize,
|
|
) -> Result<RunSummary> {
|
|
let mut summary = RunSummary {
|
|
targets: groups.values().map(Vec::len).sum(),
|
|
feeds: groups.len(),
|
|
..RunSummary::default()
|
|
};
|
|
let mut results = futures_util::stream::iter(groups)
|
|
.map(|(feed_url, targets)| async move {
|
|
refresh_group(store, fetcher, &feed_url, targets, now_ms).await
|
|
})
|
|
.buffer_unordered(concurrency.max(1));
|
|
while let Some(group) = results.next().await {
|
|
for (target, result) in group? {
|
|
summary.add(&target, &result);
|
|
}
|
|
}
|
|
Ok(summary)
|
|
}
|
|
|
|
/// Fetches `feed_url` once for all `targets` and records the outcome for
|
|
/// each. The request is conditional only when every target holds the same
|
|
/// validators (the usual case: one account).
|
|
pub(crate) async fn refresh_group(
|
|
store: &PodcastStore,
|
|
fetcher: &FeedFetcher,
|
|
feed_url: &FeedUrl,
|
|
targets: Vec<RefreshTarget>,
|
|
now_ms: i64,
|
|
) -> Result<Vec<(RefreshTarget, TargetResult)>> {
|
|
let mut states = Vec::with_capacity(targets.len());
|
|
for target in &targets {
|
|
states.push(
|
|
store
|
|
.fetch_state(target.user_id, &target.podcast_id)
|
|
.await?,
|
|
);
|
|
}
|
|
let validators = states
|
|
.iter()
|
|
.map(|state| Validators {
|
|
etag: state.etag.clone(),
|
|
last_modified: state.last_modified.clone(),
|
|
})
|
|
.reduce(|a, b| if a == b { a } else { Validators::default() })
|
|
.unwrap_or_default();
|
|
let fetched = fetcher.fetch(feed_url, &validators).await;
|
|
let mut out = Vec::with_capacity(targets.len());
|
|
for (target, state) in targets.into_iter().zip(states) {
|
|
let result =
|
|
record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?;
|
|
out.push((target, result));
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Ingests (or records the failure of) one fetch for one target.
|
|
async fn record_result(
|
|
store: &PodcastStore,
|
|
target: &RefreshTarget,
|
|
fetched: &Result<FeedFetch, FetchError>,
|
|
previous_failures: u32,
|
|
now_ms: i64,
|
|
) -> Result<TargetResult> {
|
|
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
|
|
let failure = |error: String, detail: String, retry_after: Option<Duration>| {
|
|
let consecutive_failures = previous_failures.saturating_add(1);
|
|
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
|
|
(error, detail, consecutive_failures, next_check_at_ms)
|
|
};
|
|
let (error, detail, consecutive_failures, next_check_at_ms) = match fetched {
|
|
Ok(FeedFetch::NotModified) => {
|
|
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
|
|
return match store
|
|
.record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next))
|
|
.await
|
|
{
|
|
Ok(()) => Ok(TargetResult::NotModified),
|
|
Err(error) => skipped_or(error),
|
|
};
|
|
}
|
|
Ok(FeedFetch::Fetched(document)) => {
|
|
match ingest(store, target, document, now_ms).await {
|
|
Ok(outcome) => return Ok(TargetResult::Updated(outcome)),
|
|
Err(error) if is_gone(&error) => return skipped_or(error),
|
|
// A store failure is recorded like a fetch failure, so the
|
|
// feed backs off instead of being retried every run.
|
|
Err(error) => failure(
|
|
"the feed could not be stored".to_string(),
|
|
format!("storing the feed failed: {error:#}"),
|
|
None,
|
|
),
|
|
}
|
|
}
|
|
Err(error) => failure(
|
|
error.public_message(),
|
|
error.message.clone(),
|
|
error.retry_after,
|
|
),
|
|
};
|
|
match store
|
|
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
|
|
.await
|
|
{
|
|
Ok(_) => Ok(TargetResult::Failed {
|
|
error,
|
|
detail,
|
|
consecutive_failures,
|
|
next_check_at_ms,
|
|
}),
|
|
Err(error) => skipped_or(error),
|
|
}
|
|
}
|
|
|
|
async fn ingest(
|
|
store: &PodcastStore,
|
|
target: &RefreshTarget,
|
|
document: &FetchedFeed,
|
|
now_ms: i64,
|
|
) -> Result<RefreshOutcome> {
|
|
let outcome = store
|
|
.apply_feed(
|
|
target.user_id,
|
|
&FeedRefresh {
|
|
podcast_id: &target.podcast_id,
|
|
feed: &document.feed,
|
|
fetched_from: &document.fetched_from,
|
|
moved_permanently: document.moved_permanently,
|
|
},
|
|
)
|
|
.await?;
|
|
store
|
|
.record_fetch_success(
|
|
target.user_id,
|
|
&target.podcast_id,
|
|
now_ms,
|
|
document.validators.etag.as_deref(),
|
|
document.validators.last_modified.as_deref(),
|
|
Some(next_regular_check_ms(
|
|
now_ms,
|
|
target.user_id,
|
|
&target.podcast_id,
|
|
)),
|
|
)
|
|
.await?;
|
|
Ok(outcome)
|
|
}
|
|
|
|
fn is_gone(error: &anyhow::Error) -> bool {
|
|
matches!(
|
|
error.downcast_ref::<PodcastStoreError>(),
|
|
Some(PodcastStoreError::UnknownPodcast(_))
|
|
)
|
|
}
|
|
|
|
/// A podcast removed from the account meanwhile is skipped; any other
|
|
/// store error is returned.
|
|
fn skipped_or(error: anyhow::Error) -> Result<TargetResult> {
|
|
if is_gone(&error) {
|
|
Ok(TargetResult::Skipped(error.to_string()))
|
|
} else {
|
|
Err(error)
|
|
}
|
|
}
|
|
|
|
/// Why a manual refresh was not started.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) enum ManualRefreshError {
|
|
/// The account does not know the podcast.
|
|
UnknownPodcast,
|
|
/// The account is not subscribed (unsubscribing stops monitoring).
|
|
NotSubscribed,
|
|
/// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago.
|
|
TooSoon { retry_after_ms: i64 },
|
|
}
|
|
|
|
impl fmt::Display for ManualRefreshError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
Self::UnknownPodcast => f.write_str("unknown podcast"),
|
|
Self::NotSubscribed => f.write_str("not subscribed to this podcast"),
|
|
Self::TooSoon { retry_after_ms } => write!(
|
|
f,
|
|
"the feed was checked moments ago; retry in {} s",
|
|
(retry_after_ms + 999) / 1000
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for ManualRefreshError {}
|
|
|
|
/// Refreshes one subscribed podcast of one account now, ignoring the
|
|
/// failure backoff but not more often than every
|
|
/// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a
|
|
/// [`ManualRefreshError`] when the refresh was not attempted.
|
|
pub(crate) async fn refresh_now(
|
|
store: &PodcastStore,
|
|
fetcher: &FeedFetcher,
|
|
user_id: i64,
|
|
podcast_id: &PodcastId,
|
|
now_ms: i64,
|
|
) -> Result<TargetResult> {
|
|
let feed_url = match store.podcast(user_id, podcast_id).await? {
|
|
Some(summary) if summary.subscribed => summary.meta.feed_url,
|
|
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
|
|
None => match store.subscription(user_id, podcast_id).await? {
|
|
Some(register) if register.subscribed => register.podcast.feed_url,
|
|
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
|
|
None => return Err(ManualRefreshError::UnknownPodcast.into()),
|
|
},
|
|
};
|
|
if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms {
|
|
let elapsed = now_ms - checked;
|
|
if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) {
|
|
return Err(ManualRefreshError::TooSoon {
|
|
retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed,
|
|
}
|
|
.into());
|
|
}
|
|
}
|
|
let target = RefreshTarget {
|
|
user_id,
|
|
podcast_id: podcast_id.clone(),
|
|
};
|
|
let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?;
|
|
Ok(results.pop().map(|(_, result)| result).expect("one target"))
|
|
}
|
|
|
|
/// Result of [`subscribe_url`].
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub(crate) struct UrlSubscription {
|
|
/// Podcast the URL resolved to (an existing one for a known URL or alias).
|
|
pub(crate) podcast_id: PodcastId,
|
|
/// The account was already subscribed; no subscription was written.
|
|
pub(crate) already_subscribed: bool,
|
|
/// Ingestion of the document fetched for the subscription.
|
|
pub(crate) refresh: RefreshOutcome,
|
|
}
|
|
|
|
/// Why [`subscribe_url`] did not subscribe.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) enum SubscribeUrlError {
|
|
/// The feed could not be fetched or parsed; nothing was stored.
|
|
Fetch(FetchError),
|
|
}
|
|
|
|
impl fmt::Display for SubscribeUrlError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
Self::Fetch(error) => {
|
|
write!(
|
|
f,
|
|
"the feed could not be loaded: {}",
|
|
error.public_message()
|
|
)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for SubscribeUrlError {}
|
|
|
|
/// Subscribes an account to the feed at `url` (the "add RSS" action).
|
|
///
|
|
/// The feed is fetched first: an address that is not public, a document
|
|
/// that is not a feed or a failed request subscribes nothing
|
|
/// ([`SubscribeUrlError`] inside the error). Then the subscription is
|
|
/// written with the channel metadata of the document
|
|
/// ([`PodcastStore::subscribe`]: a known URL or alias reuses the existing
|
|
/// podcast, its progress and history; an active subscription writes
|
|
/// nothing) and the document is ingested like a scheduled refresh, which
|
|
/// also stores the validators and the next check.
|
|
pub(crate) async fn subscribe_url(
|
|
store: &PodcastStore,
|
|
fetcher: &FeedFetcher,
|
|
user_id: i64,
|
|
url: &FeedUrl,
|
|
now_ms: i64,
|
|
) -> Result<UrlSubscription> {
|
|
let document = match fetcher.fetch(url, &Validators::default()).await {
|
|
Ok(FeedFetch::Fetched(document)) => document,
|
|
Ok(FeedFetch::NotModified) => {
|
|
// Unconditional requests are never answered with 304 by a
|
|
// conforming server; refuse instead of subscribing blindly.
|
|
return Err(SubscribeUrlError::Fetch(FetchError {
|
|
kind: super::fetch::FetchErrorKind::Status(304),
|
|
message: "unexpected 304 Not Modified".to_string(),
|
|
retry_after: None,
|
|
})
|
|
.into());
|
|
}
|
|
Err(error) => return Err(SubscribeUrlError::Fetch(error).into()),
|
|
};
|
|
let subscription = store
|
|
.subscribe(user_id, &document.feed.to_podcast_meta(url.clone()))
|
|
.await?;
|
|
let target = RefreshTarget {
|
|
user_id,
|
|
podcast_id: subscription.podcast_id.clone(),
|
|
};
|
|
let refresh = ingest(store, &target, &document, now_ms).await?;
|
|
Ok(UrlSubscription {
|
|
podcast_id: subscription.podcast_id,
|
|
already_subscribed: subscription.already_subscribed,
|
|
refresh,
|
|
})
|
|
}
|