//! Scheduling of feed refreshes: which feeds are due, one request per feed //! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local //! fetch state (validators, next check, backoff). use std::collections::BTreeMap; use std::fmt; use std::time::Duration; use anyhow::Result; use futures_util::StreamExt; use music_dht::podcast::{FeedUrl, PodcastId}; use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators}; use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome}; /// Regular interval between two checks of a healthy feed. pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000; /// Spread of the regular interval, so feeds added together drift apart. const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000; /// First retry delay after a failure; doubles per consecutive failure. pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000; /// Longest delay between two attempts of a failing feed. pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000; /// A manual refresh is refused when the feed was checked this recently. pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000; /// Podcasts considered per scheduled run. Due podcasts are taken in turns /// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an /// account with many (or slow) feeds cannot hold back everyone else's /// refreshes, but can use a run nobody else needs. pub(crate) const RUN_LIMIT: usize = 200; /// Feed requests in flight at once. pub(crate) const RUN_CONCURRENCY: usize = 4; /// Delay before the next attempt after `consecutive_failures` failures in a /// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a /// day. A longer server-requested `Retry-After` wins (also capped). pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option) -> i64 { let exponent = consecutive_failures.saturating_sub(1).min(16); let backoff = BACKOFF_BASE_MS .saturating_mul(1_i64 << exponent) .min(BACKOFF_MAX_MS); let requested = retry_after .map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX)) .unwrap_or(0); backoff.max(requested).min(BACKOFF_MAX_MS) } /// Next regular check after a success: the interval plus a stable per /// account and podcast jitter. pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 { let key = format!("{user_id}:{podcast_id}"); let hash = blake3::hash(key.as_bytes()); let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes")); let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0); now_ms + REFRESH_INTERVAL_MS + jitter } /// One podcast of one account to refresh. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct RefreshTarget { pub(crate) user_id: i64, pub(crate) podcast_id: PodcastId, } /// Result of refreshing one [`RefreshTarget`]. #[derive(Debug, Clone, PartialEq)] pub(crate) enum TargetResult { /// A new document was ingested. Updated(RefreshOutcome), /// `304 Not Modified`. NotModified, /// The fetch or the ingestion failed; the catalog is unchanged. Failed { /// What the user sees (stored as the last error). error: String, /// Details for the server log (no URLs). detail: String, consecutive_failures: u32, next_check_at_ms: i64, }, /// Nothing to do any more (unsubscribed or removed meanwhile). Skipped(String), } /// Counters of a scheduled run. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub(crate) struct RunSummary { /// Account podcasts that were due. pub(crate) targets: usize, /// Distinct feed URLs requested. pub(crate) feeds: usize, pub(crate) updated: usize, pub(crate) not_modified: usize, pub(crate) failed: usize, pub(crate) skipped: usize, pub(crate) new_episodes: usize, pub(crate) updated_episodes: usize, /// `(podcast_id, details)` of failed targets, for the job log (no /// account ids or URLs: feed URLs often carry access tokens). pub(crate) failures: Vec<(PodcastId, String)>, } impl RunSummary { fn add(&mut self, target: &RefreshTarget, result: &TargetResult) { match result { TargetResult::Updated(outcome) => { self.updated += 1; self.new_episodes += outcome.new_episodes; self.updated_episodes += outcome.updated_episodes; } TargetResult::NotModified => self.not_modified += 1, TargetResult::Failed { detail, .. } => { self.failed += 1; self.failures .push((target.podcast_id.clone(), detail.clone())); } TargetResult::Skipped(_) => self.skipped += 1, } } } impl fmt::Display for RunSummary { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \ {} not modified, {} failed, {} skipped", self.targets, self.feeds, self.updated, self.new_episodes, self.updated_episodes, self.not_modified, self.failed, self.skipped ) } } /// Due podcasts grouped by feed URL: accounts subscribed to the same URL /// share one request. pub(crate) type RefreshGroups = BTreeMap>; /// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`]. pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups { let mut groups = RefreshGroups::new(); for (user_id, podcast_id, feed_url) in due { groups.entry(feed_url).or_default().push(RefreshTarget { user_id, podcast_id, }); } groups } /// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in /// turns), with at most /// `concurrency` feed requests in flight. Accounts subscribed to the same /// feed URL share one request. A failing feed never stops the others. pub(crate) async fn refresh_due( store: &PodcastStore, fetcher: &FeedFetcher, now_ms: i64, limit: usize, concurrency: usize, ) -> Result { let due = store.podcasts_due_for_refresh(now_ms, limit).await?; refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await } /// Refreshes `groups`, one request per feed URL, at most `concurrency` at /// a time. pub(crate) async fn refresh_groups( store: &PodcastStore, fetcher: &FeedFetcher, groups: RefreshGroups, now_ms: i64, concurrency: usize, ) -> Result { let mut summary = RunSummary { targets: groups.values().map(Vec::len).sum(), feeds: groups.len(), ..RunSummary::default() }; let mut results = futures_util::stream::iter(groups) .map(|(feed_url, targets)| async move { refresh_group(store, fetcher, &feed_url, targets, now_ms).await }) .buffer_unordered(concurrency.max(1)); while let Some(group) = results.next().await { for (target, result) in group? { summary.add(&target, &result); } } Ok(summary) } /// Fetches `feed_url` once for all `targets` and records the outcome for /// each. The request is conditional only when every target holds the same /// validators (the usual case: one account). pub(crate) async fn refresh_group( store: &PodcastStore, fetcher: &FeedFetcher, feed_url: &FeedUrl, targets: Vec, now_ms: i64, ) -> Result> { let mut states = Vec::with_capacity(targets.len()); for target in &targets { states.push( store .fetch_state(target.user_id, &target.podcast_id) .await?, ); } let validators = states .iter() .map(|state| Validators { etag: state.etag.clone(), last_modified: state.last_modified.clone(), }) .reduce(|a, b| if a == b { a } else { Validators::default() }) .unwrap_or_default(); let fetched = fetcher.fetch(feed_url, &validators).await; let mut out = Vec::with_capacity(targets.len()); for (target, state) in targets.into_iter().zip(states) { let result = record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?; out.push((target, result)); } Ok(out) } /// Ingests (or records the failure of) one fetch for one target. async fn record_result( store: &PodcastStore, target: &RefreshTarget, fetched: &Result, previous_failures: u32, now_ms: i64, ) -> Result { let (user_id, podcast_id) = (target.user_id, &target.podcast_id); let failure = |error: String, detail: String, retry_after: Option| { let consecutive_failures = previous_failures.saturating_add(1); let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after); (error, detail, consecutive_failures, next_check_at_ms) }; let (error, detail, consecutive_failures, next_check_at_ms) = match fetched { Ok(FeedFetch::NotModified) => { let next = next_regular_check_ms(now_ms, user_id, podcast_id); return match store .record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next)) .await { Ok(()) => Ok(TargetResult::NotModified), Err(error) => skipped_or(error), }; } Ok(FeedFetch::Fetched(document)) => { match ingest(store, target, document, now_ms).await { Ok(outcome) => return Ok(TargetResult::Updated(outcome)), Err(error) if is_gone(&error) => return skipped_or(error), // A store failure is recorded like a fetch failure, so the // feed backs off instead of being retried every run. Err(error) => failure( "the feed could not be stored".to_string(), format!("storing the feed failed: {error:#}"), None, ), } } Err(error) => failure( error.public_message(), error.message.clone(), error.retry_after, ), }; match store .record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms)) .await { Ok(_) => Ok(TargetResult::Failed { error, detail, consecutive_failures, next_check_at_ms, }), Err(error) => skipped_or(error), } } async fn ingest( store: &PodcastStore, target: &RefreshTarget, document: &FetchedFeed, now_ms: i64, ) -> Result { let outcome = store .apply_feed( target.user_id, &FeedRefresh { podcast_id: &target.podcast_id, feed: &document.feed, fetched_from: &document.fetched_from, moved_permanently: document.moved_permanently, }, ) .await?; store .record_fetch_success( target.user_id, &target.podcast_id, now_ms, document.validators.etag.as_deref(), document.validators.last_modified.as_deref(), Some(next_regular_check_ms( now_ms, target.user_id, &target.podcast_id, )), ) .await?; Ok(outcome) } fn is_gone(error: &anyhow::Error) -> bool { matches!( error.downcast_ref::(), Some(PodcastStoreError::UnknownPodcast(_)) ) } /// A podcast removed from the account meanwhile is skipped; any other /// store error is returned. fn skipped_or(error: anyhow::Error) -> Result { if is_gone(&error) { Ok(TargetResult::Skipped(error.to_string())) } else { Err(error) } } /// Why a manual refresh was not started. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum ManualRefreshError { /// The account does not know the podcast. UnknownPodcast, /// The account is not subscribed (unsubscribing stops monitoring). NotSubscribed, /// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago. TooSoon { retry_after_ms: i64 }, } impl fmt::Display for ManualRefreshError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::UnknownPodcast => f.write_str("unknown podcast"), Self::NotSubscribed => f.write_str("not subscribed to this podcast"), Self::TooSoon { retry_after_ms } => write!( f, "the feed was checked moments ago; retry in {} s", (retry_after_ms + 999) / 1000 ), } } } impl std::error::Error for ManualRefreshError {} /// Refreshes one subscribed podcast of one account now, ignoring the /// failure backoff but not more often than every /// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a /// [`ManualRefreshError`] when the refresh was not attempted. pub(crate) async fn refresh_now( store: &PodcastStore, fetcher: &FeedFetcher, user_id: i64, podcast_id: &PodcastId, now_ms: i64, ) -> Result { let feed_url = match store.podcast(user_id, podcast_id).await? { Some(summary) if summary.subscribed => summary.meta.feed_url, Some(_) => return Err(ManualRefreshError::NotSubscribed.into()), None => match store.subscription(user_id, podcast_id).await? { Some(register) if register.subscribed => register.podcast.feed_url, Some(_) => return Err(ManualRefreshError::NotSubscribed.into()), None => return Err(ManualRefreshError::UnknownPodcast.into()), }, }; if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms { let elapsed = now_ms - checked; if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) { return Err(ManualRefreshError::TooSoon { retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed, } .into()); } } let target = RefreshTarget { user_id, podcast_id: podcast_id.clone(), }; let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?; Ok(results.pop().map(|(_, result)| result).expect("one target")) } /// Result of [`subscribe_url`]. #[derive(Debug, Clone, PartialEq)] pub(crate) struct UrlSubscription { /// Podcast the URL resolved to (an existing one for a known URL or alias). pub(crate) podcast_id: PodcastId, /// The account was already subscribed; no subscription was written. pub(crate) already_subscribed: bool, /// Ingestion of the document fetched for the subscription. pub(crate) refresh: RefreshOutcome, } /// Why [`subscribe_url`] did not subscribe. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum SubscribeUrlError { /// The feed could not be fetched or parsed; nothing was stored. Fetch(FetchError), } impl fmt::Display for SubscribeUrlError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Fetch(error) => { write!( f, "the feed could not be loaded: {}", error.public_message() ) } } } } impl std::error::Error for SubscribeUrlError {} /// Subscribes an account to the feed at `url` (the "add RSS" action). /// /// The feed is fetched first: an address that is not public, a document /// that is not a feed or a failed request subscribes nothing /// ([`SubscribeUrlError`] inside the error). Then the subscription is /// written with the channel metadata of the document /// ([`PodcastStore::subscribe`]: a known URL or alias reuses the existing /// podcast, its progress and history; an active subscription writes /// nothing) and the document is ingested like a scheduled refresh, which /// also stores the validators and the next check. pub(crate) async fn subscribe_url( store: &PodcastStore, fetcher: &FeedFetcher, user_id: i64, url: &FeedUrl, now_ms: i64, ) -> Result { let document = match fetcher.fetch(url, &Validators::default()).await { Ok(FeedFetch::Fetched(document)) => document, Ok(FeedFetch::NotModified) => { // Unconditional requests are never answered with 304 by a // conforming server; refuse instead of subscribing blindly. return Err(SubscribeUrlError::Fetch(FetchError { kind: super::fetch::FetchErrorKind::Status(304), message: "unexpected 304 Not Modified".to_string(), retry_after: None, }) .into()); } Err(error) => return Err(SubscribeUrlError::Fetch(error).into()), }; let subscription = store .subscribe(user_id, &document.feed.to_podcast_meta(url.clone())) .await?; let target = RefreshTarget { user_id, podcast_id: subscription.podcast_id.clone(), }; let refresh = ingest(store, &target, &document, now_ms).await?; Ok(UrlSubscription { podcast_id: subscription.podcast_id, already_subscribed: subscription.already_subscribed, refresh, }) }