diff --git a/docs/administration.md b/docs/administration.md index 9d3db2f..067d9d1 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -120,7 +120,13 @@ The `podcast_refresh` job (every five minutes, listed under Jobs) checks the feeds of subscribed podcasts: each feed about once an hour with conditional requests, failing feeds less and less often (up to once a day). It downloads feed metadata only, never episode audio. A failing feed keeps its known -episodes. Accounts subscribed to the same feed share one request. +episodes. Accounts subscribed to the same feed share one request. Each run +checks at most 200 feeds, taking accounts in turns (at most 25 feeds of one +account per run), so an account with many or slow feeds does not delay the +others. An account can add up to 1000 subscriptions, and can have two feed +downloads ("add RSS", Refresh) and four proxied audio streams in progress at +once (16 and 64 for the whole server); beyond that the request is refused +with HTTP 429. The server therefore makes outgoing HTTP(S) requests to podcast publishers. It only connects to public addresses on the web ports 80, 443, 8080 and 8443: diff --git a/src/podcasts/actions.rs b/src/podcasts/actions.rs index 9414318..2607a9f 100644 --- a/src/podcasts/actions.rs +++ b/src/podcasts/actions.rs @@ -216,6 +216,18 @@ impl PodcastStore { ops: Vec::new(), }); } + let active: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM furumusic__podcast_subscription + WHERE user_id = $1 AND subscribed", + ) + .bind(user_id) + .fetch_one(&mut *tx) + .await?; + if usize::try_from(active).unwrap_or(usize::MAX) >= super::MAX_SUBSCRIPTIONS_PER_ACCOUNT { + return Err( + PodcastStoreError::SubscriptionLimit(super::MAX_SUBSCRIPTIONS_PER_ACCOUNT).into(), + ); + } let share = apply::load_share(&mut tx, user_id, &podcast_id).await?; let generation = next_podcast_generation(current.as_ref(), share.as_ref()); let aliases = merge_feed_aliases( diff --git a/src/podcasts/api.rs b/src/podcasts/api.rs index fab96bc..586507c 100644 --- a/src/podcasts/api.rs +++ b/src/podcasts/api.rs @@ -73,6 +73,7 @@ impl From for ApiError { PodcastStoreError::UnknownPodcast(_) => Self::not_found(error.to_string()), PodcastStoreError::NotSubscribed(_) => Self::new(409, error.to_string()), PodcastStoreError::Invalid(_) => Self::bad_request(error.to_string()), + PodcastStoreError::SubscriptionLimit(_) => Self::new(422, error.to_string()), }; } if let Some(SubscribeUrlError::Fetch(fetch)) = error.downcast_ref::() { @@ -975,6 +976,10 @@ mod tests { assert_eq!(not_subscribed.status, 409); let invalid: ApiError = super::super::invalid("bad").into(); assert_eq!(invalid.status, 400); + let limit: ApiError = + anyhow::Error::from(PodcastStoreError::SubscriptionLimit(1000)).into(); + assert_eq!(limit.status, 422); + assert_eq!(limit.message, "subscription limit reached (1000 podcasts)"); let refused: ApiError = anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError { kind: FetchErrorKind::Refused, diff --git a/src/podcasts/http.rs b/src/podcasts/http.rs index a3a6afb..45f2fd1 100644 --- a/src/podcasts/http.rs +++ b/src/podcasts/http.rs @@ -52,6 +52,7 @@ use serde::{Deserialize, Serialize}; use super::api::{self, ApiError, EpisodeAction, EpisodeListParams}; use super::fetch::FeedFetcher; +use super::limits::{self, ConcurrencyLimit, Permit}; use super::proxy::AudioProxy; use super::runner::{self, ManualRefreshError, TargetResult}; use super::{PodcastStore, now_ms}; @@ -104,6 +105,22 @@ fn unauthorized() -> cot::response::Response { json_error(StatusCode::UNAUTHORIZED, "not authenticated") } +/// 429 when an account or the server has too many requests of a kind in +/// flight (see [`limits`]). +fn too_busy(message: &str) -> cot::response::Response { + let mut response = json_error(StatusCode::TOO_MANY_REQUESTS, message); + response.headers_mut().insert( + RETRY_AFTER, + limits::RETRY_AFTER_SECS + .to_string() + .parse() + .expect("digits"), + ); + response +} + +const FEEDS_BUSY: &str = "too many feed requests in progress; try again in a few seconds"; + /// Largest JSON body accepted (a playback request carries one episode reference). const MAX_JSON_BODY: usize = 64 * 1024; @@ -151,9 +168,17 @@ struct State { pool: Arc>, fetcher: Arc>, proxy: Arc>, + /// Feed downloads users started ("add RSS", manual refresh). + feed_fetches: ConcurrencyLimit, + /// Audio proxy responses being streamed. + proxy_streams: ConcurrencyLimit, } impl State { + fn feed_fetch(&self, user_id: i64) -> Option { + self.feed_fetches.try_acquire(user_id) + } + async fn store(&self) -> cot::Result { let pool = self .pool @@ -196,6 +221,14 @@ impl PodcastsApp { pool: Arc::new(tokio::sync::OnceCell::new()), fetcher: Arc::new(tokio::sync::OnceCell::new()), proxy: Arc::new(tokio::sync::OnceCell::new()), + feed_fetches: ConcurrencyLimit::new( + limits::FEED_FETCHES_PER_ACCOUNT, + limits::FEED_FETCHES_TOTAL, + ), + proxy_streams: ConcurrencyLimit::new( + limits::PROXY_STREAMS_PER_ACCOUNT, + limits::PROXY_STREAMS_TOTAL, + ), }, } } @@ -281,6 +314,9 @@ async fn refresh_handler( let Ok(podcast_id) = PodcastId::parse(&path.0.podcast_id) else { return Ok(json_error(StatusCode::BAD_REQUEST, "invalid podcast id")); }; + let Some(_permit) = state.feed_fetch(user.id) else { + return Ok(too_busy(FEEDS_BUSY)); + }; let store = state.store().await?; let fetcher = state.fetcher().await?; match runner::refresh_now(&store, fetcher, user.id, &podcast_id, now_ms()).await { @@ -389,6 +425,10 @@ async fn audio_handler( .headers() .get(RANGE) .and_then(|value| value.to_str().ok()); + // Held until the body is fully sent or the browser goes away. + let Some(permit) = state.proxy_streams.try_acquire(user.id) else { + return Ok(too_busy("too many podcast audio streams in progress")); + }; let proxy = state.proxy().await?; let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await { Ok(audio) => audio, @@ -418,9 +458,10 @@ async fn audio_handler( if audio.accept_ranges { builder = builder.header(ACCEPT_RANGES, "bytes"); } - let body = audio - .body - .map_err(|error| cot::Error::internal(error.to_string())); + let body = audio.body.map_err(move |error| { + let _held = &permit; + cot::Error::internal(error.to_string()) + }); builder.body(Body::streaming(body)).map_err(internal) } @@ -459,6 +500,9 @@ async fn subscribe_url_handler( Ok(body) => body, Err(error) => return Ok(api_error(error)), }; + let Some(_permit) = state.feed_fetch(user_id) else { + return Ok(too_busy(FEEDS_BUSY)); + }; let store = state.store().await?; let fetcher = state.fetcher().await?; api_response(api::subscribe_url(&store, fetcher, user_id, body, now_ms()).await) diff --git a/src/podcasts/limits.rs b/src/podcasts/limits.rs new file mode 100644 index 0000000..43578fa --- /dev/null +++ b/src/podcasts/limits.rs @@ -0,0 +1,123 @@ +//! Bounds on the outbound work signed-in users can start: feed downloads +//! they ask for ("add RSS", manual refresh) and audio proxy streams. Each +//! kind has a per-account and a server-wide limit on requests in flight, so +//! one account cannot tie up the server's memory, connections or bandwidth. +//! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`, +//! `runner::RUN_CONCURRENCY`, `runner::RUN_PER_ACCOUNT`). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +/// Feed downloads one account may run at once ("add RSS", refresh). +pub(crate) const FEED_FETCHES_PER_ACCOUNT: usize = 2; +/// Feed downloads started by users, server-wide (each may buffer up to +/// `fetch::MAX_FEED_BYTES`). +pub(crate) const FEED_FETCHES_TOTAL: usize = 16; +/// Audio proxy responses one account may stream at once (a player opens a +/// new range request when seeking while the old one is still closing). +pub(crate) const PROXY_STREAMS_PER_ACCOUNT: usize = 4; +/// Audio proxy responses streamed at once, server-wide. +pub(crate) const PROXY_STREAMS_TOTAL: usize = 64; +/// Seconds a refused client is asked to wait (`Retry-After`). +pub(crate) const RETRY_AFTER_SECS: u64 = 5; + +#[derive(Debug, Default)] +struct Counts { + total: usize, + accounts: HashMap, +} + +/// Requests in flight per account and in total. +#[derive(Debug, Clone)] +pub(crate) struct ConcurrencyLimit { + per_account: usize, + total: usize, + counts: Arc>, +} + +impl ConcurrencyLimit { + pub(crate) fn new(per_account: usize, total: usize) -> Self { + Self { + per_account, + total, + counts: Arc::default(), + } + } + + /// A slot for one request of `user_id`, released when the permit is + /// dropped; `None` when the account or the server is at its limit. + pub(crate) fn try_acquire(&self, user_id: i64) -> Option { + let mut counts = self + .counts + .lock() + .unwrap_or_else(|error| error.into_inner()); + let account = counts.accounts.get(&user_id).copied().unwrap_or(0); + if account >= self.per_account || counts.total >= self.total { + return None; + } + counts.total += 1; + counts.accounts.insert(user_id, account + 1); + Some(Permit { + user_id, + counts: Arc::clone(&self.counts), + }) + } + + #[cfg(test)] + fn in_flight(&self, user_id: i64) -> (usize, usize) { + let counts = self.counts.lock().unwrap(); + ( + counts.accounts.get(&user_id).copied().unwrap_or(0), + counts.total, + ) + } +} + +/// One request in flight (see [`ConcurrencyLimit::try_acquire`]). +#[derive(Debug)] +pub(crate) struct Permit { + user_id: i64, + counts: Arc>, +} + +impl Drop for Permit { + fn drop(&mut self) { + let mut counts = self + .counts + .lock() + .unwrap_or_else(|error| error.into_inner()); + counts.total = counts.total.saturating_sub(1); + if let Some(account) = counts.accounts.get_mut(&self.user_id) { + *account = account.saturating_sub(1); + if *account == 0 { + counts.accounts.remove(&self.user_id); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accounts_and_the_server_are_bounded_until_permits_are_dropped() { + let limit = ConcurrencyLimit::new(2, 3); + let a1 = limit.try_acquire(1).unwrap(); + let a2 = limit.try_acquire(1).unwrap(); + assert!(limit.try_acquire(1).is_none(), "per account"); + let b1 = limit.try_acquire(2).unwrap(); + assert!(limit.try_acquire(3).is_none(), "server-wide"); + assert_eq!(limit.in_flight(1), (2, 3)); + drop(a1); + assert_eq!(limit.in_flight(1), (1, 2)); + let a3 = limit.try_acquire(1).unwrap(); + drop((a2, a3, b1)); + assert_eq!(limit.in_flight(1), (0, 0)); + assert!(limit.counts.lock().unwrap().accounts.is_empty()); + // Clones share the counts. + let clone = limit.clone(); + let _held = clone.try_acquire(4).unwrap(); + assert_eq!(limit.in_flight(4), (1, 1)); + } +} diff --git a/src/podcasts/mod.rs b/src/podcasts/mod.rs index 6bf2eac..a07cb89 100644 --- a/src/podcasts/mod.rs +++ b/src/podcasts/mod.rs @@ -62,6 +62,7 @@ pub(crate) mod api; mod apply; mod fetch; pub(crate) mod http; +mod limits; mod net; mod proxy; mod queries; @@ -112,14 +113,24 @@ pub(crate) enum PodcastStoreError { NotSubscribed(PodcastId), /// Input violates the podcast contract. Invalid(String), + /// A new subscription would exceed [`MAX_SUBSCRIPTIONS_PER_ACCOUNT`]. + SubscriptionLimit(usize), } +/// Active subscriptions an account may add locally (subscriptions synced +/// from the account's own devices are always applied). Every subscription +/// is a feed the server downloads about once an hour. +pub(crate) const MAX_SUBSCRIPTIONS_PER_ACCOUNT: usize = 1000; + impl fmt::Display for PodcastStoreError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"), Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"), Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"), + Self::SubscriptionLimit(limit) => { + write!(f, "subscription limit reached ({limit} podcasts)") + } } } } diff --git a/src/podcasts/postgres_tests.rs b/src/podcasts/postgres_tests.rs index 10416e7..c7d31f5 100644 --- a/src/podcasts/postgres_tests.rs +++ b/src/podcasts/postgres_tests.rs @@ -1179,7 +1179,7 @@ async fn postgres_podcast_fetch_state_is_local_only() { // Scheduling: overdue first, future and unsubscribed podcasts skipped. let due: Vec = store - .podcasts_due_for_refresh(now, 10_000) + .podcasts_due_for_refresh(now, 10_000, usize::MAX) .await .unwrap() .into_iter() @@ -1606,10 +1606,101 @@ fn local_fetcher() -> FeedFetcher { ) } +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_refresh_takes_accounts_in_turns() { + let pool = test_pool().await; + let store = PodcastStore::new(pool.clone()); + let (busy, quiet) = (random_user(), random_user()); + let now = 1_900_000_000_000; + let mut busy_ids = Vec::new(); + for (index, path) in ["turn-a.xml", "turn-b.xml", "turn-c.xml"] + .iter() + .enumerate() + { + let id = store + .subscribe(busy, &meta(path, "Busy")) + .await + .unwrap() + .podcast_id; + // The busy account's podcasts are the most overdue ones. + store + .record_fetch_failure(busy, &id, now, "x", Some(now - 10_000 + index as i64)) + .await + .unwrap(); + busy_ids.push(id); + } + let quiet_id = store + .subscribe(quiet, &meta("turn-q.xml", "Quiet")) + .await + .unwrap() + .podcast_id; + store + .record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1)) + .await + .unwrap(); + let due = |per_account| { + let store = store.clone(); + async move { + store + .podcasts_due_for_refresh(now, 100_000, per_account) + .await + .unwrap() + .into_iter() + .filter(|(user, ..)| [busy, quiet].contains(user)) + .map(|(user, id, _)| (user, id)) + .collect::>() + } + }; + assert_eq!( + due(2).await, + vec![ + (busy, busy_ids[0].clone()), + (quiet, quiet_id.clone()), + (busy, busy_ids[1].clone()), + ], + "one turn per account, at most two each" + ); + assert_eq!(due(usize::MAX).await.len(), 4); + + // New subscriptions stop at the per-account limit. + sqlx::query( + "INSERT INTO furumusic__podcast_subscription + (user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id) + SELECT $1, 'pod1:cap' || n, TRUE, '{}'::jsonb, 1, 1, 'x:' || n + FROM generate_series(1, $2) AS n", + ) + .bind(quiet) + .bind(i32::try_from(MAX_SUBSCRIPTIONS_PER_ACCOUNT - 1).unwrap()) + .execute(&pool) + .await + .unwrap(); + let error = store + .subscribe(quiet, &meta("turn-over.xml", "Over")) + .await + .unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&PodcastStoreError::SubscriptionLimit( + MAX_SUBSCRIPTIONS_PER_ACCOUNT + )) + ); + // Subscribing again to a podcast already subscribed is not new. + assert!( + store + .subscribe(quiet, &meta("turn-q.xml", "Quiet")) + .await + .unwrap() + .already_subscribed + ); + + cleanup(&pool, [busy, quiet]).await; +} + /// Due groups of the given accounts only (other tests share the database). async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups { let due = store - .podcasts_due_for_refresh(now, 100_000) + .podcasts_due_for_refresh(now, 100_000, usize::MAX) .await .unwrap() .into_iter() diff --git a/src/podcasts/refresh.rs b/src/podcasts/refresh.rs index ef9afd4..73d41aa 100644 --- a/src/podcasts/refresh.rs +++ b/src/podcasts/refresh.rs @@ -439,26 +439,39 @@ impl PodcastStore { } /// Subscribed podcasts of every account due for a refresh at `now_ms` - /// (never checked or `next_check_at_ms <= now_ms`), most overdue first, - /// with their fetch URL: `(user_id, podcast_id, feed_url)`. + /// (never checked or `next_check_at_ms <= now_ms`) with their fetch + /// URL: `(user_id, podcast_id, feed_url)`. Accounts take turns: first + /// the most overdue podcast of every account, then the second, and so + /// on, at most `per_account` per account; within a turn the most + /// overdue first. pub(crate) async fn podcasts_due_for_refresh( &self, now_ms: i64, limit: usize, + per_account: usize, ) -> Result> { let rows = sqlx::query( - "SELECT p.user_id, p.podcast_id, p.feed_url - FROM furumusic__podcast p - JOIN furumusic__podcast_subscription s - ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id AND s.subscribed - LEFT JOIN furumusic__podcast_fetch_state f - ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id - WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1 - ORDER BY COALESCE(f.next_check_at_ms, 0), p.user_id, p.podcast_id + "SELECT user_id, podcast_id, feed_url + FROM (SELECT p.user_id, p.podcast_id, p.feed_url, + COALESCE(f.next_check_at_ms, 0) AS due, + ROW_NUMBER() OVER ( + PARTITION BY p.user_id + ORDER BY COALESCE(f.next_check_at_ms, 0), p.podcast_id + ) AS turn + FROM furumusic__podcast p + JOIN furumusic__podcast_subscription s + ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id + AND s.subscribed + LEFT JOIN furumusic__podcast_fetch_state f + ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id + WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due + WHERE turn <= $3 + ORDER BY turn, due, user_id, podcast_id LIMIT $2", ) .bind(now_ms) .bind(i64::try_from(limit).unwrap_or(i64::MAX)) + .bind(i64::try_from(per_account).unwrap_or(i64::MAX)) .fetch_all(&self.pool) .await?; Ok(rows diff --git a/src/podcasts/runner.rs b/src/podcasts/runner.rs index 5321b13..340d608 100644 --- a/src/podcasts/runner.rs +++ b/src/podcasts/runner.rs @@ -27,6 +27,10 @@ pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000; pub(crate) const RUN_LIMIT: usize = 200; /// Feed requests in flight at once. pub(crate) const RUN_CONCURRENCY: usize = 4; +/// Podcasts of one account considered per scheduled run. Due podcasts are +/// taken round-robin across accounts, so an account with many (or slow) +/// feeds cannot hold back everyone else's refreshes. +pub(crate) const RUN_PER_ACCOUNT: usize = 25; /// Delay before the next attempt after `consecutive_failures` failures in a /// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a @@ -150,7 +154,8 @@ pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroup groups } -/// Refreshes every podcast due at `now_ms` (at most `limit`), with at most +/// Refreshes podcasts due at `now_ms` (at most `limit`, at most +/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most /// `concurrency` feed requests in flight. Accounts subscribed to the same /// feed URL share one request. A failing feed never stops the others. pub(crate) async fn refresh_due( @@ -160,7 +165,9 @@ pub(crate) async fn refresh_due( limit: usize, concurrency: usize, ) -> Result { - let due = store.podcasts_due_for_refresh(now_ms, limit).await?; + let due = store + .podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT) + .await?; refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await }