From 0aead73135df83aeb1c6a9b2f0556a47a052ddb6 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 22:41:02 +0200 Subject: [PATCH] Let one account use the whole refresh run when others have nothing due Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++-- docs/administration.md | 13 ++++++----- docs/development.md | 3 ++- src/podcasts/limits.rs | 2 +- src/podcasts/postgres_tests.rs | 42 +++++++++++++++++++--------------- src/podcasts/refresh.rs | 12 ++++------ src/podcasts/runner.rs | 15 ++++++------ 7 files changed, 48 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6576df1..78e3f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,8 +36,8 @@ every five minutes; each feed about hourly) with conditional requests, permanent redirects learned as feed aliases, size and time limits and an exponential backoff up to a day on errors. Accounts subscribed to the same - feed share one request; accounts take turns (at most 25 feeds each per - run). `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on + feed share one request; accounts take turns within a run of up to 200 + feeds. `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on demand. Only metadata is fetched, never audio. An account can have up to 1000 subscriptions and two user-started feed downloads at once. - Optional podcast audio proxy (`podcast_audio_proxy_enabled`, off by diff --git a/docs/administration.md b/docs/administration.md index 6c266fa..221c92a 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -125,12 +125,13 @@ feeds of subscribed podcasts: each feed about once an hour with conditional requests, failing feeds less and less often (up to once a day). It downloads feed metadata only, never episode audio. A failing feed keeps its known episodes. Accounts subscribed to the same feed share one request. Each run -checks at most 200 feeds, taking accounts in turns (at most 25 feeds of one -account per run), so an account with many or slow feeds does not delay the -others. An account can add up to 1000 subscriptions, and can have two feed -downloads ("add RSS", Refresh) and four proxied audio streams in progress at -once (16 and 64 for the whole server); beyond that the request is refused -with HTTP 429. +checks at most 200 feeds (2400 an hour), taking accounts in turns: first the +most overdue feed of every account, then the second, and so on. An account +with many or slow feeds therefore does not delay the others, but can use the +capacity nobody else needs. An account can add up to 1000 subscriptions, and +can have two feed downloads ("add RSS", Refresh) and four proxied audio +streams in progress at once (16 and 64 for the whole server); beyond that the +request is refused with HTTP 429. The server therefore makes outgoing HTTP(S) requests to podcast publishers. It only connects to public addresses on the web ports 80, 443, 8080 and 8443: diff --git a/docs/development.md b/docs/development.md index 6a5a942..8ace6b5 100644 --- a/docs/development.md +++ b/docs/development.md @@ -131,7 +131,8 @@ The runtime around the store lives next to it in `src/podcasts/`: 15 min × 2ⁿ⁻¹ up to a day (a longer `Retry-After` wins). `refresh_now` is the manual refresh: subscribed podcasts only, at most every 30 s. `jobs/podcast_refresh.rs` runs `refresh_due` every five minutes (200 - podcasts, 4 requests in flight, one run at a time via an advisory lock). + podcasts taken in turns across accounts without a per-account cap, 4 + requests in flight, one run at a time via an advisory lock). - `proxy.rs` and `http.rs` — the optional audio proxy and the endpoints under `/api/podcasts`. The proxy takes an episode id, loads the URL from the account's catalog (`PodcastStore::audio_source`), forwards a single diff --git a/src/podcasts/limits.rs b/src/podcasts/limits.rs index 43578fa..d894a8b 100644 --- a/src/podcasts/limits.rs +++ b/src/podcasts/limits.rs @@ -3,7 +3,7 @@ //! kind has a per-account and a server-wide limit on requests in flight, so //! one account cannot tie up the server's memory, connections or bandwidth. //! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`, -//! `runner::RUN_CONCURRENCY`, `runner::RUN_PER_ACCOUNT`). +//! `runner::RUN_CONCURRENCY`, accounts in turns). use std::collections::HashMap; use std::sync::{Arc, Mutex}; diff --git a/src/podcasts/postgres_tests.rs b/src/podcasts/postgres_tests.rs index 38bfa1a..3655368 100644 --- a/src/podcasts/postgres_tests.rs +++ b/src/podcasts/postgres_tests.rs @@ -1179,7 +1179,7 @@ async fn postgres_podcast_fetch_state_is_local_only() { // Scheduling: overdue first, future and unsubscribed podcasts skipped. let due: Vec = store - .podcasts_due_for_refresh(now, 10_000, usize::MAX) + .podcasts_due_for_refresh(now, 10_000) .await .unwrap() .into_iter() @@ -1639,29 +1639,35 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() { .record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1)) .await .unwrap(); - let due = |per_account| { - let store = store.clone(); - async move { - store - .podcasts_due_for_refresh(now, 100_000, per_account) - .await - .unwrap() - .into_iter() - .filter(|(user, ..)| [busy, quiet].contains(user)) - .map(|(user, id, _)| (user, id)) - .collect::>() - } - }; + let due = store + .podcasts_due_for_refresh(now, 100_000) + .await + .unwrap() + .into_iter() + .filter(|(user, ..)| [busy, quiet].contains(user)) + .map(|(user, id, _)| (user, id)) + .collect::>(); + // One turn per account: the quiet account's only feed comes before the + // busy account's second one, and the busy account still gets all of + // its due feeds when the run has room (no per-account cap). assert_eq!( - due(2).await, + due, vec![ (busy, busy_ids[0].clone()), (quiet, quiet_id.clone()), (busy, busy_ids[1].clone()), + (busy, busy_ids[2].clone()), ], - "one turn per account, at most two each" ); - assert_eq!(due(usize::MAX).await.len(), 4); + // With only one account due, a run takes up to the run limit from it. + let alone = store + .podcasts_due_for_refresh(now, 100_000) + .await + .unwrap() + .into_iter() + .filter(|(user, ..)| *user == busy) + .count(); + assert_eq!(alone, 3); // New subscriptions stop at the per-account limit. sqlx::query( @@ -1700,7 +1706,7 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() { /// Due groups of the given accounts only (other tests share the database). async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups { let due = store - .podcasts_due_for_refresh(now, 100_000, usize::MAX) + .podcasts_due_for_refresh(now, 100_000) .await .unwrap() .into_iter() diff --git a/src/podcasts/refresh.rs b/src/podcasts/refresh.rs index 73d41aa..cf0a033 100644 --- a/src/podcasts/refresh.rs +++ b/src/podcasts/refresh.rs @@ -440,15 +440,15 @@ impl PodcastStore { /// Subscribed podcasts of every account due for a refresh at `now_ms` /// (never checked or `next_check_at_ms <= now_ms`) with their fetch - /// URL: `(user_id, podcast_id, feed_url)`. Accounts take turns: first - /// the most overdue podcast of every account, then the second, and so - /// on, at most `per_account` per account; within a turn the most - /// overdue first. + /// URL: `(user_id, podcast_id, feed_url)`, at most `limit`. Accounts + /// take turns: first the most overdue podcast of every account, then the + /// second, and so on; within a turn the most overdue first. An account + /// with many due feeds therefore never delays the others, and still uses + /// whatever the run has left over. pub(crate) async fn podcasts_due_for_refresh( &self, now_ms: i64, limit: usize, - per_account: usize, ) -> Result> { let rows = sqlx::query( "SELECT user_id, podcast_id, feed_url @@ -465,13 +465,11 @@ impl PodcastStore { LEFT JOIN furumusic__podcast_fetch_state f ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due - WHERE turn <= $3 ORDER BY turn, due, user_id, podcast_id LIMIT $2", ) .bind(now_ms) .bind(i64::try_from(limit).unwrap_or(i64::MAX)) - .bind(i64::try_from(per_account).unwrap_or(i64::MAX)) .fetch_all(&self.pool) .await?; Ok(rows diff --git a/src/podcasts/runner.rs b/src/podcasts/runner.rs index 340d608..7351b63 100644 --- a/src/podcasts/runner.rs +++ b/src/podcasts/runner.rs @@ -23,14 +23,13 @@ pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000; pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000; /// A manual refresh is refused when the feed was checked this recently. pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000; -/// Podcasts considered per scheduled run. +/// Podcasts considered per scheduled run. Due podcasts are taken in turns +/// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an +/// account with many (or slow) feeds cannot hold back everyone else's +/// refreshes, but can use a run nobody else needs. pub(crate) const RUN_LIMIT: usize = 200; /// Feed requests in flight at once. pub(crate) const RUN_CONCURRENCY: usize = 4; -/// Podcasts of one account considered per scheduled run. Due podcasts are -/// taken round-robin across accounts, so an account with many (or slow) -/// feeds cannot hold back everyone else's refreshes. -pub(crate) const RUN_PER_ACCOUNT: usize = 25; /// Delay before the next attempt after `consecutive_failures` failures in a /// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a @@ -154,8 +153,8 @@ pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroup groups } -/// Refreshes podcasts due at `now_ms` (at most `limit`, at most -/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most +/// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in +/// turns), with at most /// `concurrency` feed requests in flight. Accounts subscribed to the same /// feed URL share one request. A failing feed never stops the others. pub(crate) async fn refresh_due( @@ -166,7 +165,7 @@ pub(crate) async fn refresh_due( concurrency: usize, ) -> Result { let due = store - .podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT) + .podcasts_due_for_refresh(now_ms, limit) .await?; refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await }