Let one account use the whole refresh run when others have nothing due

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent 62141c0840
commit 0aead73135
7 files changed
+48 -43

No files matched your search

+2 -2
View File
@@ -36,8 +36,8 @@
every five minutes; each feed about hourly) with conditional requests,
permanent redirects learned as feed aliases, size and time limits and an
exponential backoff up to a day on errors. Accounts subscribed to the same
feed share one request; accounts take turns (at most 25 feeds each per
run). `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on
feed share one request; accounts take turns within a run of up to 200
feeds. `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on
demand. Only metadata is fetched, never audio. An account can have up to
1000 subscriptions and two user-started feed downloads at once.
- Optional podcast audio proxy (`podcast_audio_proxy_enabled`, off by
+7 -6
View File
@@ -125,12 +125,13 @@ feeds of subscribed podcasts: each feed about once an hour with conditional
requests, failing feeds less and less often (up to once a day). It downloads
feed metadata only, never episode audio. A failing feed keeps its known
episodes. Accounts subscribed to the same feed share one request. Each run
checks at most 200 feeds, taking accounts in turns (at most 25 feeds of one
account per run), so an account with many or slow feeds does not delay the
others. An account can add up to 1000 subscriptions, and can have two feed
downloads ("add RSS", Refresh) and four proxied audio streams in progress at
once (16 and 64 for the whole server); beyond that the request is refused
with HTTP 429.
checks at most 200 feeds (2400 an hour), taking accounts in turns: first the
most overdue feed of every account, then the second, and so on. An account
with many or slow feeds therefore does not delay the others, but can use the
capacity nobody else needs. An account can add up to 1000 subscriptions, and
can have two feed downloads ("add RSS", Refresh) and four proxied audio
streams in progress at once (16 and 64 for the whole server); beyond that the
request is refused with HTTP 429.
The server therefore makes outgoing HTTP(S) requests to podcast publishers.
It only connects to public addresses on the web ports 80, 443, 8080 and 8443:
+2 -1
View File
@@ -131,7 +131,8 @@ The runtime around the store lives next to it in `src/podcasts/`:
15 min × 2ⁿ⁻¹ up to a day (a longer `Retry-After` wins). `refresh_now` is
the manual refresh: subscribed podcasts only, at most every 30 s.
`jobs/podcast_refresh.rs` runs `refresh_due` every five minutes (200
podcasts, 4 requests in flight, one run at a time via an advisory lock).
podcasts taken in turns across accounts without a per-account cap, 4
requests in flight, one run at a time via an advisory lock).
- `proxy.rs` and `http.rs` — the optional audio proxy and the endpoints under
`/api/podcasts`. The proxy takes an episode id, loads the URL from the
account's catalog (`PodcastStore::audio_source`), forwards a single
+1 -1
View File
@@ -3,7 +3,7 @@
//! kind has a per-account and a server-wide limit on requests in flight, so
//! one account cannot tie up the server's memory, connections or bandwidth.
//! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`,
//! `runner::RUN_CONCURRENCY`, `runner::RUN_PER_ACCOUNT`).
//! `runner::RUN_CONCURRENCY`, accounts in turns).
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
+24 -18
View File
@@ -1179,7 +1179,7 @@ async fn postgres_podcast_fetch_state_is_local_only() {
// Scheduling: overdue first, future and unsubscribed podcasts skipped.
let due: Vec<PodcastId> = store
.podcasts_due_for_refresh(now, 10_000, usize::MAX)
.podcasts_due_for_refresh(now, 10_000)
.await
.unwrap()
.into_iter()
@@ -1639,29 +1639,35 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() {
.record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1))
.await
.unwrap();
let due = |per_account| {
let store = store.clone();
async move {
store
.podcasts_due_for_refresh(now, 100_000, per_account)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| [busy, quiet].contains(user))
.map(|(user, id, _)| (user, id))
.collect::<Vec<_>>()
}
};
let due = store
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| [busy, quiet].contains(user))
.map(|(user, id, _)| (user, id))
.collect::<Vec<_>>();
// One turn per account: the quiet account's only feed comes before the
// busy account's second one, and the busy account still gets all of
// its due feeds when the run has room (no per-account cap).
assert_eq!(
due(2).await,
due,
vec![
(busy, busy_ids[0].clone()),
(quiet, quiet_id.clone()),
(busy, busy_ids[1].clone()),
(busy, busy_ids[2].clone()),
],
"one turn per account, at most two each"
);
assert_eq!(due(usize::MAX).await.len(), 4);
// With only one account due, a run takes up to the run limit from it.
let alone = store
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| *user == busy)
.count();
assert_eq!(alone, 3);
// New subscriptions stop at the per-account limit.
sqlx::query(
@@ -1700,7 +1706,7 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() {
/// Due groups of the given accounts only (other tests share the database).
async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups {
let due = store
.podcasts_due_for_refresh(now, 100_000, usize::MAX)
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
+5 -7
View File
@@ -440,15 +440,15 @@ impl PodcastStore {
/// Subscribed podcasts of every account due for a refresh at `now_ms`
/// (never checked or `next_check_at_ms <= now_ms`) with their fetch
/// URL: `(user_id, podcast_id, feed_url)`. Accounts take turns: first
/// the most overdue podcast of every account, then the second, and so
/// on, at most `per_account` per account; within a turn the most
/// overdue first.
/// URL: `(user_id, podcast_id, feed_url)`, at most `limit`. Accounts
/// take turns: first the most overdue podcast of every account, then the
/// second, and so on; within a turn the most overdue first. An account
/// with many due feeds therefore never delays the others, and still uses
/// whatever the run has left over.
pub(crate) async fn podcasts_due_for_refresh(
&self,
now_ms: i64,
limit: usize,
per_account: usize,
) -> Result<Vec<(i64, PodcastId, FeedUrl)>> {
let rows = sqlx::query(
"SELECT user_id, podcast_id, feed_url
@@ -465,13 +465,11 @@ impl PodcastStore {
LEFT JOIN furumusic__podcast_fetch_state f
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due
WHERE turn <= $3
ORDER BY turn, due, user_id, podcast_id
LIMIT $2",
)
.bind(now_ms)
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.bind(i64::try_from(per_account).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
+7 -8
View File
@@ -23,14 +23,13 @@ pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
/// A manual refresh is refused when the feed was checked this recently.
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
/// Podcasts considered per scheduled run.
/// Podcasts considered per scheduled run. Due podcasts are taken in turns
/// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an
/// account with many (or slow) feeds cannot hold back everyone else's
/// refreshes, but can use a run nobody else needs.
pub(crate) const RUN_LIMIT: usize = 200;
/// Feed requests in flight at once.
pub(crate) const RUN_CONCURRENCY: usize = 4;
/// Podcasts of one account considered per scheduled run. Due podcasts are
/// taken round-robin across accounts, so an account with many (or slow)
/// feeds cannot hold back everyone else's refreshes.
pub(crate) const RUN_PER_ACCOUNT: usize = 25;
/// Delay before the next attempt after `consecutive_failures` failures in a
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
@@ -154,8 +153,8 @@ pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroup
groups
}
/// Refreshes podcasts due at `now_ms` (at most `limit`, at most
/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most
/// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in
/// turns), with at most
/// `concurrency` feed requests in flight. Accounts subscribed to the same
/// feed URL share one request. A failing feed never stops the others.
pub(crate) async fn refresh_due(
@@ -166,7 +165,7 @@ pub(crate) async fn refresh_due(
concurrency: usize,
) -> Result<RunSummary> {
let due = store
.podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT)
.podcasts_due_for_refresh(now_ms, limit)
.await?;
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
}