Let one account use the whole refresh run when others have nothing due
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
62141c0840
commit
0aead73135
7 files changed
+48
-43
No files matched your search
+2
-2
@@ -36,8 +36,8 @@
|
||||
every five minutes; each feed about hourly) with conditional requests,
|
||||
permanent redirects learned as feed aliases, size and time limits and an
|
||||
exponential backoff up to a day on errors. Accounts subscribed to the same
|
||||
feed share one request; accounts take turns (at most 25 feeds each per
|
||||
run). `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on
|
||||
feed share one request; accounts take turns within a run of up to 200
|
||||
feeds. `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on
|
||||
demand. Only metadata is fetched, never audio. An account can have up to
|
||||
1000 subscriptions and two user-started feed downloads at once.
|
||||
- Optional podcast audio proxy (`podcast_audio_proxy_enabled`, off by
|
||||
|
||||
@@ -125,12 +125,13 @@ feeds of subscribed podcasts: each feed about once an hour with conditional
|
||||
requests, failing feeds less and less often (up to once a day). It downloads
|
||||
feed metadata only, never episode audio. A failing feed keeps its known
|
||||
episodes. Accounts subscribed to the same feed share one request. Each run
|
||||
checks at most 200 feeds, taking accounts in turns (at most 25 feeds of one
|
||||
account per run), so an account with many or slow feeds does not delay the
|
||||
others. An account can add up to 1000 subscriptions, and can have two feed
|
||||
downloads ("add RSS", Refresh) and four proxied audio streams in progress at
|
||||
once (16 and 64 for the whole server); beyond that the request is refused
|
||||
with HTTP 429.
|
||||
checks at most 200 feeds (2400 an hour), taking accounts in turns: first the
|
||||
most overdue feed of every account, then the second, and so on. An account
|
||||
with many or slow feeds therefore does not delay the others, but can use the
|
||||
capacity nobody else needs. An account can add up to 1000 subscriptions, and
|
||||
can have two feed downloads ("add RSS", Refresh) and four proxied audio
|
||||
streams in progress at once (16 and 64 for the whole server); beyond that the
|
||||
request is refused with HTTP 429.
|
||||
|
||||
The server therefore makes outgoing HTTP(S) requests to podcast publishers.
|
||||
It only connects to public addresses on the web ports 80, 443, 8080 and 8443:
|
||||
|
||||
+2
-1
@@ -131,7 +131,8 @@ The runtime around the store lives next to it in `src/podcasts/`:
|
||||
15 min × 2ⁿ⁻¹ up to a day (a longer `Retry-After` wins). `refresh_now` is
|
||||
the manual refresh: subscribed podcasts only, at most every 30 s.
|
||||
`jobs/podcast_refresh.rs` runs `refresh_due` every five minutes (200
|
||||
podcasts, 4 requests in flight, one run at a time via an advisory lock).
|
||||
podcasts taken in turns across accounts without a per-account cap, 4
|
||||
requests in flight, one run at a time via an advisory lock).
|
||||
- `proxy.rs` and `http.rs` — the optional audio proxy and the endpoints under
|
||||
`/api/podcasts`. The proxy takes an episode id, loads the URL from the
|
||||
account's catalog (`PodcastStore::audio_source`), forwards a single
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! kind has a per-account and a server-wide limit on requests in flight, so
|
||||
//! one account cannot tie up the server's memory, connections or bandwidth.
|
||||
//! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`,
|
||||
//! `runner::RUN_CONCURRENCY`, `runner::RUN_PER_ACCOUNT`).
|
||||
//! `runner::RUN_CONCURRENCY`, accounts in turns).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -1179,7 +1179,7 @@ async fn postgres_podcast_fetch_state_is_local_only() {
|
||||
|
||||
// Scheduling: overdue first, future and unsubscribed podcasts skipped.
|
||||
let due: Vec<PodcastId> = store
|
||||
.podcasts_due_for_refresh(now, 10_000, usize::MAX)
|
||||
.podcasts_due_for_refresh(now, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
@@ -1639,29 +1639,35 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() {
|
||||
.record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1))
|
||||
.await
|
||||
.unwrap();
|
||||
let due = |per_account| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
store
|
||||
.podcasts_due_for_refresh(now, 100_000, per_account)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|(user, ..)| [busy, quiet].contains(user))
|
||||
.map(|(user, id, _)| (user, id))
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
};
|
||||
let due = store
|
||||
.podcasts_due_for_refresh(now, 100_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|(user, ..)| [busy, quiet].contains(user))
|
||||
.map(|(user, id, _)| (user, id))
|
||||
.collect::<Vec<_>>();
|
||||
// One turn per account: the quiet account's only feed comes before the
|
||||
// busy account's second one, and the busy account still gets all of
|
||||
// its due feeds when the run has room (no per-account cap).
|
||||
assert_eq!(
|
||||
due(2).await,
|
||||
due,
|
||||
vec![
|
||||
(busy, busy_ids[0].clone()),
|
||||
(quiet, quiet_id.clone()),
|
||||
(busy, busy_ids[1].clone()),
|
||||
(busy, busy_ids[2].clone()),
|
||||
],
|
||||
"one turn per account, at most two each"
|
||||
);
|
||||
assert_eq!(due(usize::MAX).await.len(), 4);
|
||||
// With only one account due, a run takes up to the run limit from it.
|
||||
let alone = store
|
||||
.podcasts_due_for_refresh(now, 100_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|(user, ..)| *user == busy)
|
||||
.count();
|
||||
assert_eq!(alone, 3);
|
||||
|
||||
// New subscriptions stop at the per-account limit.
|
||||
sqlx::query(
|
||||
@@ -1700,7 +1706,7 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() {
|
||||
/// Due groups of the given accounts only (other tests share the database).
|
||||
async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups {
|
||||
let due = store
|
||||
.podcasts_due_for_refresh(now, 100_000, usize::MAX)
|
||||
.podcasts_due_for_refresh(now, 100_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
|
||||
@@ -440,15 +440,15 @@ impl PodcastStore {
|
||||
|
||||
/// Subscribed podcasts of every account due for a refresh at `now_ms`
|
||||
/// (never checked or `next_check_at_ms <= now_ms`) with their fetch
|
||||
/// URL: `(user_id, podcast_id, feed_url)`. Accounts take turns: first
|
||||
/// the most overdue podcast of every account, then the second, and so
|
||||
/// on, at most `per_account` per account; within a turn the most
|
||||
/// overdue first.
|
||||
/// URL: `(user_id, podcast_id, feed_url)`, at most `limit`. Accounts
|
||||
/// take turns: first the most overdue podcast of every account, then the
|
||||
/// second, and so on; within a turn the most overdue first. An account
|
||||
/// with many due feeds therefore never delays the others, and still uses
|
||||
/// whatever the run has left over.
|
||||
pub(crate) async fn podcasts_due_for_refresh(
|
||||
&self,
|
||||
now_ms: i64,
|
||||
limit: usize,
|
||||
per_account: usize,
|
||||
) -> Result<Vec<(i64, PodcastId, FeedUrl)>> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT user_id, podcast_id, feed_url
|
||||
@@ -465,13 +465,11 @@ impl PodcastStore {
|
||||
LEFT JOIN furumusic__podcast_fetch_state f
|
||||
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
|
||||
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due
|
||||
WHERE turn <= $3
|
||||
ORDER BY turn, due, user_id, podcast_id
|
||||
LIMIT $2",
|
||||
)
|
||||
.bind(now_ms)
|
||||
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
|
||||
.bind(i64::try_from(per_account).unwrap_or(i64::MAX))
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
|
||||
@@ -23,14 +23,13 @@ pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
|
||||
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
|
||||
/// A manual refresh is refused when the feed was checked this recently.
|
||||
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
|
||||
/// Podcasts considered per scheduled run.
|
||||
/// Podcasts considered per scheduled run. Due podcasts are taken in turns
|
||||
/// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an
|
||||
/// account with many (or slow) feeds cannot hold back everyone else's
|
||||
/// refreshes, but can use a run nobody else needs.
|
||||
pub(crate) const RUN_LIMIT: usize = 200;
|
||||
/// Feed requests in flight at once.
|
||||
pub(crate) const RUN_CONCURRENCY: usize = 4;
|
||||
/// Podcasts of one account considered per scheduled run. Due podcasts are
|
||||
/// taken round-robin across accounts, so an account with many (or slow)
|
||||
/// feeds cannot hold back everyone else's refreshes.
|
||||
pub(crate) const RUN_PER_ACCOUNT: usize = 25;
|
||||
|
||||
/// Delay before the next attempt after `consecutive_failures` failures in a
|
||||
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
|
||||
@@ -154,8 +153,8 @@ pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroup
|
||||
groups
|
||||
}
|
||||
|
||||
/// Refreshes podcasts due at `now_ms` (at most `limit`, at most
|
||||
/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most
|
||||
/// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in
|
||||
/// turns), with at most
|
||||
/// `concurrency` feed requests in flight. Accounts subscribed to the same
|
||||
/// feed URL share one request. A failing feed never stops the others.
|
||||
pub(crate) async fn refresh_due(
|
||||
@@ -166,7 +165,7 @@ pub(crate) async fn refresh_due(
|
||||
concurrency: usize,
|
||||
) -> Result<RunSummary> {
|
||||
let due = store
|
||||
.podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT)
|
||||
.podcasts_due_for_refresh(now_ms, limit)
|
||||
.await?;
|
||||
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user