Let one account use the whole refresh run when others have nothing due

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent 62141c0840
commit 0aead73135
7 files changed
+48 -43

No files matched your search

+1 -1
View File
@@ -3,7 +3,7 @@
//! kind has a per-account and a server-wide limit on requests in flight, so
//! one account cannot tie up the server's memory, connections or bandwidth.
//! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`,
//! `runner::RUN_CONCURRENCY`, `runner::RUN_PER_ACCOUNT`).
//! `runner::RUN_CONCURRENCY`, accounts in turns).
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
+24 -18
View File
@@ -1179,7 +1179,7 @@ async fn postgres_podcast_fetch_state_is_local_only() {
// Scheduling: overdue first, future and unsubscribed podcasts skipped.
let due: Vec<PodcastId> = store
.podcasts_due_for_refresh(now, 10_000, usize::MAX)
.podcasts_due_for_refresh(now, 10_000)
.await
.unwrap()
.into_iter()
@@ -1639,29 +1639,35 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() {
.record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1))
.await
.unwrap();
let due = |per_account| {
let store = store.clone();
async move {
store
.podcasts_due_for_refresh(now, 100_000, per_account)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| [busy, quiet].contains(user))
.map(|(user, id, _)| (user, id))
.collect::<Vec<_>>()
}
};
let due = store
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| [busy, quiet].contains(user))
.map(|(user, id, _)| (user, id))
.collect::<Vec<_>>();
// One turn per account: the quiet account's only feed comes before the
// busy account's second one, and the busy account still gets all of
// its due feeds when the run has room (no per-account cap).
assert_eq!(
due(2).await,
due,
vec![
(busy, busy_ids[0].clone()),
(quiet, quiet_id.clone()),
(busy, busy_ids[1].clone()),
(busy, busy_ids[2].clone()),
],
"one turn per account, at most two each"
);
assert_eq!(due(usize::MAX).await.len(), 4);
// With only one account due, a run takes up to the run limit from it.
let alone = store
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| *user == busy)
.count();
assert_eq!(alone, 3);
// New subscriptions stop at the per-account limit.
sqlx::query(
@@ -1700,7 +1706,7 @@ async fn postgres_podcast_refresh_takes_accounts_in_turns() {
/// Due groups of the given accounts only (other tests share the database).
async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups {
let due = store
.podcasts_due_for_refresh(now, 100_000, usize::MAX)
.podcasts_due_for_refresh(now, 100_000)
.await
.unwrap()
.into_iter()
+5 -7
View File
@@ -440,15 +440,15 @@ impl PodcastStore {
/// Subscribed podcasts of every account due for a refresh at `now_ms`
/// (never checked or `next_check_at_ms <= now_ms`) with their fetch
/// URL: `(user_id, podcast_id, feed_url)`. Accounts take turns: first
/// the most overdue podcast of every account, then the second, and so
/// on, at most `per_account` per account; within a turn the most
/// overdue first.
/// URL: `(user_id, podcast_id, feed_url)`, at most `limit`. Accounts
/// take turns: first the most overdue podcast of every account, then the
/// second, and so on; within a turn the most overdue first. An account
/// with many due feeds therefore never delays the others, and still uses
/// whatever the run has left over.
pub(crate) async fn podcasts_due_for_refresh(
&self,
now_ms: i64,
limit: usize,
per_account: usize,
) -> Result<Vec<(i64, PodcastId, FeedUrl)>> {
let rows = sqlx::query(
"SELECT user_id, podcast_id, feed_url
@@ -465,13 +465,11 @@ impl PodcastStore {
LEFT JOIN furumusic__podcast_fetch_state f
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due
WHERE turn <= $3
ORDER BY turn, due, user_id, podcast_id
LIMIT $2",
)
.bind(now_ms)
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.bind(i64::try_from(per_account).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
+7 -8
View File
@@ -23,14 +23,13 @@ pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
/// A manual refresh is refused when the feed was checked this recently.
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
/// Podcasts considered per scheduled run.
/// Podcasts considered per scheduled run. Due podcasts are taken in turns
/// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an
/// account with many (or slow) feeds cannot hold back everyone else's
/// refreshes, but can use a run nobody else needs.
pub(crate) const RUN_LIMIT: usize = 200;
/// Feed requests in flight at once.
pub(crate) const RUN_CONCURRENCY: usize = 4;
/// Podcasts of one account considered per scheduled run. Due podcasts are
/// taken round-robin across accounts, so an account with many (or slow)
/// feeds cannot hold back everyone else's refreshes.
pub(crate) const RUN_PER_ACCOUNT: usize = 25;
/// Delay before the next attempt after `consecutive_failures` failures in a
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
@@ -154,8 +153,8 @@ pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroup
groups
}
/// Refreshes podcasts due at `now_ms` (at most `limit`, at most
/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most
/// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in
/// turns), with at most
/// `concurrency` feed requests in flight. Accounts subscribed to the same
/// feed URL share one request. A failing feed never stops the others.
pub(crate) async fn refresh_due(
@@ -166,7 +165,7 @@ pub(crate) async fn refresh_due(
concurrency: usize,
) -> Result<RunSummary> {
let due = store
.podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT)
.podcasts_due_for_refresh(now_ms, limit)
.await?;
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
}