Bound user-started feed and audio requests per account

Feed downloads and proxy streams have per-account and server-wide
limits, new subscriptions stop at 1000 per account, and scheduled
refreshes take accounts in turns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent 2eb27cfe77
commit cfebe1898f
9 files changed
+330 -18

No files matched your search

+7 -1
View File
@@ -120,7 +120,13 @@ The `podcast_refresh` job (every five minutes, listed under Jobs) checks the
feeds of subscribed podcasts: each feed about once an hour with conditional
requests, failing feeds less and less often (up to once a day). It downloads
feed metadata only, never episode audio. A failing feed keeps its known
episodes. Accounts subscribed to the same feed share one request.
episodes. Accounts subscribed to the same feed share one request. Each run
checks at most 200 feeds, taking accounts in turns (at most 25 feeds of one
account per run), so an account with many or slow feeds does not delay the
others. An account can add up to 1000 subscriptions, and can have two feed
downloads ("add RSS", Refresh) and four proxied audio streams in progress at
once (16 and 64 for the whole server); beyond that the request is refused
with HTTP 429.
The server therefore makes outgoing HTTP(S) requests to podcast publishers.
It only connects to public addresses on the web ports 80, 443, 8080 and 8443:
+12
View File
@@ -216,6 +216,18 @@ impl PodcastStore {
ops: Vec::new(),
});
}
let active: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_subscription
WHERE user_id = $1 AND subscribed",
)
.bind(user_id)
.fetch_one(&mut *tx)
.await?;
if usize::try_from(active).unwrap_or(usize::MAX) >= super::MAX_SUBSCRIPTIONS_PER_ACCOUNT {
return Err(
PodcastStoreError::SubscriptionLimit(super::MAX_SUBSCRIPTIONS_PER_ACCOUNT).into(),
);
}
let share = apply::load_share(&mut tx, user_id, &podcast_id).await?;
let generation = next_podcast_generation(current.as_ref(), share.as_ref());
let aliases = merge_feed_aliases(
+5
View File
@@ -73,6 +73,7 @@ impl From<anyhow::Error> for ApiError {
PodcastStoreError::UnknownPodcast(_) => Self::not_found(error.to_string()),
PodcastStoreError::NotSubscribed(_) => Self::new(409, error.to_string()),
PodcastStoreError::Invalid(_) => Self::bad_request(error.to_string()),
PodcastStoreError::SubscriptionLimit(_) => Self::new(422, error.to_string()),
};
}
if let Some(SubscribeUrlError::Fetch(fetch)) = error.downcast_ref::<SubscribeUrlError>() {
@@ -975,6 +976,10 @@ mod tests {
assert_eq!(not_subscribed.status, 409);
let invalid: ApiError = super::super::invalid("bad").into();
assert_eq!(invalid.status, 400);
let limit: ApiError =
anyhow::Error::from(PodcastStoreError::SubscriptionLimit(1000)).into();
assert_eq!(limit.status, 422);
assert_eq!(limit.message, "subscription limit reached (1000 podcasts)");
let refused: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Refused,
+47 -3
View File
@@ -52,6 +52,7 @@ use serde::{Deserialize, Serialize};
use super::api::{self, ApiError, EpisodeAction, EpisodeListParams};
use super::fetch::FeedFetcher;
use super::limits::{self, ConcurrencyLimit, Permit};
use super::proxy::AudioProxy;
use super::runner::{self, ManualRefreshError, TargetResult};
use super::{PodcastStore, now_ms};
@@ -104,6 +105,22 @@ fn unauthorized() -> cot::response::Response {
json_error(StatusCode::UNAUTHORIZED, "not authenticated")
}
/// 429 when an account or the server has too many requests of a kind in
/// flight (see [`limits`]).
fn too_busy(message: &str) -> cot::response::Response {
let mut response = json_error(StatusCode::TOO_MANY_REQUESTS, message);
response.headers_mut().insert(
RETRY_AFTER,
limits::RETRY_AFTER_SECS
.to_string()
.parse()
.expect("digits"),
);
response
}
const FEEDS_BUSY: &str = "too many feed requests in progress; try again in a few seconds";
/// Largest JSON body accepted (a playback request carries one episode reference).
const MAX_JSON_BODY: usize = 64 * 1024;
@@ -151,9 +168,17 @@ struct State {
pool: Arc<tokio::sync::OnceCell<sqlx::PgPool>>,
fetcher: Arc<tokio::sync::OnceCell<FeedFetcher>>,
proxy: Arc<tokio::sync::OnceCell<AudioProxy>>,
/// Feed downloads users started ("add RSS", manual refresh).
feed_fetches: ConcurrencyLimit,
/// Audio proxy responses being streamed.
proxy_streams: ConcurrencyLimit,
}
impl State {
fn feed_fetch(&self, user_id: i64) -> Option<Permit> {
self.feed_fetches.try_acquire(user_id)
}
async fn store(&self) -> cot::Result<PodcastStore> {
let pool = self
.pool
@@ -196,6 +221,14 @@ impl PodcastsApp {
pool: Arc::new(tokio::sync::OnceCell::new()),
fetcher: Arc::new(tokio::sync::OnceCell::new()),
proxy: Arc::new(tokio::sync::OnceCell::new()),
feed_fetches: ConcurrencyLimit::new(
limits::FEED_FETCHES_PER_ACCOUNT,
limits::FEED_FETCHES_TOTAL,
),
proxy_streams: ConcurrencyLimit::new(
limits::PROXY_STREAMS_PER_ACCOUNT,
limits::PROXY_STREAMS_TOTAL,
),
},
}
}
@@ -281,6 +314,9 @@ async fn refresh_handler(
let Ok(podcast_id) = PodcastId::parse(&path.0.podcast_id) else {
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid podcast id"));
};
let Some(_permit) = state.feed_fetch(user.id) else {
return Ok(too_busy(FEEDS_BUSY));
};
let store = state.store().await?;
let fetcher = state.fetcher().await?;
match runner::refresh_now(&store, fetcher, user.id, &podcast_id, now_ms()).await {
@@ -389,6 +425,10 @@ async fn audio_handler(
.headers()
.get(RANGE)
.and_then(|value| value.to_str().ok());
// Held until the body is fully sent or the browser goes away.
let Some(permit) = state.proxy_streams.try_acquire(user.id) else {
return Ok(too_busy("too many podcast audio streams in progress"));
};
let proxy = state.proxy().await?;
let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await {
Ok(audio) => audio,
@@ -418,9 +458,10 @@ async fn audio_handler(
if audio.accept_ranges {
builder = builder.header(ACCEPT_RANGES, "bytes");
}
let body = audio
.body
.map_err(|error| cot::Error::internal(error.to_string()));
let body = audio.body.map_err(move |error| {
let _held = &permit;
cot::Error::internal(error.to_string())
});
builder.body(Body::streaming(body)).map_err(internal)
}
@@ -459,6 +500,9 @@ async fn subscribe_url_handler(
Ok(body) => body,
Err(error) => return Ok(api_error(error)),
};
let Some(_permit) = state.feed_fetch(user_id) else {
return Ok(too_busy(FEEDS_BUSY));
};
let store = state.store().await?;
let fetcher = state.fetcher().await?;
api_response(api::subscribe_url(&store, fetcher, user_id, body, now_ms()).await)
+123
View File
@@ -0,0 +1,123 @@
//! Bounds on the outbound work signed-in users can start: feed downloads
//! they ask for ("add RSS", manual refresh) and audio proxy streams. Each
//! kind has a per-account and a server-wide limit on requests in flight, so
//! one account cannot tie up the server's memory, connections or bandwidth.
//! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`,
//! `runner::RUN_CONCURRENCY`, `runner::RUN_PER_ACCOUNT`).
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
/// Feed downloads one account may run at once ("add RSS", refresh).
pub(crate) const FEED_FETCHES_PER_ACCOUNT: usize = 2;
/// Feed downloads started by users, server-wide (each may buffer up to
/// `fetch::MAX_FEED_BYTES`).
pub(crate) const FEED_FETCHES_TOTAL: usize = 16;
/// Audio proxy responses one account may stream at once (a player opens a
/// new range request when seeking while the old one is still closing).
pub(crate) const PROXY_STREAMS_PER_ACCOUNT: usize = 4;
/// Audio proxy responses streamed at once, server-wide.
pub(crate) const PROXY_STREAMS_TOTAL: usize = 64;
/// Seconds a refused client is asked to wait (`Retry-After`).
pub(crate) const RETRY_AFTER_SECS: u64 = 5;
#[derive(Debug, Default)]
struct Counts {
total: usize,
accounts: HashMap<i64, usize>,
}
/// Requests in flight per account and in total.
#[derive(Debug, Clone)]
pub(crate) struct ConcurrencyLimit {
per_account: usize,
total: usize,
counts: Arc<Mutex<Counts>>,
}
impl ConcurrencyLimit {
pub(crate) fn new(per_account: usize, total: usize) -> Self {
Self {
per_account,
total,
counts: Arc::default(),
}
}
/// A slot for one request of `user_id`, released when the permit is
/// dropped; `None` when the account or the server is at its limit.
pub(crate) fn try_acquire(&self, user_id: i64) -> Option<Permit> {
let mut counts = self
.counts
.lock()
.unwrap_or_else(|error| error.into_inner());
let account = counts.accounts.get(&user_id).copied().unwrap_or(0);
if account >= self.per_account || counts.total >= self.total {
return None;
}
counts.total += 1;
counts.accounts.insert(user_id, account + 1);
Some(Permit {
user_id,
counts: Arc::clone(&self.counts),
})
}
#[cfg(test)]
fn in_flight(&self, user_id: i64) -> (usize, usize) {
let counts = self.counts.lock().unwrap();
(
counts.accounts.get(&user_id).copied().unwrap_or(0),
counts.total,
)
}
}
/// One request in flight (see [`ConcurrencyLimit::try_acquire`]).
#[derive(Debug)]
pub(crate) struct Permit {
user_id: i64,
counts: Arc<Mutex<Counts>>,
}
impl Drop for Permit {
fn drop(&mut self) {
let mut counts = self
.counts
.lock()
.unwrap_or_else(|error| error.into_inner());
counts.total = counts.total.saturating_sub(1);
if let Some(account) = counts.accounts.get_mut(&self.user_id) {
*account = account.saturating_sub(1);
if *account == 0 {
counts.accounts.remove(&self.user_id);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accounts_and_the_server_are_bounded_until_permits_are_dropped() {
let limit = ConcurrencyLimit::new(2, 3);
let a1 = limit.try_acquire(1).unwrap();
let a2 = limit.try_acquire(1).unwrap();
assert!(limit.try_acquire(1).is_none(), "per account");
let b1 = limit.try_acquire(2).unwrap();
assert!(limit.try_acquire(3).is_none(), "server-wide");
assert_eq!(limit.in_flight(1), (2, 3));
drop(a1);
assert_eq!(limit.in_flight(1), (1, 2));
let a3 = limit.try_acquire(1).unwrap();
drop((a2, a3, b1));
assert_eq!(limit.in_flight(1), (0, 0));
assert!(limit.counts.lock().unwrap().accounts.is_empty());
// Clones share the counts.
let clone = limit.clone();
let _held = clone.try_acquire(4).unwrap();
assert_eq!(limit.in_flight(4), (1, 1));
}
}
+11
View File
@@ -62,6 +62,7 @@ pub(crate) mod api;
mod apply;
mod fetch;
pub(crate) mod http;
mod limits;
mod net;
mod proxy;
mod queries;
@@ -112,14 +113,24 @@ pub(crate) enum PodcastStoreError {
NotSubscribed(PodcastId),
/// Input violates the podcast contract.
Invalid(String),
/// A new subscription would exceed [`MAX_SUBSCRIPTIONS_PER_ACCOUNT`].
SubscriptionLimit(usize),
}
/// Active subscriptions an account may add locally (subscriptions synced
/// from the account's own devices are always applied). Every subscription
/// is a feed the server downloads about once an hour.
pub(crate) const MAX_SUBSCRIPTIONS_PER_ACCOUNT: usize = 1000;
impl fmt::Display for PodcastStoreError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"),
Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"),
Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"),
Self::SubscriptionLimit(limit) => {
write!(f, "subscription limit reached ({limit} podcasts)")
}
}
}
}
+93 -2
View File
@@ -1179,7 +1179,7 @@ async fn postgres_podcast_fetch_state_is_local_only() {
// Scheduling: overdue first, future and unsubscribed podcasts skipped.
let due: Vec<PodcastId> = store
.podcasts_due_for_refresh(now, 10_000)
.podcasts_due_for_refresh(now, 10_000, usize::MAX)
.await
.unwrap()
.into_iter()
@@ -1606,10 +1606,101 @@ fn local_fetcher() -> FeedFetcher {
)
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_refresh_takes_accounts_in_turns() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let (busy, quiet) = (random_user(), random_user());
let now = 1_900_000_000_000;
let mut busy_ids = Vec::new();
for (index, path) in ["turn-a.xml", "turn-b.xml", "turn-c.xml"]
.iter()
.enumerate()
{
let id = store
.subscribe(busy, &meta(path, "Busy"))
.await
.unwrap()
.podcast_id;
// The busy account's podcasts are the most overdue ones.
store
.record_fetch_failure(busy, &id, now, "x", Some(now - 10_000 + index as i64))
.await
.unwrap();
busy_ids.push(id);
}
let quiet_id = store
.subscribe(quiet, &meta("turn-q.xml", "Quiet"))
.await
.unwrap()
.podcast_id;
store
.record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1))
.await
.unwrap();
let due = |per_account| {
let store = store.clone();
async move {
store
.podcasts_due_for_refresh(now, 100_000, per_account)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| [busy, quiet].contains(user))
.map(|(user, id, _)| (user, id))
.collect::<Vec<_>>()
}
};
assert_eq!(
due(2).await,
vec![
(busy, busy_ids[0].clone()),
(quiet, quiet_id.clone()),
(busy, busy_ids[1].clone()),
],
"one turn per account, at most two each"
);
assert_eq!(due(usize::MAX).await.len(), 4);
// New subscriptions stop at the per-account limit.
sqlx::query(
"INSERT INTO furumusic__podcast_subscription
(user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
SELECT $1, 'pod1:cap' || n, TRUE, '{}'::jsonb, 1, 1, 'x:' || n
FROM generate_series(1, $2) AS n",
)
.bind(quiet)
.bind(i32::try_from(MAX_SUBSCRIPTIONS_PER_ACCOUNT - 1).unwrap())
.execute(&pool)
.await
.unwrap();
let error = store
.subscribe(quiet, &meta("turn-over.xml", "Over"))
.await
.unwrap_err();
assert_eq!(
error.downcast_ref::<PodcastStoreError>(),
Some(&PodcastStoreError::SubscriptionLimit(
MAX_SUBSCRIPTIONS_PER_ACCOUNT
))
);
// Subscribing again to a podcast already subscribed is not new.
assert!(
store
.subscribe(quiet, &meta("turn-q.xml", "Quiet"))
.await
.unwrap()
.already_subscribed
);
cleanup(&pool, [busy, quiet]).await;
}
/// Due groups of the given accounts only (other tests share the database).
async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups {
let due = store
.podcasts_due_for_refresh(now, 100_000)
.podcasts_due_for_refresh(now, 100_000, usize::MAX)
.await
.unwrap()
.into_iter()
+23 -10
View File
@@ -439,26 +439,39 @@ impl PodcastStore {
}
/// Subscribed podcasts of every account due for a refresh at `now_ms`
/// (never checked or `next_check_at_ms <= now_ms`), most overdue first,
/// with their fetch URL: `(user_id, podcast_id, feed_url)`.
/// (never checked or `next_check_at_ms <= now_ms`) with their fetch
/// URL: `(user_id, podcast_id, feed_url)`. Accounts take turns: first
/// the most overdue podcast of every account, then the second, and so
/// on, at most `per_account` per account; within a turn the most
/// overdue first.
pub(crate) async fn podcasts_due_for_refresh(
&self,
now_ms: i64,
limit: usize,
per_account: usize,
) -> Result<Vec<(i64, PodcastId, FeedUrl)>> {
let rows = sqlx::query(
"SELECT p.user_id, p.podcast_id, p.feed_url
FROM furumusic__podcast p
JOIN furumusic__podcast_subscription s
ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id AND s.subscribed
LEFT JOIN furumusic__podcast_fetch_state f
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1
ORDER BY COALESCE(f.next_check_at_ms, 0), p.user_id, p.podcast_id
"SELECT user_id, podcast_id, feed_url
FROM (SELECT p.user_id, p.podcast_id, p.feed_url,
COALESCE(f.next_check_at_ms, 0) AS due,
ROW_NUMBER() OVER (
PARTITION BY p.user_id
ORDER BY COALESCE(f.next_check_at_ms, 0), p.podcast_id
) AS turn
FROM furumusic__podcast p
JOIN furumusic__podcast_subscription s
ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id
AND s.subscribed
LEFT JOIN furumusic__podcast_fetch_state f
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due
WHERE turn <= $3
ORDER BY turn, due, user_id, podcast_id
LIMIT $2",
)
.bind(now_ms)
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.bind(i64::try_from(per_account).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
+9 -2
View File
@@ -27,6 +27,10 @@ pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
pub(crate) const RUN_LIMIT: usize = 200;
/// Feed requests in flight at once.
pub(crate) const RUN_CONCURRENCY: usize = 4;
/// Podcasts of one account considered per scheduled run. Due podcasts are
/// taken round-robin across accounts, so an account with many (or slow)
/// feeds cannot hold back everyone else's refreshes.
pub(crate) const RUN_PER_ACCOUNT: usize = 25;
/// Delay before the next attempt after `consecutive_failures` failures in a
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
@@ -150,7 +154,8 @@ pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroup
groups
}
/// Refreshes every podcast due at `now_ms` (at most `limit`), with at most
/// Refreshes podcasts due at `now_ms` (at most `limit`, at most
/// [`RUN_PER_ACCOUNT`] per account, accounts in turns), with at most
/// `concurrency` feed requests in flight. Accounts subscribed to the same
/// feed URL share one request. A failing feed never stops the others.
pub(crate) async fn refresh_due(
@@ -160,7 +165,9 @@ pub(crate) async fn refresh_due(
limit: usize,
concurrency: usize,
) -> Result<RunSummary> {
let due = store.podcasts_due_for_refresh(now_ms, limit).await?;
let due = store
.podcasts_due_for_refresh(now_ms, limit, RUN_PER_ACCOUNT)
.await?;
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
}