Share the publication budget fairly and keep published cards stable

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:52 +02:00
1 parent 14f310f145
commit 5b2973f958
2 files changed
+318 -67

No files matched your search

+310 -64
View File
@@ -59,7 +59,7 @@
//! and every other podcast is reconciled ([`load_shared`]), so revocation
//! stays fail-safe for each podcast on its own.
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
@@ -125,6 +125,9 @@ pub(crate) struct AccountShare {
pub(crate) card: Result<PodcastCardSpec, String>,
/// Stored privacy confirmation, if any (possibly stale).
pub(crate) confirmation: Option<ShareConfirmation>,
/// The catalog metadata comes from the server's own fetch of the feed,
/// not (yet) from a synced subscription register of a client.
pub(crate) meta_from_feed: bool,
}
impl AccountShare {
@@ -252,23 +255,26 @@ pub(crate) async fn load_account_share(
};
let share = apply::load_share(conn, user_id, podcast_id).await?;
let catalog = sqlx::query(
"SELECT feed_url, title, author, description, link, image_url
"SELECT feed_url, title, author, description, link, image_url, meta_from_feed
FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let catalog = match catalog {
Some(row) => Some(PodcastMeta {
feed_url: feed_url_col(&row, "feed_url")?,
title: row.try_get("title")?,
author: row.try_get("author")?,
description: row.try_get("description")?,
link: row.try_get("link")?,
image_url: row.try_get("image_url")?,
}),
None => None,
let (catalog, meta_from_feed) = match catalog {
Some(row) => (
Some(PodcastMeta {
feed_url: feed_url_col(&row, "feed_url")?,
title: row.try_get("title")?,
author: row.try_get("author")?,
description: row.try_get("description")?,
link: row.try_get("link")?,
image_url: row.try_get("image_url")?,
}),
row.try_get("meta_from_feed")?,
),
None => (None, false),
};
// Episodes of the current feed document only: references learned from
// the account's own queue or progress (`source = 'sync'`) and episodes
@@ -293,6 +299,7 @@ pub(crate) async fn load_account_share(
share,
card,
confirmation,
meta_from_feed,
}))
}
@@ -384,27 +391,46 @@ pub(crate) struct Aggregation {
/// Shared podcasts whose card cannot be built (no origin URL, invalid
/// metadata).
pub(crate) failed: usize,
/// Publishable podcasts left out by [`MAX_SERVER_SHARED_PODCASTS`].
/// Publishable podcasts left out by [`MAX_SERVER_SHARED_PODCASTS`]
/// (sorted).
pub(crate) over_limit: Vec<PodcastId>,
/// Shared `(account, podcast)` pairs left out because their stored data
/// does not decode ([`SharedLoad::unreadable`]).
pub(crate) unreadable: usize,
}
/// What the overlay publishes now: keeps the publication stable between
/// rounds.
#[derive(Debug, Clone, Default)]
pub(crate) struct Incumbents {
/// Podcasts with an active card of this node (`PodcastDht::published`).
pub(crate) published: BTreeSet<PodcastId>,
/// Representative account of each card in the previous round (kept in
/// memory; empty after a restart).
pub(crate) representatives: BTreeMap<PodcastId, i64>,
}
/// Computes what the server node publishes from the accounts' views.
///
/// * Only accounts for which [`AccountShare::publishable`] holds count.
/// * One state per podcast. The representative account is the one whose
/// card has no privacy warnings, then the smallest account id; its
/// registers and card are passed with `privacy_confirmed = true`
/// (no warnings, or the account confirmed exactly these URLs for its
/// current share decision), so `sync_shares` never holds anything back
/// on its own.
/// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, the ones shared the
/// longest first (smallest `hlc_ms` of a representative share decision,
/// then podcast id), so an existing publication is not displaced by a
/// newly shared podcast.
pub(crate) fn aggregate_share_states(shares: Vec<AccountShare>) -> Aggregation {
/// * One state per podcast, built from one *representative* account. Its
/// registers and card are passed with `privacy_confirmed = true` (no
/// warnings, or the account confirmed exactly these URLs for its current
/// share decision), so `sync_shares` never holds anything back on its
/// own. The representative is, in this order: an account whose card has
/// no privacy warnings; an account whose catalog comes from the server's
/// own fetch of the feed (`meta_from_feed`), not from metadata a client
/// synced; the previous round's representative, so another account
/// sharing the same podcast does not take over a published card; the
/// smallest account id.
/// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, shared fairly between
/// the accounts ([`allocate`]): an account whose podcasts fit in its fair
/// part of the budget never loses a published card because of other
/// accounts, whatever their share times say.
pub(crate) fn aggregate_share_states(
shares: Vec<AccountShare>,
incumbents: &Incumbents,
) -> Aggregation {
let mut aggregation = Aggregation::default();
let mut by_podcast: BTreeMap<PodcastId, Vec<AccountShare>> = BTreeMap::new();
for share in shares {
@@ -416,35 +442,46 @@ pub(crate) fn aggregate_share_states(shares: Vec<AccountShare>) -> Aggregation {
.or_default()
.push(share);
}
let mut chosen: Vec<(i64, AccountShare)> = Vec::new();
for (_, accounts) in by_podcast {
let shared_since = accounts
let mut candidates: BTreeMap<PodcastId, AccountShare> = BTreeMap::new();
let mut queues: BTreeMap<i64, Vec<QueuedPodcast>> = BTreeMap::new();
for (podcast_id, accounts) in by_podcast {
let previous = incumbents.representatives.get(&podcast_id).copied();
let publishable: Vec<&AccountShare> = accounts
.iter()
.filter(|share| share.publishable())
.filter_map(|share| share.share.as_ref().map(|record| record.hlc_ms))
.min();
let representative = accounts
.iter()
.filter(|share| share.publishable())
.min_by_key(|share| (!share.warnings().is_empty(), share.user_id));
.collect();
let representative = publishable.iter().min_by_key(|share| {
(
!share.warnings().is_empty(),
!share.meta_from_feed,
previous != Some(share.user_id),
share.user_id,
)
});
match representative {
Some(share) => chosen.push((shared_since.unwrap_or(i64::MAX), share.clone())),
Some(representative) => {
let published = incumbents.published.contains(&podcast_id);
for share in &publishable {
queues
.entry(share.user_id)
.or_default()
.push(QueuedPodcast {
newcomer: !published,
shared_at_ms: share.share.as_ref().map_or(i64::MAX, |r| r.hlc_ms),
podcast_id: podcast_id.clone(),
});
}
candidates.insert(podcast_id, (*representative).clone());
}
None if accounts.iter().any(AccountShare::pending_confirmation) => {
aggregation.pending_confirmation += 1;
}
None => aggregation.failed += 1,
}
}
chosen.sort_by(|(left_since, left), (right_since, right)| {
left_since.cmp(right_since).then_with(|| {
left.subscription
.podcast_id
.cmp(&right.subscription.podcast_id)
})
});
for (index, (_, share)) in chosen.into_iter().enumerate() {
let podcast_id = share.subscription.podcast_id.clone();
if index >= MAX_SERVER_SHARED_PODCASTS {
let selected = allocate(queues, &incumbents.published, MAX_SERVER_SHARED_PODCASTS);
for (podcast_id, share) in candidates {
if !selected.contains(&podcast_id) {
aggregation.over_limit.push(podcast_id);
continue;
}
@@ -461,15 +498,67 @@ pub(crate) fn aggregate_share_states(shares: Vec<AccountShare>) -> Aggregation {
privacy_confirmed: true,
});
}
aggregation.states.sort_by(|left, right| {
left.subscription
.podcast_id
.cmp(&right.subscription.podcast_id)
});
aggregation.over_limit.sort();
aggregation
}
/// One publishable podcast in an account's queue, in priority order.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
struct QueuedPodcast {
/// Not published by this node yet (published cards come first).
newcomer: bool,
/// The account's own share decision time: it only orders the account's
/// own podcasts, never those of other accounts.
shared_at_ms: i64,
podcast_id: PodcastId,
}
/// Picks at most `budget` podcasts from the accounts' queues, round-robin:
/// every round each account (by id) takes its next podcast not picked yet
/// (a podcast several accounts share counts once and is free for the
/// others). Inside a round already published podcasts go first, so the
/// last, partial round never replaces a published card by a new one.
///
/// So with `n` sharing accounts every account keeps at least
/// `min(its podcasts, budget / n)` of them, its published ones first; only
/// podcasts of accounts beyond their fair part are left out (and revoked),
/// newest first.
fn allocate(
mut queues: BTreeMap<i64, Vec<QueuedPodcast>>,
published: &BTreeSet<PodcastId>,
budget: usize,
) -> BTreeSet<PodcastId> {
for queue in queues.values_mut() {
// Popped from the end.
queue.sort_by(|left, right| right.cmp(left));
}
let mut selected = BTreeSet::new();
while selected.len() < budget {
let mut picks: Vec<(bool, i64, PodcastId)> = Vec::new();
for (user_id, queue) in &mut queues {
while let Some(next) = queue.pop() {
let podcast_id = next.podcast_id;
if !selected.contains(&podcast_id)
&& !picks.iter().any(|(_, _, picked)| *picked == podcast_id)
{
picks.push((!published.contains(&podcast_id), *user_id, podcast_id));
break;
}
}
}
if picks.is_empty() {
break;
}
picks.sort();
for (_, _, podcast_id) in picks {
if selected.len() >= budget {
break;
}
selected.insert(podcast_id);
}
}
selected
}
/// Outcome of one reconciliation round.
#[derive(Debug, Clone)]
pub(crate) struct ReconcileReport {
@@ -479,20 +568,60 @@ pub(crate) struct ReconcileReport {
pub(crate) stats: PodcastDhtSyncStats,
}
/// What the last round left over, kept in memory for the next round and
/// for the accounts' share status.
#[derive(Debug, Default)]
struct RoundMemory {
representatives: BTreeMap<PodcastId, i64>,
over_limit: BTreeSet<PodcastId>,
failed: BTreeSet<PodcastId>,
}
fn memory() -> std::sync::MutexGuard<'static, RoundMemory> {
static MEMORY: OnceLock<std::sync::Mutex<RoundMemory>> = OnceLock::new();
MEMORY
.get_or_init(Default::default)
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
/// One round: load the shared podcasts of every account (or of `users`),
/// aggregate them and let the overlay publish and revoke.
/// aggregate them against what the overlay publishes now and let it
/// publish and revoke.
pub(crate) async fn reconcile_once(
pool: &PgPool,
dht: &PodcastDht,
users: Option<&[i64]>,
) -> Result<ReconcileReport> {
let published = dht
.published()
.await
.map_err(|error| anyhow::anyhow!("podcast publications unavailable: {error}"))?
.into_iter()
.filter(|publication| publication.active)
.map(|publication| publication.podcast_id)
.collect();
let load = load_shared(pool, users).await?;
let mut aggregation = aggregate_share_states(load.shares);
let incumbents = Incumbents {
published,
representatives: memory().representatives.clone(),
};
let mut aggregation = aggregate_share_states(load.shares, &incumbents);
aggregation.unreadable = load.unreadable;
let stats = dht
.sync_shares(aggregation.states.clone())
.await
.map_err(|error| anyhow::anyhow!("podcast publication failed: {error}"))?;
{
let mut memory = memory();
memory.representatives = aggregation.representatives.clone();
memory.over_limit = aggregation.over_limit.iter().cloned().collect();
memory.failed = stats
.failed
.iter()
.map(|failure| failure.podcast_id.clone())
.collect();
}
Ok(ReconcileReport { aggregation, stats })
}
@@ -629,9 +758,18 @@ mod tests {
share: Some(share),
card,
confirmation: None,
meta_from_feed: true,
}
}
fn aggregate(shares: Vec<AccountShare>) -> Aggregation {
aggregate_share_states(shares, &Incumbents::default())
}
fn id(path: &str) -> PodcastId {
PodcastId::from_feed_url(&url(path))
}
fn confirm(share: &mut AccountShare) {
let card = share.card.clone().unwrap();
share.confirmation = Some(ShareConfirmation {
@@ -643,7 +781,7 @@ mod tests {
#[test]
fn one_card_per_podcast_from_the_smallest_sharing_account() {
let aggregation = aggregate_share_states(vec![
let aggregation = aggregate(vec![
account(7, "a.xml", true, 10),
account(3, "a.xml", true, 20),
account(1, "a.xml", false, 30),
@@ -668,7 +806,7 @@ mod tests {
let mut stale = account(1, "a.xml", true, 10);
// Subscribed again after the share decision: private again.
stale.subscription.generation = 3;
let aggregation = aggregate_share_states(vec![stale]);
let aggregation = aggregate(vec![stale]);
assert!(aggregation.states.is_empty());
assert_eq!(aggregation.failed, 0);
assert_eq!(aggregation.pending_confirmation, 0);
@@ -680,14 +818,14 @@ mod tests {
let unconfirmed = account(1, secret, true, 10);
assert!(!unconfirmed.warnings().is_empty());
assert!(unconfirmed.pending_confirmation());
let aggregation = aggregate_share_states(vec![unconfirmed.clone()]);
let aggregation = aggregate(vec![unconfirmed.clone()]);
assert!(aggregation.states.is_empty());
assert_eq!(aggregation.pending_confirmation, 1);
let mut confirmed = account(2, secret, true, 20);
confirm(&mut confirmed);
assert!(confirmed.publishable());
let aggregation = aggregate_share_states(vec![unconfirmed.clone(), confirmed.clone()]);
let aggregation = aggregate(vec![unconfirmed.clone(), confirmed.clone()]);
assert_eq!(aggregation.states.len(), 1);
assert_eq!(
aggregation
@@ -723,7 +861,7 @@ mod tests {
);
confirm(&mut moved_secret);
let plain = account(9, "a.xml", true, 20);
let aggregation = aggregate_share_states(vec![moved_secret, plain]);
let aggregation = aggregate(vec![moved_secret, plain]);
assert_eq!(
aggregation
.representatives
@@ -735,20 +873,128 @@ mod tests {
}
#[test]
fn the_server_cap_keeps_the_longest_shared_podcasts() {
fn inside_an_account_the_longest_shared_podcasts_go_first() {
let shares = (0..MAX_SERVER_SHARED_PODCASTS + 3)
.map(|index| account(1, &format!("{index}.xml"), true, 1_000 - index as i64))
.collect();
let aggregation = aggregate_share_states(shares);
let aggregation = aggregate(shares);
assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS);
assert_eq!(aggregation.over_limit.len(), 3);
// The three most recently shared (largest hlc) are left out.
let left_out: Vec<PodcastId> = (0..3)
.map(|index| PodcastId::from_feed_url(&url(&format!("{index}.xml"))))
let mut left_out: Vec<PodcastId> =
(0..3).map(|index| id(&format!("{index}.xml"))).collect();
left_out.sort();
assert_eq!(aggregation.over_limit, left_out);
}
#[test]
fn many_shares_of_another_account_never_unpublish_a_card() {
// A's card is published. B (a device with a lagging or forged
// clock: share times near 0) shares far more than the cap.
let mine = account(1, "mine.xml", true, 5_000);
let theirs: Vec<AccountShare> = (0..300)
.map(|index| account(2, &format!("b{index}.xml"), true, index))
.collect();
for id in left_out {
assert!(aggregation.over_limit.contains(&id));
let mut published: BTreeSet<PodcastId> = (0..MAX_SERVER_SHARED_PODCASTS - 1)
.map(|index| id(&format!("b{index}.xml")))
.collect();
published.insert(id("mine.xml"));
let incumbents = Incumbents {
published: published.clone(),
representatives: BTreeMap::new(),
};
let mut shares = theirs;
shares.push(mine);
let aggregation = aggregate_share_states(shares, &incumbents);
let states: BTreeSet<PodcastId> = aggregation
.states
.iter()
.map(|state| state.subscription.podcast_id.clone())
.collect();
assert_eq!(states.len(), MAX_SERVER_SHARED_PODCASTS);
// Nothing published is displaced; only B's new podcasts wait.
assert_eq!(states, published);
assert_eq!(
aggregation.over_limit.len(),
300 - (MAX_SERVER_SHARED_PODCASTS - 1)
);
assert!(!aggregation.over_limit.contains(&id("mine.xml")));
}
#[test]
fn the_budget_is_shared_fairly_between_accounts() {
// B alone filled the budget; A shares five podcasts afterwards.
let mut shares: Vec<AccountShare> = (0..MAX_SERVER_SHARED_PODCASTS)
.map(|index| account(2, &format!("b{index}.xml"), true, index as i64))
.collect();
let published: BTreeSet<PodcastId> = shares
.iter()
.map(|share| share.subscription.podcast_id.clone())
.collect();
for index in 0..5 {
shares.push(account(1, &format!("a{index}.xml"), true, 10_000 + index));
}
let incumbents = Incumbents {
published,
representatives: BTreeMap::new(),
};
let aggregation = aggregate_share_states(shares, &incumbents);
assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS);
for index in 0..5 {
assert_eq!(
aggregation
.representatives
.get(&id(&format!("a{index}.xml"))),
Some(&1),
"A gets its fair part"
);
}
// B, beyond its fair part, gives up its five most recently shared.
let mut left_out: Vec<PodcastId> = (MAX_SERVER_SHARED_PODCASTS - 5
..MAX_SERVER_SHARED_PODCASTS)
.map(|index| id(&format!("b{index}.xml")))
.collect();
left_out.sort();
assert_eq!(aggregation.over_limit, left_out);
// A podcast both share counts once and is free for both.
let shared = vec![
account(1, "both.xml", true, 1),
account(2, "both.xml", true, 1),
account(2, "b-only.xml", true, 2),
];
let selected = aggregate(shared);
assert_eq!(selected.states.len(), 2);
}
#[test]
fn a_published_card_keeps_its_representative_and_server_fetched_metadata() {
// Account 9 provides the published card; account 3 starts sharing
// the same podcast with its own (synced) metadata.
let provider = account(9, "a.xml", true, 10);
let mut newcomer = account(3, "a.xml", true, 20);
newcomer.card.as_mut().unwrap().title = "Rewritten".into();
let incumbents = Incumbents {
published: BTreeSet::from([id("a.xml")]),
representatives: BTreeMap::from([(id("a.xml"), 9)]),
};
let aggregation =
aggregate_share_states(vec![provider.clone(), newcomer.clone()], &incumbents);
assert_eq!(aggregation.representatives[&id("a.xml")], 9);
assert_eq!(
aggregation.states[0].card.as_ref().unwrap().title,
"Show a.xml"
);
// Metadata only a client synced loses to the server's own fetch,
// even for a smaller account id or the previous representative.
newcomer.meta_from_feed = false;
let aggregation = aggregate(vec![provider.clone(), newcomer.clone()]);
assert_eq!(aggregation.representatives[&id("a.xml")], 9);
let mut synced_provider = provider;
synced_provider.meta_from_feed = false;
newcomer.meta_from_feed = true;
let aggregation = aggregate_share_states(vec![synced_provider, newcomer], &incumbents);
assert_eq!(aggregation.representatives[&id("a.xml")], 3);
}
#[test]
+8 -3
View File
@@ -95,7 +95,10 @@ fn confirm_body(view: &api::ShareView) -> serde_json::Value {
}
async fn published_ids(pool: &sqlx::PgPool, users: &[i64]) -> Vec<(PodcastId, i64)> {
let aggregation = aggregate_share_states(load_shared(pool, Some(users)).await.unwrap().shares);
let aggregation = aggregate_share_states(
load_shared(pool, Some(users)).await.unwrap().shares,
&Default::default(),
);
aggregation.representatives.into_iter().collect()
}
@@ -278,8 +281,10 @@ async fn postgres_shared_podcasts_aggregate_across_accounts() {
done(share(&store, b, &podcast_id, json!({"shared": true})).await);
let published = published_ids(&pool, &users).await;
assert_eq!(published, vec![(podcast_id.clone(), a.min(b))]);
let aggregation =
aggregate_share_states(load_shared(&pool, Some(&users)).await.unwrap().shares);
let aggregation = aggregate_share_states(
load_shared(&pool, Some(&users)).await.unwrap().shares,
&Default::default(),
);
let state = &aggregation.states[0];
assert!(state.is_shared() && state.privacy_confirmed);
let card = state.card.as_ref().unwrap();