From 5b2973f9583e1a297a5f41c483a2eff7d61e904f Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 23:58:52 +0200 Subject: [PATCH] Share the publication budget fairly and keep published cards stable Co-Authored-By: Claude Opus 5.5 --- src/podcasts/discovery.rs | 374 ++++++++++++++++++++++++++------ src/podcasts/discovery_tests.rs | 11 +- 2 files changed, 318 insertions(+), 67 deletions(-) diff --git a/src/podcasts/discovery.rs b/src/podcasts/discovery.rs index 9d40ec4..265056c 100644 --- a/src/podcasts/discovery.rs +++ b/src/podcasts/discovery.rs @@ -59,7 +59,7 @@ //! and every other podcast is reconciled ([`load_shared`]), so revocation //! stays fail-safe for each podcast on its own. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -125,6 +125,9 @@ pub(crate) struct AccountShare { pub(crate) card: Result, /// Stored privacy confirmation, if any (possibly stale). pub(crate) confirmation: Option, + /// The catalog metadata comes from the server's own fetch of the feed, + /// not (yet) from a synced subscription register of a client. + pub(crate) meta_from_feed: bool, } impl AccountShare { @@ -252,23 +255,26 @@ pub(crate) async fn load_account_share( }; let share = apply::load_share(conn, user_id, podcast_id).await?; let catalog = sqlx::query( - "SELECT feed_url, title, author, description, link, image_url + "SELECT feed_url, title, author, description, link, image_url, meta_from_feed FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2", ) .bind(user_id) .bind(podcast_id.as_str()) .fetch_optional(&mut *conn) .await?; - let catalog = match catalog { - Some(row) => Some(PodcastMeta { - feed_url: feed_url_col(&row, "feed_url")?, - title: row.try_get("title")?, - author: row.try_get("author")?, - description: row.try_get("description")?, - link: row.try_get("link")?, - image_url: row.try_get("image_url")?, - }), - None => None, + let (catalog, meta_from_feed) = match catalog { + Some(row) => ( + Some(PodcastMeta { + feed_url: feed_url_col(&row, "feed_url")?, + title: row.try_get("title")?, + author: row.try_get("author")?, + description: row.try_get("description")?, + link: row.try_get("link")?, + image_url: row.try_get("image_url")?, + }), + row.try_get("meta_from_feed")?, + ), + None => (None, false), }; // Episodes of the current feed document only: references learned from // the account's own queue or progress (`source = 'sync'`) and episodes @@ -293,6 +299,7 @@ pub(crate) async fn load_account_share( share, card, confirmation, + meta_from_feed, })) } @@ -384,27 +391,46 @@ pub(crate) struct Aggregation { /// Shared podcasts whose card cannot be built (no origin URL, invalid /// metadata). pub(crate) failed: usize, - /// Publishable podcasts left out by [`MAX_SERVER_SHARED_PODCASTS`]. + /// Publishable podcasts left out by [`MAX_SERVER_SHARED_PODCASTS`] + /// (sorted). pub(crate) over_limit: Vec, /// Shared `(account, podcast)` pairs left out because their stored data /// does not decode ([`SharedLoad::unreadable`]). pub(crate) unreadable: usize, } +/// What the overlay publishes now: keeps the publication stable between +/// rounds. +#[derive(Debug, Clone, Default)] +pub(crate) struct Incumbents { + /// Podcasts with an active card of this node (`PodcastDht::published`). + pub(crate) published: BTreeSet, + /// Representative account of each card in the previous round (kept in + /// memory; empty after a restart). + pub(crate) representatives: BTreeMap, +} + /// Computes what the server node publishes from the accounts' views. /// /// * Only accounts for which [`AccountShare::publishable`] holds count. -/// * One state per podcast. The representative account is the one whose -/// card has no privacy warnings, then the smallest account id; its -/// registers and card are passed with `privacy_confirmed = true` -/// (no warnings, or the account confirmed exactly these URLs for its -/// current share decision), so `sync_shares` never holds anything back -/// on its own. -/// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, the ones shared the -/// longest first (smallest `hlc_ms` of a representative share decision, -/// then podcast id), so an existing publication is not displaced by a -/// newly shared podcast. -pub(crate) fn aggregate_share_states(shares: Vec) -> Aggregation { +/// * One state per podcast, built from one *representative* account. Its +/// registers and card are passed with `privacy_confirmed = true` (no +/// warnings, or the account confirmed exactly these URLs for its current +/// share decision), so `sync_shares` never holds anything back on its +/// own. The representative is, in this order: an account whose card has +/// no privacy warnings; an account whose catalog comes from the server's +/// own fetch of the feed (`meta_from_feed`), not from metadata a client +/// synced; the previous round's representative, so another account +/// sharing the same podcast does not take over a published card; the +/// smallest account id. +/// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, shared fairly between +/// the accounts ([`allocate`]): an account whose podcasts fit in its fair +/// part of the budget never loses a published card because of other +/// accounts, whatever their share times say. +pub(crate) fn aggregate_share_states( + shares: Vec, + incumbents: &Incumbents, +) -> Aggregation { let mut aggregation = Aggregation::default(); let mut by_podcast: BTreeMap> = BTreeMap::new(); for share in shares { @@ -416,35 +442,46 @@ pub(crate) fn aggregate_share_states(shares: Vec) -> Aggregation { .or_default() .push(share); } - let mut chosen: Vec<(i64, AccountShare)> = Vec::new(); - for (_, accounts) in by_podcast { - let shared_since = accounts + let mut candidates: BTreeMap = BTreeMap::new(); + let mut queues: BTreeMap> = BTreeMap::new(); + for (podcast_id, accounts) in by_podcast { + let previous = incumbents.representatives.get(&podcast_id).copied(); + let publishable: Vec<&AccountShare> = accounts .iter() .filter(|share| share.publishable()) - .filter_map(|share| share.share.as_ref().map(|record| record.hlc_ms)) - .min(); - let representative = accounts - .iter() - .filter(|share| share.publishable()) - .min_by_key(|share| (!share.warnings().is_empty(), share.user_id)); + .collect(); + let representative = publishable.iter().min_by_key(|share| { + ( + !share.warnings().is_empty(), + !share.meta_from_feed, + previous != Some(share.user_id), + share.user_id, + ) + }); match representative { - Some(share) => chosen.push((shared_since.unwrap_or(i64::MAX), share.clone())), + Some(representative) => { + let published = incumbents.published.contains(&podcast_id); + for share in &publishable { + queues + .entry(share.user_id) + .or_default() + .push(QueuedPodcast { + newcomer: !published, + shared_at_ms: share.share.as_ref().map_or(i64::MAX, |r| r.hlc_ms), + podcast_id: podcast_id.clone(), + }); + } + candidates.insert(podcast_id, (*representative).clone()); + } None if accounts.iter().any(AccountShare::pending_confirmation) => { aggregation.pending_confirmation += 1; } None => aggregation.failed += 1, } } - chosen.sort_by(|(left_since, left), (right_since, right)| { - left_since.cmp(right_since).then_with(|| { - left.subscription - .podcast_id - .cmp(&right.subscription.podcast_id) - }) - }); - for (index, (_, share)) in chosen.into_iter().enumerate() { - let podcast_id = share.subscription.podcast_id.clone(); - if index >= MAX_SERVER_SHARED_PODCASTS { + let selected = allocate(queues, &incumbents.published, MAX_SERVER_SHARED_PODCASTS); + for (podcast_id, share) in candidates { + if !selected.contains(&podcast_id) { aggregation.over_limit.push(podcast_id); continue; } @@ -461,15 +498,67 @@ pub(crate) fn aggregate_share_states(shares: Vec) -> Aggregation { privacy_confirmed: true, }); } - aggregation.states.sort_by(|left, right| { - left.subscription - .podcast_id - .cmp(&right.subscription.podcast_id) - }); - aggregation.over_limit.sort(); aggregation } +/// One publishable podcast in an account's queue, in priority order. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct QueuedPodcast { + /// Not published by this node yet (published cards come first). + newcomer: bool, + /// The account's own share decision time: it only orders the account's + /// own podcasts, never those of other accounts. + shared_at_ms: i64, + podcast_id: PodcastId, +} + +/// Picks at most `budget` podcasts from the accounts' queues, round-robin: +/// every round each account (by id) takes its next podcast not picked yet +/// (a podcast several accounts share counts once and is free for the +/// others). Inside a round already published podcasts go first, so the +/// last, partial round never replaces a published card by a new one. +/// +/// So with `n` sharing accounts every account keeps at least +/// `min(its podcasts, budget / n)` of them, its published ones first; only +/// podcasts of accounts beyond their fair part are left out (and revoked), +/// newest first. +fn allocate( + mut queues: BTreeMap>, + published: &BTreeSet, + budget: usize, +) -> BTreeSet { + for queue in queues.values_mut() { + // Popped from the end. + queue.sort_by(|left, right| right.cmp(left)); + } + let mut selected = BTreeSet::new(); + while selected.len() < budget { + let mut picks: Vec<(bool, i64, PodcastId)> = Vec::new(); + for (user_id, queue) in &mut queues { + while let Some(next) = queue.pop() { + let podcast_id = next.podcast_id; + if !selected.contains(&podcast_id) + && !picks.iter().any(|(_, _, picked)| *picked == podcast_id) + { + picks.push((!published.contains(&podcast_id), *user_id, podcast_id)); + break; + } + } + } + if picks.is_empty() { + break; + } + picks.sort(); + for (_, _, podcast_id) in picks { + if selected.len() >= budget { + break; + } + selected.insert(podcast_id); + } + } + selected +} + /// Outcome of one reconciliation round. #[derive(Debug, Clone)] pub(crate) struct ReconcileReport { @@ -479,20 +568,60 @@ pub(crate) struct ReconcileReport { pub(crate) stats: PodcastDhtSyncStats, } +/// What the last round left over, kept in memory for the next round and +/// for the accounts' share status. +#[derive(Debug, Default)] +struct RoundMemory { + representatives: BTreeMap, + over_limit: BTreeSet, + failed: BTreeSet, +} + +fn memory() -> std::sync::MutexGuard<'static, RoundMemory> { + static MEMORY: OnceLock> = OnceLock::new(); + MEMORY + .get_or_init(Default::default) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// One round: load the shared podcasts of every account (or of `users`), -/// aggregate them and let the overlay publish and revoke. +/// aggregate them against what the overlay publishes now and let it +/// publish and revoke. pub(crate) async fn reconcile_once( pool: &PgPool, dht: &PodcastDht, users: Option<&[i64]>, ) -> Result { + let published = dht + .published() + .await + .map_err(|error| anyhow::anyhow!("podcast publications unavailable: {error}"))? + .into_iter() + .filter(|publication| publication.active) + .map(|publication| publication.podcast_id) + .collect(); let load = load_shared(pool, users).await?; - let mut aggregation = aggregate_share_states(load.shares); + let incumbents = Incumbents { + published, + representatives: memory().representatives.clone(), + }; + let mut aggregation = aggregate_share_states(load.shares, &incumbents); aggregation.unreadable = load.unreadable; let stats = dht .sync_shares(aggregation.states.clone()) .await .map_err(|error| anyhow::anyhow!("podcast publication failed: {error}"))?; + { + let mut memory = memory(); + memory.representatives = aggregation.representatives.clone(); + memory.over_limit = aggregation.over_limit.iter().cloned().collect(); + memory.failed = stats + .failed + .iter() + .map(|failure| failure.podcast_id.clone()) + .collect(); + } Ok(ReconcileReport { aggregation, stats }) } @@ -629,9 +758,18 @@ mod tests { share: Some(share), card, confirmation: None, + meta_from_feed: true, } } + fn aggregate(shares: Vec) -> Aggregation { + aggregate_share_states(shares, &Incumbents::default()) + } + + fn id(path: &str) -> PodcastId { + PodcastId::from_feed_url(&url(path)) + } + fn confirm(share: &mut AccountShare) { let card = share.card.clone().unwrap(); share.confirmation = Some(ShareConfirmation { @@ -643,7 +781,7 @@ mod tests { #[test] fn one_card_per_podcast_from_the_smallest_sharing_account() { - let aggregation = aggregate_share_states(vec![ + let aggregation = aggregate(vec![ account(7, "a.xml", true, 10), account(3, "a.xml", true, 20), account(1, "a.xml", false, 30), @@ -668,7 +806,7 @@ mod tests { let mut stale = account(1, "a.xml", true, 10); // Subscribed again after the share decision: private again. stale.subscription.generation = 3; - let aggregation = aggregate_share_states(vec![stale]); + let aggregation = aggregate(vec![stale]); assert!(aggregation.states.is_empty()); assert_eq!(aggregation.failed, 0); assert_eq!(aggregation.pending_confirmation, 0); @@ -680,14 +818,14 @@ mod tests { let unconfirmed = account(1, secret, true, 10); assert!(!unconfirmed.warnings().is_empty()); assert!(unconfirmed.pending_confirmation()); - let aggregation = aggregate_share_states(vec![unconfirmed.clone()]); + let aggregation = aggregate(vec![unconfirmed.clone()]); assert!(aggregation.states.is_empty()); assert_eq!(aggregation.pending_confirmation, 1); let mut confirmed = account(2, secret, true, 20); confirm(&mut confirmed); assert!(confirmed.publishable()); - let aggregation = aggregate_share_states(vec![unconfirmed.clone(), confirmed.clone()]); + let aggregation = aggregate(vec![unconfirmed.clone(), confirmed.clone()]); assert_eq!(aggregation.states.len(), 1); assert_eq!( aggregation @@ -723,7 +861,7 @@ mod tests { ); confirm(&mut moved_secret); let plain = account(9, "a.xml", true, 20); - let aggregation = aggregate_share_states(vec![moved_secret, plain]); + let aggregation = aggregate(vec![moved_secret, plain]); assert_eq!( aggregation .representatives @@ -735,20 +873,128 @@ mod tests { } #[test] - fn the_server_cap_keeps_the_longest_shared_podcasts() { + fn inside_an_account_the_longest_shared_podcasts_go_first() { let shares = (0..MAX_SERVER_SHARED_PODCASTS + 3) .map(|index| account(1, &format!("{index}.xml"), true, 1_000 - index as i64)) .collect(); - let aggregation = aggregate_share_states(shares); + let aggregation = aggregate(shares); assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS); - assert_eq!(aggregation.over_limit.len(), 3); // The three most recently shared (largest hlc) are left out. - let left_out: Vec = (0..3) - .map(|index| PodcastId::from_feed_url(&url(&format!("{index}.xml")))) + let mut left_out: Vec = + (0..3).map(|index| id(&format!("{index}.xml"))).collect(); + left_out.sort(); + assert_eq!(aggregation.over_limit, left_out); + } + + #[test] + fn many_shares_of_another_account_never_unpublish_a_card() { + // A's card is published. B (a device with a lagging or forged + // clock: share times near 0) shares far more than the cap. + let mine = account(1, "mine.xml", true, 5_000); + let theirs: Vec = (0..300) + .map(|index| account(2, &format!("b{index}.xml"), true, index)) .collect(); - for id in left_out { - assert!(aggregation.over_limit.contains(&id)); + let mut published: BTreeSet = (0..MAX_SERVER_SHARED_PODCASTS - 1) + .map(|index| id(&format!("b{index}.xml"))) + .collect(); + published.insert(id("mine.xml")); + let incumbents = Incumbents { + published: published.clone(), + representatives: BTreeMap::new(), + }; + let mut shares = theirs; + shares.push(mine); + let aggregation = aggregate_share_states(shares, &incumbents); + let states: BTreeSet = aggregation + .states + .iter() + .map(|state| state.subscription.podcast_id.clone()) + .collect(); + assert_eq!(states.len(), MAX_SERVER_SHARED_PODCASTS); + // Nothing published is displaced; only B's new podcasts wait. + assert_eq!(states, published); + assert_eq!( + aggregation.over_limit.len(), + 300 - (MAX_SERVER_SHARED_PODCASTS - 1) + ); + assert!(!aggregation.over_limit.contains(&id("mine.xml"))); + } + + #[test] + fn the_budget_is_shared_fairly_between_accounts() { + // B alone filled the budget; A shares five podcasts afterwards. + let mut shares: Vec = (0..MAX_SERVER_SHARED_PODCASTS) + .map(|index| account(2, &format!("b{index}.xml"), true, index as i64)) + .collect(); + let published: BTreeSet = shares + .iter() + .map(|share| share.subscription.podcast_id.clone()) + .collect(); + for index in 0..5 { + shares.push(account(1, &format!("a{index}.xml"), true, 10_000 + index)); } + let incumbents = Incumbents { + published, + representatives: BTreeMap::new(), + }; + let aggregation = aggregate_share_states(shares, &incumbents); + assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS); + for index in 0..5 { + assert_eq!( + aggregation + .representatives + .get(&id(&format!("a{index}.xml"))), + Some(&1), + "A gets its fair part" + ); + } + // B, beyond its fair part, gives up its five most recently shared. + let mut left_out: Vec = (MAX_SERVER_SHARED_PODCASTS - 5 + ..MAX_SERVER_SHARED_PODCASTS) + .map(|index| id(&format!("b{index}.xml"))) + .collect(); + left_out.sort(); + assert_eq!(aggregation.over_limit, left_out); + + // A podcast both share counts once and is free for both. + let shared = vec![ + account(1, "both.xml", true, 1), + account(2, "both.xml", true, 1), + account(2, "b-only.xml", true, 2), + ]; + let selected = aggregate(shared); + assert_eq!(selected.states.len(), 2); + } + + #[test] + fn a_published_card_keeps_its_representative_and_server_fetched_metadata() { + // Account 9 provides the published card; account 3 starts sharing + // the same podcast with its own (synced) metadata. + let provider = account(9, "a.xml", true, 10); + let mut newcomer = account(3, "a.xml", true, 20); + newcomer.card.as_mut().unwrap().title = "Rewritten".into(); + let incumbents = Incumbents { + published: BTreeSet::from([id("a.xml")]), + representatives: BTreeMap::from([(id("a.xml"), 9)]), + }; + let aggregation = + aggregate_share_states(vec![provider.clone(), newcomer.clone()], &incumbents); + assert_eq!(aggregation.representatives[&id("a.xml")], 9); + assert_eq!( + aggregation.states[0].card.as_ref().unwrap().title, + "Show a.xml" + ); + + // Metadata only a client synced loses to the server's own fetch, + // even for a smaller account id or the previous representative. + newcomer.meta_from_feed = false; + let aggregation = aggregate(vec![provider.clone(), newcomer.clone()]); + assert_eq!(aggregation.representatives[&id("a.xml")], 9); + let mut synced_provider = provider; + synced_provider.meta_from_feed = false; + newcomer.meta_from_feed = true; + let aggregation = aggregate_share_states(vec![synced_provider, newcomer], &incumbents); + assert_eq!(aggregation.representatives[&id("a.xml")], 3); } #[test] diff --git a/src/podcasts/discovery_tests.rs b/src/podcasts/discovery_tests.rs index 671bcf8..33eb0fb 100644 --- a/src/podcasts/discovery_tests.rs +++ b/src/podcasts/discovery_tests.rs @@ -95,7 +95,10 @@ fn confirm_body(view: &api::ShareView) -> serde_json::Value { } async fn published_ids(pool: &sqlx::PgPool, users: &[i64]) -> Vec<(PodcastId, i64)> { - let aggregation = aggregate_share_states(load_shared(pool, Some(users)).await.unwrap().shares); + let aggregation = aggregate_share_states( + load_shared(pool, Some(users)).await.unwrap().shares, + &Default::default(), + ); aggregation.representatives.into_iter().collect() } @@ -278,8 +281,10 @@ async fn postgres_shared_podcasts_aggregate_across_accounts() { done(share(&store, b, &podcast_id, json!({"shared": true})).await); let published = published_ids(&pool, &users).await; assert_eq!(published, vec![(podcast_id.clone(), a.min(b))]); - let aggregation = - aggregate_share_states(load_shared(&pool, Some(&users)).await.unwrap().shares); + let aggregation = aggregate_share_states( + load_shared(&pool, Some(&users)).await.unwrap().shares, + &Default::default(), + ); let state = &aggregation.states[0]; assert!(state.is_shared() && state.privacy_confirmed); let card = state.card.as_ref().unwrap();