diff --git a/docs/administration.md b/docs/administration.md index 7895353..195e561 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -168,7 +168,10 @@ your progress, history, listening counts, device names or your account. because other accounts share many podcasts. When the limit is reached, only podcasts of accounts beyond their part wait (newest shares first; cards already published are kept before new ones), and the podcast page - says that the card is held back. + says that the card is held back. Members' caches keep at most 256 + records per server, including revocations of the last 24 hours, so a + new card also waits while recent revocations fill that room, and no + published card is given up for a card the federation would not store. - A podcast whose stored data cannot be read (for example after a damaged import) is treated as not shared, and its card is revoked; the other podcasts are still published and revoked as usual. diff --git a/docs/development.md b/docs/development.md index 9771e1a..6562442 100644 --- a/docs/development.md +++ b/docs/development.md @@ -248,7 +248,12 @@ only while the overlay runs. each account's queue holds its podcasts the node already publishes first, then by its own share time; in every round each account takes its next podcast not taken yet, published ones first inside a round. An - account's share times therefore only order its own podcasts. + account's share times therefore only order its own podcasts. A podcast + without a live record of the node (an active card or an unexpired + tombstone, `Incumbents::live_keys`) needs a new replica key; at most + `256 - live keys` of them are taken per round (`Incumbents::key_headroom`), + the others wait in `over_limit`, so a rebalance never displaces more + cards than replicas accept new ones for. `PodcastDht::sync_shares` publishes the result and tombstones everything else. The podcasts left out (`over_limit`) and those `sync_shares` reported as failed are kept in memory for `ShareView.held`. diff --git a/src/podcasts/discovery.rs b/src/podcasts/discovery.rs index 4889eee..40f64df 100644 --- a/src/podcasts/discovery.rs +++ b/src/podcasts/discovery.rs @@ -66,7 +66,8 @@ use std::time::Duration; use anyhow::Result; use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta, ShareRecord, SubscriptionRecord}; use music_dht::podcast_dht::{ - FeedUrlPrivacyWarning, PodcastCardSpec, PodcastDht, PodcastDhtSyncStats, PodcastShareState, + FeedUrlPrivacyWarning, MAX_PODCAST_RECORDS_PER_PUBLISHER, PodcastCardSpec, PodcastDht, + PodcastDhtSyncStats, PodcastShareState, PublishedPodcast, }; use sqlx::{PgConnection, PgPool, Row}; use tokio::sync::Notify; @@ -83,10 +84,12 @@ pub(crate) const RECONCILE_DEBOUNCE: Duration = Duration::from_secs(2); /// Podcasts the server node publishes at most. /// /// A replica keeps at most `MAX_PODCAST_RECORDS_PER_PUBLISHER` (256) records -/// of one publisher, tombstones included, and refuses a publisher over it. -/// The server is one publisher for all of its accounts, so it keeps room -/// for the tombstones of recently revoked cards (they live 24 h). Podcasts -/// beyond the cap stay unpublished (see [`aggregate_share_states`]). +/// of one publisher, tombstones included, and refuses a new +/// `(podcast_id, publisher)` key over it. The server is one publisher for +/// all of its accounts, so it keeps room for the tombstones of recently +/// revoked cards (they live 24 h). Podcasts beyond the cap stay unpublished +/// (see [`aggregate_share_states`]); a round also never asks for more keys +/// than replicas still accept ([`Incumbents::key_headroom`]). pub(crate) const MAX_SERVER_SHARED_PODCASTS: usize = 192; fn notifier() -> &'static Notify { @@ -432,16 +435,53 @@ pub(crate) struct Aggregation { } /// What the overlay publishes now: keeps the publication stable between -/// rounds. +/// rounds and within what replicas accept. #[derive(Debug, Clone, Default)] pub(crate) struct Incumbents { /// Podcasts with an active card of this node (`PodcastDht::published`). pub(crate) published: BTreeSet, + /// Podcasts of which replicas still hold a record of this node: active + /// cards and tombstones that have not expired. Each one occupies a + /// `(podcast_id, publisher)` key in the replicas; republishing it + /// reuses the key. + pub(crate) live_keys: BTreeSet, /// Representative account of each card in the previous round (kept in /// memory; empty after a restart). pub(crate) representatives: BTreeMap, } +impl Incumbents { + /// What the overlay publishes at `now_ms` (unix milliseconds). + pub(crate) fn from_published(publications: &[PublishedPodcast], now_ms: u64) -> Self { + let mut incumbents = Self::default(); + for publication in publications { + if publication.active { + incumbents.published.insert(publication.podcast_id.clone()); + } + if publication.active || publication.record.expires_at_ms() > now_ms { + incumbents.live_keys.insert(publication.podcast_id.clone()); + } + } + incumbents + } + + /// Whether publishing `podcast_id` needs a key replicas do not hold + /// for this node yet. + fn needs_new_key(&self, podcast_id: &PodcastId) -> bool { + !self.published.contains(podcast_id) && !self.live_keys.contains(podcast_id) + } + + /// New `(podcast_id, publisher)` keys replicas still accept from this + /// node: `MAX_PODCAST_RECORDS_PER_PUBLISHER` minus the records they + /// keep (active cards and unexpired tombstones, including the + /// tombstones of cards displaced in this very round, which replace + /// their card in place). + pub(crate) fn key_headroom(&self) -> usize { + let live = self.live_keys.union(&self.published).count(); + MAX_PODCAST_RECORDS_PER_PUBLISHER.saturating_sub(live) + } +} + /// Computes what the server node publishes from the accounts' views. /// /// * Only accounts for which [`AccountShare::publishable`] holds count. @@ -458,7 +498,10 @@ pub(crate) struct Incumbents { /// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, shared fairly between /// the accounts ([`allocate`]): an account whose podcasts fit in its fair /// part of the budget never loses a published card because of other -/// accounts, whatever their share times say. +/// accounts, whatever their share times say. A round never asks for more +/// new keys than replicas accept ([`Incumbents::key_headroom`]): a +/// podcast that would need one waits (`over_limit`) until tombstones +/// expire, and no published card is displaced for it. pub(crate) fn aggregate_share_states( shares: Vec, incumbents: &Incumbents, @@ -511,7 +554,7 @@ pub(crate) fn aggregate_share_states( None => aggregation.failed += 1, } } - let selected = allocate(queues, &incumbents.published, MAX_SERVER_SHARED_PODCASTS); + let selected = allocate(queues, incumbents, MAX_SERVER_SHARED_PODCASTS); for (podcast_id, share) in candidates { if !selected.contains(&podcast_id) { aggregation.over_limit.push(podcast_id); @@ -554,27 +597,45 @@ struct QueuedPodcast { /// `min(its podcasts, budget / n)` of them, its published ones first; only /// podcasts of accounts beyond their fair part are left out (and revoked), /// newest first. +/// +/// A podcast that needs a new replica key ([`Incumbents::needs_new_key`]) +/// is picked only while [`Incumbents::key_headroom`] lasts; otherwise it is +/// skipped (left out) and the account's next podcast is taken, so a +/// rebalance displaces at most as many published cards as replicas can +/// take new cards for. fn allocate( mut queues: BTreeMap>, - published: &BTreeSet, + incumbents: &Incumbents, budget: usize, ) -> BTreeSet { for queue in queues.values_mut() { // Popped from the end. queue.sort_by(|left, right| right.cmp(left)); } + let mut new_keys_left = incumbents.key_headroom(); let mut selected = BTreeSet::new(); while selected.len() < budget { let mut picks: Vec<(bool, i64, PodcastId)> = Vec::new(); for (user_id, queue) in &mut queues { while let Some(next) = queue.pop() { let podcast_id = next.podcast_id; - if !selected.contains(&podcast_id) - && !picks.iter().any(|(_, _, picked)| *picked == podcast_id) + if selected.contains(&podcast_id) + || picks.iter().any(|(_, _, picked)| *picked == podcast_id) { - picks.push((!published.contains(&podcast_id), *user_id, podcast_id)); - break; + continue; } + if incumbents.needs_new_key(&podcast_id) { + if new_keys_left == 0 { + continue; + } + new_keys_left -= 1; + } + picks.push(( + !incumbents.published.contains(&podcast_id), + *user_id, + podcast_id, + )); + break; } } if picks.is_empty() { @@ -640,19 +701,13 @@ pub(crate) async fn reconcile_once( dht: &PodcastDht, users: Option<&[i64]>, ) -> Result { - let published = dht + let publications = dht .published() .await - .map_err(|error| anyhow::anyhow!("podcast publications unavailable: {error}"))? - .into_iter() - .filter(|publication| publication.active) - .map(|publication| publication.podcast_id) - .collect(); + .map_err(|error| anyhow::anyhow!("podcast publications unavailable: {error}"))?; let load = load_shared(pool, users).await?; - let incumbents = Incumbents { - published, - representatives: memory().representatives.clone(), - }; + let mut incumbents = Incumbents::from_published(&publications, unix_now_ms()); + incumbents.representatives = memory().representatives.clone(); let mut aggregation = aggregate_share_states(load.shares, &incumbents); aggregation.unreadable = load.unreadable; let stats = dht @@ -672,6 +727,14 @@ pub(crate) async fn reconcile_once( Ok(ReconcileReport { aggregation, stats }) } +fn unix_now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }) +} + /// Runs the reconciler until the task is aborted (federation stop). pub(crate) async fn run(pool: PgPool, dht: Arc) { let mut interval = tokio::time::interval(RECONCILE_INTERVAL); @@ -947,7 +1010,7 @@ mod tests { published.insert(id("mine.xml")); let incumbents = Incumbents { published: published.clone(), - representatives: BTreeMap::new(), + ..Incumbents::default() }; let mut shares = theirs; shares.push(mine); @@ -982,7 +1045,7 @@ mod tests { } let incumbents = Incumbents { published, - representatives: BTreeMap::new(), + ..Incumbents::default() }; let aggregation = aggregate_share_states(shares, &incumbents); assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS); @@ -1013,6 +1076,120 @@ mod tests { assert_eq!(selected.states.len(), 2); } + /// Selected podcasts that need a new replica key, and live keys. + fn keys_asked(aggregation: &Aggregation, incumbents: &Incumbents) -> usize { + let new_keys = aggregation + .states + .iter() + .filter(|state| incumbents.needs_new_key(&state.subscription.podcast_id)) + .count(); + incumbents.live_keys.union(&incumbents.published).count() + new_keys + } + + #[test] + fn a_rebalance_never_asks_replicas_for_more_keys_than_they_accept() { + // B alone published the whole budget; A shares 96 new podcasts, so + // its fair part is 96, but replicas take only 256 - 192 = 64 new + // keys from this node (displaced cards keep theirs as tombstones). + let mut shares: Vec = (0..MAX_SERVER_SHARED_PODCASTS) + .map(|index| account(2, &format!("b{index}.xml"), true, index as i64)) + .collect(); + let published: BTreeSet = shares + .iter() + .map(|share| share.subscription.podcast_id.clone()) + .collect(); + for index in 0..96 { + shares.push(account(1, &format!("a{index}.xml"), true, 10_000 + index)); + } + let incumbents = Incumbents { + live_keys: published.clone(), + published, + ..Incumbents::default() + }; + assert_eq!( + incumbents.key_headroom(), + MAX_PODCAST_RECORDS_PER_PUBLISHER - MAX_SERVER_SHARED_PODCASTS + ); + let aggregation = aggregate_share_states(shares.clone(), &incumbents); + assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS); + assert!(keys_asked(&aggregation, &incumbents) <= MAX_PODCAST_RECORDS_PER_PUBLISHER); + let of = |user: i64| { + aggregation + .representatives + .values() + .filter(|representative| **representative == user) + .count() + }; + let headroom = MAX_PODCAST_RECORDS_PER_PUBLISHER - MAX_SERVER_SHARED_PODCASTS; + assert_eq!(of(1), headroom, "A gets as many as replicas accept"); + assert_eq!(of(2), MAX_SERVER_SHARED_PODCASTS - headroom); + // A's podcasts beyond the headroom wait; B's displaced ones too. + assert_eq!(aggregation.over_limit.len(), (96 - headroom) + headroom); + + // The next round (the displaced cards are tombstones now) asks for + // nothing new and displaces nothing more until they expire. + let mut next = Incumbents::default(); + for state in &aggregation.states { + next.published.insert(state.subscription.podcast_id.clone()); + } + next.live_keys = incumbents + .live_keys + .union(&next.published) + .cloned() + .collect(); + assert_eq!(next.key_headroom(), 0); + let again = aggregate_share_states(shares.clone(), &next); + let selected: BTreeSet = again + .states + .iter() + .map(|state| state.subscription.podcast_id.clone()) + .collect(); + assert_eq!(selected, next.published, "nothing moves without headroom"); + assert!(keys_asked(&again, &next) <= MAX_PODCAST_RECORDS_PER_PUBLISHER); + + // Once the tombstones expired, A gets the rest of its fair part. + next.live_keys = next.published.clone(); + let later = aggregate_share_states(shares, &next); + assert!(keys_asked(&later, &next) <= MAX_PODCAST_RECORDS_PER_PUBLISHER); + let of = |user: i64| { + later + .representatives + .values() + .filter(|representative| **representative == user) + .count() + }; + assert_eq!(of(1), 96); + assert_eq!(of(2), 96); + } + + #[test] + fn a_shared_tombstoned_podcast_reuses_its_key() { + // Every key is taken (192 cards, 64 tombstones): a podcast whose + // tombstone is still live can be published again, a new one waits. + let mut shares: Vec = (0..MAX_SERVER_SHARED_PODCASTS - 1) + .map(|index| account(2, &format!("b{index}.xml"), true, index as i64)) + .collect(); + let published: BTreeSet = shares + .iter() + .map(|share| share.subscription.podcast_id.clone()) + .collect(); + let mut live_keys = published.clone(); + for index in 0..65 { + live_keys.insert(id(&format!("gone{index}.xml"))); + } + shares.push(account(1, "gone0.xml", true, 1)); + shares.push(account(1, "fresh.xml", true, 0)); + let incumbents = Incumbents { + published, + live_keys, + ..Incumbents::default() + }; + assert_eq!(incumbents.key_headroom(), 0); + let aggregation = aggregate_share_states(shares, &incumbents); + assert_eq!(aggregation.representatives.get(&id("gone0.xml")), Some(&1)); + assert_eq!(aggregation.over_limit, vec![id("fresh.xml")]); + } + #[test] fn a_published_card_keeps_its_representative_and_server_fetched_metadata() { // Account 9 provides the published card; account 3 starts sharing @@ -1023,6 +1200,7 @@ mod tests { let incumbents = Incumbents { published: BTreeSet::from([id("a.xml")]), representatives: BTreeMap::from([(id("a.xml"), 9)]), + ..Incumbents::default() }; let aggregation = aggregate_share_states(vec![provider.clone(), newcomer.clone()], &incumbents);