Keep replica room for the cards a rebalance publishes

Replicas count unexpired tombstones toward the 256 records per
publisher and refuse new keys beyond it. A round now admits at most as
many podcasts that need a new key as that room allows, so a fair-share
rebalance never displaces cards for newcomers the replicas would refuse.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:52 +02:00
1 parent 4b45aa5805
commit 767134d1dd
3 files changed
+213 -27

No files matched your search

+203 -25
View File
@@ -66,7 +66,8 @@ use std::time::Duration;
use anyhow::Result;
use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta, ShareRecord, SubscriptionRecord};
use music_dht::podcast_dht::{
FeedUrlPrivacyWarning, PodcastCardSpec, PodcastDht, PodcastDhtSyncStats, PodcastShareState,
FeedUrlPrivacyWarning, MAX_PODCAST_RECORDS_PER_PUBLISHER, PodcastCardSpec, PodcastDht,
PodcastDhtSyncStats, PodcastShareState, PublishedPodcast,
};
use sqlx::{PgConnection, PgPool, Row};
use tokio::sync::Notify;
@@ -83,10 +84,12 @@ pub(crate) const RECONCILE_DEBOUNCE: Duration = Duration::from_secs(2);
/// Podcasts the server node publishes at most.
///
/// A replica keeps at most `MAX_PODCAST_RECORDS_PER_PUBLISHER` (256) records
/// of one publisher, tombstones included, and refuses a publisher over it.
/// The server is one publisher for all of its accounts, so it keeps room
/// for the tombstones of recently revoked cards (they live 24 h). Podcasts
/// beyond the cap stay unpublished (see [`aggregate_share_states`]).
/// of one publisher, tombstones included, and refuses a new
/// `(podcast_id, publisher)` key over it. The server is one publisher for
/// all of its accounts, so it keeps room for the tombstones of recently
/// revoked cards (they live 24 h). Podcasts beyond the cap stay unpublished
/// (see [`aggregate_share_states`]); a round also never asks for more keys
/// than replicas still accept ([`Incumbents::key_headroom`]).
pub(crate) const MAX_SERVER_SHARED_PODCASTS: usize = 192;
fn notifier() -> &'static Notify {
@@ -432,16 +435,53 @@ pub(crate) struct Aggregation {
}
/// What the overlay publishes now: keeps the publication stable between
/// rounds.
/// rounds and within what replicas accept.
#[derive(Debug, Clone, Default)]
pub(crate) struct Incumbents {
/// Podcasts with an active card of this node (`PodcastDht::published`).
pub(crate) published: BTreeSet<PodcastId>,
/// Podcasts of which replicas still hold a record of this node: active
/// cards and tombstones that have not expired. Each one occupies a
/// `(podcast_id, publisher)` key in the replicas; republishing it
/// reuses the key.
pub(crate) live_keys: BTreeSet<PodcastId>,
/// Representative account of each card in the previous round (kept in
/// memory; empty after a restart).
pub(crate) representatives: BTreeMap<PodcastId, i64>,
}
impl Incumbents {
/// What the overlay publishes at `now_ms` (unix milliseconds).
pub(crate) fn from_published(publications: &[PublishedPodcast], now_ms: u64) -> Self {
let mut incumbents = Self::default();
for publication in publications {
if publication.active {
incumbents.published.insert(publication.podcast_id.clone());
}
if publication.active || publication.record.expires_at_ms() > now_ms {
incumbents.live_keys.insert(publication.podcast_id.clone());
}
}
incumbents
}
/// Whether publishing `podcast_id` needs a key replicas do not hold
/// for this node yet.
fn needs_new_key(&self, podcast_id: &PodcastId) -> bool {
!self.published.contains(podcast_id) && !self.live_keys.contains(podcast_id)
}
/// New `(podcast_id, publisher)` keys replicas still accept from this
/// node: `MAX_PODCAST_RECORDS_PER_PUBLISHER` minus the records they
/// keep (active cards and unexpired tombstones, including the
/// tombstones of cards displaced in this very round, which replace
/// their card in place).
pub(crate) fn key_headroom(&self) -> usize {
let live = self.live_keys.union(&self.published).count();
MAX_PODCAST_RECORDS_PER_PUBLISHER.saturating_sub(live)
}
}
/// Computes what the server node publishes from the accounts' views.
///
/// * Only accounts for which [`AccountShare::publishable`] holds count.
@@ -458,7 +498,10 @@ pub(crate) struct Incumbents {
/// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, shared fairly between
/// the accounts ([`allocate`]): an account whose podcasts fit in its fair
/// part of the budget never loses a published card because of other
/// accounts, whatever their share times say.
/// accounts, whatever their share times say. A round never asks for more
/// new keys than replicas accept ([`Incumbents::key_headroom`]): a
/// podcast that would need one waits (`over_limit`) until tombstones
/// expire, and no published card is displaced for it.
pub(crate) fn aggregate_share_states(
shares: Vec<AccountShare>,
incumbents: &Incumbents,
@@ -511,7 +554,7 @@ pub(crate) fn aggregate_share_states(
None => aggregation.failed += 1,
}
}
let selected = allocate(queues, &incumbents.published, MAX_SERVER_SHARED_PODCASTS);
let selected = allocate(queues, incumbents, MAX_SERVER_SHARED_PODCASTS);
for (podcast_id, share) in candidates {
if !selected.contains(&podcast_id) {
aggregation.over_limit.push(podcast_id);
@@ -554,27 +597,45 @@ struct QueuedPodcast {
/// `min(its podcasts, budget / n)` of them, its published ones first; only
/// podcasts of accounts beyond their fair part are left out (and revoked),
/// newest first.
///
/// A podcast that needs a new replica key ([`Incumbents::needs_new_key`])
/// is picked only while [`Incumbents::key_headroom`] lasts; otherwise it is
/// skipped (left out) and the account's next podcast is taken, so a
/// rebalance displaces at most as many published cards as replicas can
/// take new cards for.
fn allocate(
mut queues: BTreeMap<i64, Vec<QueuedPodcast>>,
published: &BTreeSet<PodcastId>,
incumbents: &Incumbents,
budget: usize,
) -> BTreeSet<PodcastId> {
for queue in queues.values_mut() {
// Popped from the end.
queue.sort_by(|left, right| right.cmp(left));
}
let mut new_keys_left = incumbents.key_headroom();
let mut selected = BTreeSet::new();
while selected.len() < budget {
let mut picks: Vec<(bool, i64, PodcastId)> = Vec::new();
for (user_id, queue) in &mut queues {
while let Some(next) = queue.pop() {
let podcast_id = next.podcast_id;
if !selected.contains(&podcast_id)
&& !picks.iter().any(|(_, _, picked)| *picked == podcast_id)
if selected.contains(&podcast_id)
|| picks.iter().any(|(_, _, picked)| *picked == podcast_id)
{
picks.push((!published.contains(&podcast_id), *user_id, podcast_id));
break;
continue;
}
if incumbents.needs_new_key(&podcast_id) {
if new_keys_left == 0 {
continue;
}
new_keys_left -= 1;
}
picks.push((
!incumbents.published.contains(&podcast_id),
*user_id,
podcast_id,
));
break;
}
}
if picks.is_empty() {
@@ -640,19 +701,13 @@ pub(crate) async fn reconcile_once(
dht: &PodcastDht,
users: Option<&[i64]>,
) -> Result<ReconcileReport> {
let published = dht
let publications = dht
.published()
.await
.map_err(|error| anyhow::anyhow!("podcast publications unavailable: {error}"))?
.into_iter()
.filter(|publication| publication.active)
.map(|publication| publication.podcast_id)
.collect();
.map_err(|error| anyhow::anyhow!("podcast publications unavailable: {error}"))?;
let load = load_shared(pool, users).await?;
let incumbents = Incumbents {
published,
representatives: memory().representatives.clone(),
};
let mut incumbents = Incumbents::from_published(&publications, unix_now_ms());
incumbents.representatives = memory().representatives.clone();
let mut aggregation = aggregate_share_states(load.shares, &incumbents);
aggregation.unreadable = load.unreadable;
let stats = dht
@@ -672,6 +727,14 @@ pub(crate) async fn reconcile_once(
Ok(ReconcileReport { aggregation, stats })
}
fn unix_now_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |elapsed| {
u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX)
})
}
/// Runs the reconciler until the task is aborted (federation stop).
pub(crate) async fn run(pool: PgPool, dht: Arc<PodcastDht>) {
let mut interval = tokio::time::interval(RECONCILE_INTERVAL);
@@ -947,7 +1010,7 @@ mod tests {
published.insert(id("mine.xml"));
let incumbents = Incumbents {
published: published.clone(),
representatives: BTreeMap::new(),
..Incumbents::default()
};
let mut shares = theirs;
shares.push(mine);
@@ -982,7 +1045,7 @@ mod tests {
}
let incumbents = Incumbents {
published,
representatives: BTreeMap::new(),
..Incumbents::default()
};
let aggregation = aggregate_share_states(shares, &incumbents);
assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS);
@@ -1013,6 +1076,120 @@ mod tests {
assert_eq!(selected.states.len(), 2);
}
/// Selected podcasts that need a new replica key, and live keys.
fn keys_asked(aggregation: &Aggregation, incumbents: &Incumbents) -> usize {
let new_keys = aggregation
.states
.iter()
.filter(|state| incumbents.needs_new_key(&state.subscription.podcast_id))
.count();
incumbents.live_keys.union(&incumbents.published).count() + new_keys
}
#[test]
fn a_rebalance_never_asks_replicas_for_more_keys_than_they_accept() {
// B alone published the whole budget; A shares 96 new podcasts, so
// its fair part is 96, but replicas take only 256 - 192 = 64 new
// keys from this node (displaced cards keep theirs as tombstones).
let mut shares: Vec<AccountShare> = (0..MAX_SERVER_SHARED_PODCASTS)
.map(|index| account(2, &format!("b{index}.xml"), true, index as i64))
.collect();
let published: BTreeSet<PodcastId> = shares
.iter()
.map(|share| share.subscription.podcast_id.clone())
.collect();
for index in 0..96 {
shares.push(account(1, &format!("a{index}.xml"), true, 10_000 + index));
}
let incumbents = Incumbents {
live_keys: published.clone(),
published,
..Incumbents::default()
};
assert_eq!(
incumbents.key_headroom(),
MAX_PODCAST_RECORDS_PER_PUBLISHER - MAX_SERVER_SHARED_PODCASTS
);
let aggregation = aggregate_share_states(shares.clone(), &incumbents);
assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS);
assert!(keys_asked(&aggregation, &incumbents) <= MAX_PODCAST_RECORDS_PER_PUBLISHER);
let of = |user: i64| {
aggregation
.representatives
.values()
.filter(|representative| **representative == user)
.count()
};
let headroom = MAX_PODCAST_RECORDS_PER_PUBLISHER - MAX_SERVER_SHARED_PODCASTS;
assert_eq!(of(1), headroom, "A gets as many as replicas accept");
assert_eq!(of(2), MAX_SERVER_SHARED_PODCASTS - headroom);
// A's podcasts beyond the headroom wait; B's displaced ones too.
assert_eq!(aggregation.over_limit.len(), (96 - headroom) + headroom);
// The next round (the displaced cards are tombstones now) asks for
// nothing new and displaces nothing more until they expire.
let mut next = Incumbents::default();
for state in &aggregation.states {
next.published.insert(state.subscription.podcast_id.clone());
}
next.live_keys = incumbents
.live_keys
.union(&next.published)
.cloned()
.collect();
assert_eq!(next.key_headroom(), 0);
let again = aggregate_share_states(shares.clone(), &next);
let selected: BTreeSet<PodcastId> = again
.states
.iter()
.map(|state| state.subscription.podcast_id.clone())
.collect();
assert_eq!(selected, next.published, "nothing moves without headroom");
assert!(keys_asked(&again, &next) <= MAX_PODCAST_RECORDS_PER_PUBLISHER);
// Once the tombstones expired, A gets the rest of its fair part.
next.live_keys = next.published.clone();
let later = aggregate_share_states(shares, &next);
assert!(keys_asked(&later, &next) <= MAX_PODCAST_RECORDS_PER_PUBLISHER);
let of = |user: i64| {
later
.representatives
.values()
.filter(|representative| **representative == user)
.count()
};
assert_eq!(of(1), 96);
assert_eq!(of(2), 96);
}
#[test]
fn a_shared_tombstoned_podcast_reuses_its_key() {
// Every key is taken (192 cards, 64 tombstones): a podcast whose
// tombstone is still live can be published again, a new one waits.
let mut shares: Vec<AccountShare> = (0..MAX_SERVER_SHARED_PODCASTS - 1)
.map(|index| account(2, &format!("b{index}.xml"), true, index as i64))
.collect();
let published: BTreeSet<PodcastId> = shares
.iter()
.map(|share| share.subscription.podcast_id.clone())
.collect();
let mut live_keys = published.clone();
for index in 0..65 {
live_keys.insert(id(&format!("gone{index}.xml")));
}
shares.push(account(1, "gone0.xml", true, 1));
shares.push(account(1, "fresh.xml", true, 0));
let incumbents = Incumbents {
published,
live_keys,
..Incumbents::default()
};
assert_eq!(incumbents.key_headroom(), 0);
let aggregation = aggregate_share_states(shares, &incumbents);
assert_eq!(aggregation.representatives.get(&id("gone0.xml")), Some(&1));
assert_eq!(aggregation.over_limit, vec![id("fresh.xml")]);
}
#[test]
fn a_published_card_keeps_its_representative_and_server_fetched_metadata() {
// Account 9 provides the published card; account 3 starts sharing
@@ -1023,6 +1200,7 @@ mod tests {
let incumbents = Incumbents {
published: BTreeSet::from([id("a.xml")]),
representatives: BTreeMap::from([(id("a.xml"), 9)]),
..Incumbents::default()
};
let aggregation =
aggregate_share_states(vec![provider.clone(), newcomer.clone()], &incumbents);