forked from Furumi/furumi-web
Decide which shared podcasts the server publishes
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e0f39f503f
commit
0400a08718
5 files changed
+847
-5
No files matched your search
+92
-3
@@ -246,6 +246,9 @@ impl PodcastStore {
|
||||
})
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
// A new subscription is private, but it supersedes an earlier share
|
||||
// decision of a former subscription.
|
||||
super::discovery::request_reconcile();
|
||||
Ok(SubscribeOutcome {
|
||||
podcast_id,
|
||||
already_subscribed: false,
|
||||
@@ -294,7 +297,9 @@ impl PodcastStore {
|
||||
.await?;
|
||||
let revoke_payload = share_payload(&mut tx, user_id, podcast_id, false).await?;
|
||||
let revoke = emit(&mut tx, user_id, &device_id, |_| revoke_payload.clone()).await?;
|
||||
super::discovery::clear_confirmation(&mut tx, user_id, podcast_id).await?;
|
||||
tx.commit().await?;
|
||||
super::discovery::request_reconcile();
|
||||
Ok(vec![unsubscribe, revoke])
|
||||
}
|
||||
|
||||
@@ -304,9 +309,10 @@ impl PodcastStore {
|
||||
/// is already effectively shared (`effective_share`). Revoking is a no-op
|
||||
/// when no share decision says `true`. The write uses the next podcast
|
||||
/// generation, so it orders after the subscription it observed.
|
||||
// The publication toggle is part of public discovery (MR-W3); until
|
||||
// then only the tests call it.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
///
|
||||
/// Revoking also forgets the account's privacy confirmation, so sharing
|
||||
/// again asks again. Publishing and revoking the card is the
|
||||
/// reconciler's job (`super::discovery`), which this requests.
|
||||
pub(crate) async fn set_shared(
|
||||
&self,
|
||||
user_id: i64,
|
||||
@@ -332,10 +338,93 @@ impl PodcastStore {
|
||||
}
|
||||
let payload = share_payload(&mut tx, user_id, podcast_id, shared).await?;
|
||||
let op = emit(&mut tx, user_id, &device_id, |_| payload.clone()).await?;
|
||||
if !shared {
|
||||
super::discovery::clear_confirmation(&mut tx, user_id, podcast_id).await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
super::discovery::request_reconcile();
|
||||
Ok(vec![op])
|
||||
}
|
||||
|
||||
/// Shares a podcast whose card has privacy warnings, with the
|
||||
/// account's confirmation of exactly `feed_url` / `id_feed_url`.
|
||||
///
|
||||
/// In one transaction: the card the account would publish now is
|
||||
/// rebuilt and compared with the confirmed URLs (a feed that moved
|
||||
/// meanwhile returns `Ok(None)` and writes nothing, so the user is asked
|
||||
/// about the new URLs); the share decision is written like
|
||||
/// [`PodcastStore::set_shared`] when the podcast is not already
|
||||
/// effectively shared; the confirmation is stored for the resulting
|
||||
/// share decision (`ShareRecord::op_id`). Returns the emitted operations.
|
||||
pub(crate) async fn share_confirmed(
|
||||
&self,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
feed_url: &FeedUrl,
|
||||
id_feed_url: Option<&FeedUrl>,
|
||||
now_ms: i64,
|
||||
) -> Result<Option<Vec<SyncOpWire>>> {
|
||||
let device_id = devices::local_device_id(&self.pool, user_id).await?;
|
||||
let mut tx = self.begin_write(user_id).await?;
|
||||
let Some(account) =
|
||||
super::discovery::load_account_share(&mut tx, user_id, podcast_id).await?
|
||||
else {
|
||||
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
|
||||
};
|
||||
if !account.subscription.subscribed {
|
||||
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
|
||||
}
|
||||
let card = account.card.as_ref().map_err(invalid)?;
|
||||
if card.feed_url != *feed_url || card.id_feed_url.as_ref() != id_feed_url {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut ops = Vec::new();
|
||||
let share_op_id = if account.is_shared() {
|
||||
account
|
||||
.share
|
||||
.as_ref()
|
||||
.map(|share| share.op_id.clone())
|
||||
.expect("an effective share has a share register")
|
||||
} else {
|
||||
let payload = share_payload(&mut tx, user_id, podcast_id, true).await?;
|
||||
let op = emit(&mut tx, user_id, &device_id, |_| payload.clone()).await?;
|
||||
let op_id = op.op_id.clone();
|
||||
ops.push(op);
|
||||
op_id
|
||||
};
|
||||
let confirmation = super::discovery::ShareConfirmation {
|
||||
share_op_id,
|
||||
feed_url: card.feed_url.clone(),
|
||||
id_feed_url: card.id_feed_url.clone(),
|
||||
};
|
||||
super::discovery::store_confirmation(&mut tx, user_id, podcast_id, &confirmation, now_ms)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
super::discovery::request_reconcile();
|
||||
Ok(Some(ops))
|
||||
}
|
||||
|
||||
/// Forgets the account's privacy confirmation of a podcast.
|
||||
pub(crate) async fn clear_share_confirmation(
|
||||
&self,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<()> {
|
||||
let mut conn = self.pool.acquire().await?;
|
||||
super::discovery::clear_confirmation(&mut conn, user_id, podcast_id).await
|
||||
}
|
||||
|
||||
/// One account's publication view of a podcast (registers, card,
|
||||
/// confirmation); `None` without a subscription register.
|
||||
pub(crate) async fn account_share(
|
||||
&self,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<super::discovery::AccountShare>> {
|
||||
let mut conn = self.pool.acquire().await?;
|
||||
super::discovery::load_account_share(&mut conn, user_id, podcast_id).await
|
||||
}
|
||||
|
||||
async fn explicit_progress(
|
||||
&self,
|
||||
user_id: i64,
|
||||
|
||||
@@ -648,6 +648,18 @@ pub(crate) async fn apply_op(
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an applied operation can change what is published
|
||||
/// (`super::discovery`): subscription and share registers, and the alias set
|
||||
/// the card's origin URL comes from.
|
||||
fn affects_publication(payload: &SyncOpPayload) -> bool {
|
||||
matches!(
|
||||
payload,
|
||||
SyncOpPayload::PodcastSubscriptionSet { .. }
|
||||
| SyncOpPayload::PodcastShareSet { .. }
|
||||
| SyncOpPayload::PodcastFeedAliasesAdded { .. }
|
||||
)
|
||||
}
|
||||
|
||||
fn record_outcome(stats: &mut SnapshotApplyStats, result: Result<bool>) -> Result<()> {
|
||||
match result {
|
||||
Ok(true) => stats.changed += 1,
|
||||
@@ -682,6 +694,9 @@ impl PodcastStore {
|
||||
let mut tx = self.begin_write(user_id).await?;
|
||||
let outcome = apply_op(&mut tx, user_id, op).await?;
|
||||
tx.commit().await?;
|
||||
if outcome == ApplyOutcome::Changed && affects_publication(&op.payload) {
|
||||
super::discovery::request_reconcile();
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
@@ -737,6 +752,11 @@ impl PodcastStore {
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
if stats.changed > 0
|
||||
&& !(snapshot.podcast_subscriptions.is_empty() && snapshot.podcast_shares.is_empty())
|
||||
{
|
||||
super::discovery::request_reconcile();
|
||||
}
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,715 @@
|
||||
//! Public discovery of shared podcasts (`music_dht::podcast_dht`).
|
||||
//!
|
||||
//! The web server is one federation node that serves many accounts. Its
|
||||
//! node identity (the persistent iroh key stored in PostgreSQL) is the only
|
||||
//! key the server can sign with, so **the server node publishes**, never an
|
||||
//! account: a card carries the server's key as `publisher` and nothing that
|
||||
//! identifies the account (frid `PODCASTS.md`, "Card").
|
||||
//!
|
||||
//! # Rule for several accounts
|
||||
//!
|
||||
//! * A podcast is published only when at least one active account
|
||||
//! explicitly shares it: `effective_share` of that account's merged registers holds
|
||||
//! (subscribed, shared, and the share decision orders after the latest
|
||||
//! subscription write). Every other podcast of every account is private
|
||||
//! and produces no network traffic at all.
|
||||
//! * The server publishes **one card per podcast**, whatever the number of
|
||||
//! accounts sharing it. Cards are keyed by `(podcast_id, publisher)`, so
|
||||
//! two accounts can never produce conflicting records: the card is built
|
||||
//! from one sharing account (the *representative*, see
|
||||
//! [`aggregate_share_states`]) and stays published while any account
|
||||
//! still shares it.
|
||||
//! * Revoking (share off, unsubscribing, on any device of the account)
|
||||
//! removes the account from the set. The card is tombstoned when no
|
||||
//! account shares the podcast any more. This is the §3 rule "turning
|
||||
//! sharing off revokes the user's own publication and does not remove
|
||||
//! publications made by other users": the card of the server publisher
|
||||
//! holds no per-account data, so while another account still shares the
|
||||
//! podcast nothing published is attributable to the account that
|
||||
//! stopped; the shared card is the other account's publication.
|
||||
//! * A late or stale device state cannot resurrect a publication: only the
|
||||
//! merged registers count, and an old `shared = true` loses to the newer
|
||||
//! share decision (`ShareRecord` LWW). [`PodcastDht::sync_shares`] is
|
||||
//! declarative: everything not in the computed set is unpublished.
|
||||
//!
|
||||
//! # Privacy confirmation
|
||||
//!
|
||||
//! `PodcastShareState::privacy_confirmed` is client-held state. The web
|
||||
//! keeps it per account in `furumusic__podcast_share_confirmation` (never
|
||||
//! synced), bound to the share decision (`share_op_id`) and to the exact
|
||||
//! card URLs the user confirmed. A card with privacy warnings
|
||||
//! (`PodcastCardSpec::privacy_warnings`: a token in the query, a member-feed
|
||||
//! host, ...) is published only when its account confirmed those URLs for
|
||||
//! its current share decision; otherwise the podcast is *pending
|
||||
//! confirmation* for that account and the account does not count as
|
||||
//! sharing it. A share decision taken on another device (for example a TUI
|
||||
//! that asked its own user) therefore still asks the web user, and a feed
|
||||
//! that moves to a URL that looks private asks again.
|
||||
//!
|
||||
//! # Reconciler
|
||||
//!
|
||||
//! [`run`] recomputes the set and calls [`PodcastDht::sync_shares`] after
|
||||
//! every change that can affect it ([`request_reconcile`]: share toggles,
|
||||
//! subscribe and unsubscribe, received sync operations and snapshots, feed
|
||||
//! refreshes of shared podcasts) and every [`RECONCILE_INTERVAL`]. A
|
||||
//! database error skips the round instead of unpublishing everything; the
|
||||
//! overlay's own maintenance republishes the cards in the meantime.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta, ShareRecord, SubscriptionRecord};
|
||||
use music_dht::podcast_dht::{
|
||||
FeedUrlPrivacyWarning, PodcastCardSpec, PodcastDht, PodcastDhtSyncStats, PodcastShareState,
|
||||
};
|
||||
use sqlx::{PgConnection, PgPool, Row};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::{apply, feed_url_col, podcast_id_col};
|
||||
|
||||
/// Periodic reconciliation, also after missed notifications.
|
||||
pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// Wait after a change notification so that a burst of changes (a sync
|
||||
/// batch, a refresh run) becomes one round.
|
||||
pub(crate) const RECONCILE_DEBOUNCE: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Podcasts the server node publishes at most.
|
||||
///
|
||||
/// A replica keeps at most `MAX_PODCAST_RECORDS_PER_PUBLISHER` (256) records
|
||||
/// of one publisher, tombstones included, and refuses a publisher over it.
|
||||
/// The server is one publisher for all of its accounts, so it keeps room
|
||||
/// for the tombstones of recently revoked cards (they live 24 h). Podcasts
|
||||
/// beyond the cap stay unpublished (see [`aggregate_share_states`]).
|
||||
pub(crate) const MAX_SERVER_SHARED_PODCASTS: usize = 192;
|
||||
|
||||
fn notifier() -> &'static Notify {
|
||||
static NOTIFY: OnceLock<Notify> = OnceLock::new();
|
||||
NOTIFY.get_or_init(Notify::new)
|
||||
}
|
||||
|
||||
/// Asks the running reconciler (if any) for a round soon. Cheap and
|
||||
/// idempotent; call it after committing a change.
|
||||
pub(crate) fn request_reconcile() {
|
||||
notifier().notify_one();
|
||||
}
|
||||
|
||||
/// A confirmation that an account may publish a card despite its privacy
|
||||
/// warnings.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct ShareConfirmation {
|
||||
/// Share decision the confirmation belongs to (`ShareRecord::op_id`).
|
||||
pub(crate) share_op_id: String,
|
||||
/// Confirmed `PodcastCardSpec::feed_url`.
|
||||
pub(crate) feed_url: FeedUrl,
|
||||
/// Confirmed `PodcastCardSpec::id_feed_url`.
|
||||
pub(crate) id_feed_url: Option<FeedUrl>,
|
||||
}
|
||||
|
||||
/// One account's view of the publication of one podcast.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct AccountShare {
|
||||
/// Account.
|
||||
pub(crate) user_id: i64,
|
||||
/// Merged subscription register with its alias set.
|
||||
pub(crate) subscription: SubscriptionRecord,
|
||||
/// Merged share register.
|
||||
pub(crate) share: Option<ShareRecord>,
|
||||
/// Card this account would publish, or why it cannot be built.
|
||||
pub(crate) card: Result<PodcastCardSpec, String>,
|
||||
/// Stored privacy confirmation, if any (possibly stale).
|
||||
pub(crate) confirmation: Option<ShareConfirmation>,
|
||||
}
|
||||
|
||||
impl AccountShare {
|
||||
/// Whether the account's registers say "publish" (`effective_share`).
|
||||
pub(crate) fn is_shared(&self) -> bool {
|
||||
music_dht::podcast::effective_share(Some(&self.subscription), self.share.as_ref())
|
||||
}
|
||||
|
||||
/// Privacy warnings of the card (empty when the card cannot be built).
|
||||
pub(crate) fn warnings(&self) -> Vec<FeedUrlPrivacyWarning> {
|
||||
self.card
|
||||
.as_ref()
|
||||
.map(PodcastCardSpec::privacy_warnings)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Whether the stored confirmation covers the current share decision
|
||||
/// and exactly the current card URLs.
|
||||
pub(crate) fn privacy_confirmed(&self) -> bool {
|
||||
let (Some(confirmation), Some(share), Ok(card)) =
|
||||
(&self.confirmation, &self.share, &self.card)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
confirmation.share_op_id == share.op_id
|
||||
&& confirmation.feed_url == card.feed_url
|
||||
&& confirmation.id_feed_url == card.id_feed_url
|
||||
}
|
||||
|
||||
/// Shared, but the card has privacy warnings the account has not
|
||||
/// confirmed for this share decision.
|
||||
pub(crate) fn pending_confirmation(&self) -> bool {
|
||||
self.is_shared()
|
||||
&& self.card.is_ok()
|
||||
&& !self.warnings().is_empty()
|
||||
&& !self.privacy_confirmed()
|
||||
}
|
||||
|
||||
/// Whether the account counts as sharing the podcast publicly.
|
||||
pub(crate) fn publishable(&self) -> bool {
|
||||
self.is_shared()
|
||||
&& self.card.is_ok()
|
||||
&& (self.warnings().is_empty() || self.privacy_confirmed())
|
||||
}
|
||||
}
|
||||
|
||||
/// The card an account would publish: the public projection of its
|
||||
/// subscription (`PodcastCardSpec::from_subscription`), with the URL it
|
||||
/// fetches and the channel metadata from its catalog, the origin URL as
|
||||
/// `id_feed_url` when the feed moved, and the feed's episode count and
|
||||
/// newest publication time. Never progress, history, device or account
|
||||
/// data.
|
||||
pub(crate) fn build_card(
|
||||
subscription: &SubscriptionRecord,
|
||||
catalog: Option<&PodcastMeta>,
|
||||
episode_count: Option<u32>,
|
||||
latest_episode_at_ms: Option<i64>,
|
||||
) -> Result<PodcastCardSpec, String> {
|
||||
let mut card = PodcastCardSpec::from_subscription(subscription).map_err(|e| e.to_string())?;
|
||||
let origin = card
|
||||
.id_feed_url
|
||||
.clone()
|
||||
.unwrap_or_else(|| card.feed_url.clone());
|
||||
if let Some(meta) = catalog {
|
||||
card.feed_url = meta.feed_url.clone();
|
||||
card.title = meta.title.clone();
|
||||
card.author = meta.author.clone();
|
||||
card.description = meta.description.clone();
|
||||
card.link = meta.link.clone();
|
||||
card.image_url = meta.image_url.clone();
|
||||
}
|
||||
card.id_feed_url = (origin != card.feed_url).then_some(origin);
|
||||
card.episode_count = episode_count;
|
||||
card.latest_episode_at_ms = latest_episode_at_ms;
|
||||
let card = card.bounded();
|
||||
card.validate().map_err(|e| e.to_string())?;
|
||||
Ok(card)
|
||||
}
|
||||
|
||||
async fn load_confirmation(
|
||||
conn: &mut PgConnection,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<ShareConfirmation>> {
|
||||
let row = sqlx::query(
|
||||
"SELECT share_op_id, feed_url, id_feed_url
|
||||
FROM furumusic__podcast_share_confirmation
|
||||
WHERE user_id = $1 AND podcast_id = $2",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(podcast_id.as_str())
|
||||
.fetch_optional(&mut *conn)
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let id_feed_url = match row.try_get::<Option<String>, _>("id_feed_url")? {
|
||||
Some(url) => match FeedUrl::try_from(url) {
|
||||
Ok(url) => Some(url),
|
||||
// A row that no longer parses confirms nothing.
|
||||
Err(_) => return Ok(None),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let Ok(feed_url) = feed_url_col(&row, "feed_url") else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(ShareConfirmation {
|
||||
share_op_id: row.try_get("share_op_id")?,
|
||||
feed_url,
|
||||
id_feed_url,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Loads one account's publication view of a podcast; `None` without a
|
||||
/// subscription register.
|
||||
pub(crate) async fn load_account_share(
|
||||
conn: &mut PgConnection,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<AccountShare>> {
|
||||
let Some(subscription) = apply::load_subscription_register(conn, user_id, podcast_id).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let share = apply::load_share(conn, user_id, podcast_id).await?;
|
||||
let catalog = sqlx::query(
|
||||
"SELECT feed_url, title, author, description, link, image_url
|
||||
FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(podcast_id.as_str())
|
||||
.fetch_optional(&mut *conn)
|
||||
.await?;
|
||||
let catalog = match catalog {
|
||||
Some(row) => Some(PodcastMeta {
|
||||
feed_url: feed_url_col(&row, "feed_url")?,
|
||||
title: row.try_get("title")?,
|
||||
author: row.try_get("author")?,
|
||||
description: row.try_get("description")?,
|
||||
link: row.try_get("link")?,
|
||||
image_url: row.try_get("image_url")?,
|
||||
}),
|
||||
None => None,
|
||||
};
|
||||
// Episodes of the current feed document only: references learned from
|
||||
// the account's own queue or progress (`source = 'sync'`) and episodes
|
||||
// that left the feed are not part of the public projection.
|
||||
let stats = sqlx::query(
|
||||
"SELECT COUNT(*) AS episodes, MAX(published_at_ms) AS latest
|
||||
FROM furumusic__podcast_episode
|
||||
WHERE user_id = $1 AND podcast_id = $2 AND source = 'feed' AND in_feed",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(podcast_id.as_str())
|
||||
.fetch_one(&mut *conn)
|
||||
.await?;
|
||||
let episodes: i64 = stats.try_get("episodes")?;
|
||||
let latest: Option<i64> = stats.try_get("latest")?;
|
||||
let episode_count = (episodes > 0).then(|| u32::try_from(episodes).unwrap_or(u32::MAX));
|
||||
let card = build_card(&subscription, catalog.as_ref(), episode_count, latest);
|
||||
let confirmation = load_confirmation(conn, user_id, podcast_id).await?;
|
||||
Ok(Some(AccountShare {
|
||||
user_id,
|
||||
subscription,
|
||||
share,
|
||||
card,
|
||||
confirmation,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Every `(account, podcast)` whose registers say subscribed and shared,
|
||||
/// for active accounts only (a deactivated account publishes nothing),
|
||||
/// optionally restricted to `users` (tests run against a shared database).
|
||||
/// `effective_share` is decided in Rust ([`AccountShare::is_shared`]).
|
||||
pub(crate) async fn load_shared(pool: &PgPool, users: Option<&[i64]>) -> Result<Vec<AccountShare>> {
|
||||
let mut conn = pool.acquire().await?;
|
||||
let rows = sqlx::query(
|
||||
"SELECT s.user_id, s.podcast_id
|
||||
FROM furumusic__podcast_subscription s
|
||||
JOIN furumusic__podcast_share sh
|
||||
ON sh.user_id = s.user_id AND sh.podcast_id = s.podcast_id
|
||||
JOIN furumusic__user u ON u.id = s.user_id AND u.is_active
|
||||
WHERE s.subscribed AND sh.shared
|
||||
AND ($1::BIGINT[] IS NULL OR s.user_id = ANY($1))
|
||||
ORDER BY s.podcast_id COLLATE \"C\", s.user_id",
|
||||
)
|
||||
.bind(users)
|
||||
.fetch_all(&mut *conn)
|
||||
.await?;
|
||||
let mut out = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let user_id: i64 = row.try_get("user_id")?;
|
||||
let podcast_id = podcast_id_col(&row, "podcast_id")?;
|
||||
if let Some(share) = load_account_share(&mut conn, user_id, &podcast_id).await? {
|
||||
out.push(share);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Result of [`aggregate_share_states`].
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct Aggregation {
|
||||
/// One state per published podcast, sorted by podcast id.
|
||||
pub(crate) states: Vec<PodcastShareState>,
|
||||
/// Account that provides each published card.
|
||||
pub(crate) representatives: BTreeMap<PodcastId, i64>,
|
||||
/// Shared podcasts held back until their accounts confirm the privacy
|
||||
/// warnings (no account confirmed).
|
||||
pub(crate) pending_confirmation: usize,
|
||||
/// Shared podcasts whose card cannot be built (no origin URL, invalid
|
||||
/// metadata).
|
||||
pub(crate) failed: usize,
|
||||
/// Publishable podcasts left out by [`MAX_SERVER_SHARED_PODCASTS`].
|
||||
pub(crate) over_limit: Vec<PodcastId>,
|
||||
}
|
||||
|
||||
/// Computes what the server node publishes from the accounts' views.
|
||||
///
|
||||
/// * Only accounts for which [`AccountShare::publishable`] holds count.
|
||||
/// * One state per podcast. The representative account is the one whose
|
||||
/// card has no privacy warnings, then the smallest account id; its
|
||||
/// registers and card are passed with `privacy_confirmed = true`
|
||||
/// (no warnings, or the account confirmed exactly these URLs for its
|
||||
/// current share decision), so `sync_shares` never holds anything back
|
||||
/// on its own.
|
||||
/// * At most [`MAX_SERVER_SHARED_PODCASTS`] podcasts, the ones shared the
|
||||
/// longest first (smallest `hlc_ms` of a representative share decision,
|
||||
/// then podcast id), so an existing publication is not displaced by a
|
||||
/// newly shared podcast.
|
||||
pub(crate) fn aggregate_share_states(shares: Vec<AccountShare>) -> Aggregation {
|
||||
let mut aggregation = Aggregation::default();
|
||||
let mut by_podcast: BTreeMap<PodcastId, Vec<AccountShare>> = BTreeMap::new();
|
||||
for share in shares {
|
||||
if !share.is_shared() {
|
||||
continue;
|
||||
}
|
||||
by_podcast
|
||||
.entry(share.subscription.podcast_id.clone())
|
||||
.or_default()
|
||||
.push(share);
|
||||
}
|
||||
let mut chosen: Vec<(i64, AccountShare)> = Vec::new();
|
||||
for (_, accounts) in by_podcast {
|
||||
let shared_since = accounts
|
||||
.iter()
|
||||
.filter(|share| share.publishable())
|
||||
.filter_map(|share| share.share.as_ref().map(|record| record.hlc_ms))
|
||||
.min();
|
||||
let representative = accounts
|
||||
.iter()
|
||||
.filter(|share| share.publishable())
|
||||
.min_by_key(|share| (!share.warnings().is_empty(), share.user_id));
|
||||
match representative {
|
||||
Some(share) => chosen.push((shared_since.unwrap_or(i64::MAX), share.clone())),
|
||||
None if accounts.iter().any(AccountShare::pending_confirmation) => {
|
||||
aggregation.pending_confirmation += 1;
|
||||
}
|
||||
None => aggregation.failed += 1,
|
||||
}
|
||||
}
|
||||
chosen.sort_by(|(left_since, left), (right_since, right)| {
|
||||
left_since.cmp(right_since).then_with(|| {
|
||||
left.subscription
|
||||
.podcast_id
|
||||
.cmp(&right.subscription.podcast_id)
|
||||
})
|
||||
});
|
||||
for (index, (_, share)) in chosen.into_iter().enumerate() {
|
||||
let podcast_id = share.subscription.podcast_id.clone();
|
||||
if index >= MAX_SERVER_SHARED_PODCASTS {
|
||||
aggregation.over_limit.push(podcast_id);
|
||||
continue;
|
||||
}
|
||||
let Ok(card) = share.card else {
|
||||
continue;
|
||||
};
|
||||
aggregation
|
||||
.representatives
|
||||
.insert(podcast_id.clone(), share.user_id);
|
||||
aggregation.states.push(PodcastShareState {
|
||||
subscription: share.subscription,
|
||||
share: share.share,
|
||||
card: Some(card),
|
||||
privacy_confirmed: true,
|
||||
});
|
||||
}
|
||||
aggregation.states.sort_by(|left, right| {
|
||||
left.subscription
|
||||
.podcast_id
|
||||
.cmp(&right.subscription.podcast_id)
|
||||
});
|
||||
aggregation.over_limit.sort();
|
||||
aggregation
|
||||
}
|
||||
|
||||
/// Outcome of one reconciliation round.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ReconcileReport {
|
||||
/// What was computed.
|
||||
pub(crate) aggregation: Aggregation,
|
||||
/// What the overlay did.
|
||||
pub(crate) stats: PodcastDhtSyncStats,
|
||||
}
|
||||
|
||||
/// One round: load the shared podcasts of every account (or of `users`),
|
||||
/// aggregate them and let the overlay publish and revoke.
|
||||
pub(crate) async fn reconcile_once(
|
||||
pool: &PgPool,
|
||||
dht: &PodcastDht,
|
||||
users: Option<&[i64]>,
|
||||
) -> Result<ReconcileReport> {
|
||||
let shares = load_shared(pool, users).await?;
|
||||
let aggregation = aggregate_share_states(shares);
|
||||
let stats = dht
|
||||
.sync_shares(aggregation.states.clone())
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("podcast publication failed: {error}"))?;
|
||||
Ok(ReconcileReport { aggregation, stats })
|
||||
}
|
||||
|
||||
/// Runs the reconciler until the task is aborted (federation stop).
|
||||
pub(crate) async fn run(pool: PgPool, dht: Arc<PodcastDht>) {
|
||||
let mut interval = tokio::time::interval(RECONCILE_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {}
|
||||
_ = notifier().notified() => tokio::time::sleep(RECONCILE_DEBOUNCE).await,
|
||||
}
|
||||
// Only counts are logged: never feed URLs, titles or accounts.
|
||||
match reconcile_once(&pool, &dht, None).await {
|
||||
Ok(report) => {
|
||||
let stats = &report.stats;
|
||||
let aggregation = &report.aggregation;
|
||||
if stats.published > 0
|
||||
|| stats.unpublished > 0
|
||||
|| !stats.failed.is_empty()
|
||||
|| !stats.held_back.is_empty()
|
||||
|| !aggregation.over_limit.is_empty()
|
||||
{
|
||||
tracing::info!(
|
||||
shared = aggregation.states.len(),
|
||||
published = stats.published,
|
||||
unchanged = stats.unchanged,
|
||||
unpublished = stats.unpublished,
|
||||
failed = stats.failed.len(),
|
||||
held_back = stats.held_back.len(),
|
||||
pending_confirmation = aggregation.pending_confirmation,
|
||||
over_limit = aggregation.over_limit.len(),
|
||||
"podcast publications reconciled"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) => tracing::warn!("podcast publication round skipped: {error:#}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Confirmations (local, never synced)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Records that the account confirmed publishing `card`'s URLs for the
|
||||
/// share decision `share_op_id`.
|
||||
pub(crate) async fn store_confirmation(
|
||||
conn: &mut PgConnection,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
confirmation: &ShareConfirmation,
|
||||
now_ms: i64,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO furumusic__podcast_share_confirmation
|
||||
(user_id, podcast_id, share_op_id, feed_url, id_feed_url, confirmed_at_ms)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
|
||||
share_op_id = EXCLUDED.share_op_id, feed_url = EXCLUDED.feed_url,
|
||||
id_feed_url = EXCLUDED.id_feed_url, confirmed_at_ms = EXCLUDED.confirmed_at_ms",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(podcast_id.as_str())
|
||||
.bind(&confirmation.share_op_id)
|
||||
.bind(confirmation.feed_url.as_str())
|
||||
.bind(confirmation.id_feed_url.as_ref().map(FeedUrl::as_str))
|
||||
.bind(now_ms)
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forgets the account's confirmation (sharing turned off, unsubscribed).
|
||||
pub(crate) async fn clear_confirmation(
|
||||
conn: &mut PgConnection,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
"DELETE FROM furumusic__podcast_share_confirmation
|
||||
WHERE user_id = $1 AND podcast_id = $2",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(podcast_id.as_str())
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn url(path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap()
|
||||
}
|
||||
|
||||
fn subscription(path: &str, generation: u64) -> SubscriptionRecord {
|
||||
let feed_url = url(path);
|
||||
SubscriptionRecord {
|
||||
podcast_id: PodcastId::from_feed_url(&feed_url),
|
||||
subscribed: true,
|
||||
podcast: PodcastMeta {
|
||||
feed_url: feed_url.clone(),
|
||||
title: format!("Show {path}"),
|
||||
author: Some("Host".into()),
|
||||
description: None,
|
||||
link: None,
|
||||
image_url: None,
|
||||
},
|
||||
feed_aliases: vec![feed_url],
|
||||
generation,
|
||||
hlc_ms: 1_000,
|
||||
op_id: format!("dev:{generation}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn account(user_id: i64, path: &str, shared: bool, share_hlc: i64) -> AccountShare {
|
||||
let subscription = subscription(path, 1);
|
||||
let share = ShareRecord {
|
||||
podcast_id: subscription.podcast_id.clone(),
|
||||
shared,
|
||||
generation: 2,
|
||||
hlc_ms: share_hlc,
|
||||
op_id: format!("u{user_id}:share"),
|
||||
};
|
||||
let card = build_card(&subscription, None, Some(3), Some(42));
|
||||
AccountShare {
|
||||
user_id,
|
||||
subscription,
|
||||
share: Some(share),
|
||||
card,
|
||||
confirmation: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn confirm(share: &mut AccountShare) {
|
||||
let card = share.card.clone().unwrap();
|
||||
share.confirmation = Some(ShareConfirmation {
|
||||
share_op_id: share.share.as_ref().unwrap().op_id.clone(),
|
||||
feed_url: card.feed_url,
|
||||
id_feed_url: card.id_feed_url,
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_card_per_podcast_from_the_smallest_sharing_account() {
|
||||
let aggregation = aggregate_share_states(vec![
|
||||
account(7, "a.xml", true, 10),
|
||||
account(3, "a.xml", true, 20),
|
||||
account(1, "a.xml", false, 30),
|
||||
account(5, "b.xml", false, 10),
|
||||
]);
|
||||
assert_eq!(aggregation.states.len(), 1);
|
||||
let state = &aggregation.states[0];
|
||||
assert!(state.is_shared());
|
||||
assert!(state.privacy_confirmed);
|
||||
assert_eq!(
|
||||
aggregation.representatives[&state.subscription.podcast_id],
|
||||
3
|
||||
);
|
||||
let card = state.card.as_ref().unwrap();
|
||||
assert_eq!(card.episode_count, Some(3));
|
||||
assert_eq!(card.latest_episode_at_ms, Some(42));
|
||||
assert!(card.id_feed_url.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_share_older_than_the_subscription_does_not_count() {
|
||||
let mut stale = account(1, "a.xml", true, 10);
|
||||
// Subscribed again after the share decision: private again.
|
||||
stale.subscription.generation = 3;
|
||||
let aggregation = aggregate_share_states(vec![stale]);
|
||||
assert!(aggregation.states.is_empty());
|
||||
assert_eq!(aggregation.failed, 0);
|
||||
assert_eq!(aggregation.pending_confirmation, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn urls_that_look_private_need_the_accounts_own_confirmation() {
|
||||
let secret = "feed.xml?token=8f3a9c2e7d1b4a6f";
|
||||
let unconfirmed = account(1, secret, true, 10);
|
||||
assert!(!unconfirmed.warnings().is_empty());
|
||||
assert!(unconfirmed.pending_confirmation());
|
||||
let aggregation = aggregate_share_states(vec![unconfirmed.clone()]);
|
||||
assert!(aggregation.states.is_empty());
|
||||
assert_eq!(aggregation.pending_confirmation, 1);
|
||||
|
||||
let mut confirmed = account(2, secret, true, 20);
|
||||
confirm(&mut confirmed);
|
||||
assert!(confirmed.publishable());
|
||||
let aggregation = aggregate_share_states(vec![unconfirmed.clone(), confirmed.clone()]);
|
||||
assert_eq!(aggregation.states.len(), 1);
|
||||
assert_eq!(
|
||||
aggregation
|
||||
.representatives
|
||||
.values()
|
||||
.copied()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![2]
|
||||
);
|
||||
|
||||
// A new share decision (another op) needs a new confirmation.
|
||||
let mut redecided = confirmed.clone();
|
||||
redecided.share.as_mut().unwrap().op_id = "u2:share-again".into();
|
||||
assert!(redecided.pending_confirmation());
|
||||
// So does a feed that moved to other URLs.
|
||||
let mut moved = confirmed;
|
||||
moved.card.as_mut().unwrap().feed_url = url("other.xml?token=8f3a9c2e7d1b4a6f");
|
||||
assert!(!moved.privacy_confirmed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_account_without_warnings_is_preferred_as_representative() {
|
||||
let mut moved_secret = account(1, "a.xml", true, 10);
|
||||
// Account 1 fetches the feed from a URL that looks private.
|
||||
moved_secret.card = build_card(
|
||||
&moved_secret.subscription,
|
||||
Some(&PodcastMeta {
|
||||
feed_url: url("a.xml?key=8f3a9c2e7d1b4a6f"),
|
||||
..moved_secret.subscription.podcast.clone()
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
confirm(&mut moved_secret);
|
||||
let plain = account(9, "a.xml", true, 20);
|
||||
let aggregation = aggregate_share_states(vec![moved_secret, plain]);
|
||||
assert_eq!(
|
||||
aggregation
|
||||
.representatives
|
||||
.values()
|
||||
.copied()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![9]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_server_cap_keeps_the_longest_shared_podcasts() {
|
||||
let shares = (0..MAX_SERVER_SHARED_PODCASTS + 3)
|
||||
.map(|index| account(1, &format!("{index}.xml"), true, 1_000 - index as i64))
|
||||
.collect();
|
||||
let aggregation = aggregate_share_states(shares);
|
||||
assert_eq!(aggregation.states.len(), MAX_SERVER_SHARED_PODCASTS);
|
||||
assert_eq!(aggregation.over_limit.len(), 3);
|
||||
// The three most recently shared (largest hlc) are left out.
|
||||
let left_out: Vec<PodcastId> = (0..3)
|
||||
.map(|index| PodcastId::from_feed_url(&url(&format!("{index}.xml"))))
|
||||
.collect();
|
||||
for id in left_out {
|
||||
assert!(aggregation.over_limit.contains(&id));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_card_never_carries_personal_state() {
|
||||
let share = account(1, "a.xml", true, 10);
|
||||
let card = share.card.unwrap();
|
||||
let json = serde_json::to_string(&card).unwrap();
|
||||
for personal in [
|
||||
"generation",
|
||||
"op_id",
|
||||
"hlc_ms",
|
||||
"u1:share",
|
||||
"dev:1",
|
||||
"device",
|
||||
] {
|
||||
assert!(!json.contains(personal), "{personal} in {json}");
|
||||
}
|
||||
}
|
||||
}
|
||||
+8
-2
@@ -20,6 +20,7 @@
|
||||
//! | `furumusic__podcast_progress` | `episode_id` | progress register | yes |
|
||||
//! | `furumusic__podcast_listen_session` | `session_id` | grow-only set of sessions | yes |
|
||||
//! | `furumusic__podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, check times, errors, cover cache | **never** |
|
||||
//! | `furumusic__podcast_share_confirmation` | `podcast_id` | privacy confirmation of a shared card (share decision, card URLs) | **never** |
|
||||
//!
|
||||
//! # One apply path
|
||||
//!
|
||||
@@ -54,12 +55,17 @@
|
||||
//! `/api/podcasts` endpoints. Audio is streamed from the publisher, never
|
||||
//! stored.
|
||||
//! * [`api`] — what the endpoints of the podcast UI and the web player do
|
||||
//! (subscribe by URL, lists, explicit progress actions, playback sessions
|
||||
//! and progress reports), independent of HTTP.
|
||||
//! (subscribe by URL, lists, explicit progress actions, sharing, playback
|
||||
//! sessions and progress reports), independent of HTTP.
|
||||
//! * [`discovery`] — public discovery: which podcasts the server node
|
||||
//! publishes to `music_dht::podcast_dht` for the accounts that explicitly
|
||||
//! shared them, the local privacy confirmations and the reconciler that
|
||||
//! calls `PodcastDht::sync_shares`.
|
||||
|
||||
mod actions;
|
||||
pub(crate) mod api;
|
||||
mod apply;
|
||||
pub(crate) mod discovery;
|
||||
mod fetch;
|
||||
pub(crate) mod http;
|
||||
mod limits;
|
||||
|
||||
@@ -292,7 +292,19 @@ impl PodcastStore {
|
||||
}
|
||||
outcome.learned_aliases = learned;
|
||||
}
|
||||
// The card of a shared podcast carries the channel metadata, the
|
||||
// fetch URL and the episode counters of the feed.
|
||||
let shared: Option<bool> = sqlx::query_scalar(
|
||||
"SELECT shared FROM furumusic__podcast_share WHERE user_id = $1 AND podcast_id = $2",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(podcast_id.as_str())
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
if shared == Some(true) {
|
||||
super::discovery::request_reconcile();
|
||||
}
|
||||
outcome.feed_url = Some(fetch_url);
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user