forked from Furumi/furumi-web
Publish podcasts after the first device sync on restart
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
abd100d6e7
commit
0e985b4267
8 files changed
+328
-27
No files matched your search
@@ -800,16 +800,25 @@ pub async fn serve_peers(
|
||||
}
|
||||
}
|
||||
|
||||
/// Polls every trusted device of every account every
|
||||
/// [`DEVICE_SYNC_INTERVAL`] (one exchange per device at a time).
|
||||
///
|
||||
/// `first_round` turns `true` once the first complete listing of the
|
||||
/// devices has been polled: every exchange in flight then has answered,
|
||||
/// failed or timed out (30 s). The podcast publication reconciler waits for
|
||||
/// it (see [`crate::podcasts::discovery::run`]).
|
||||
pub async fn sync_loop(
|
||||
pool: sqlx::PgPool,
|
||||
service: Arc<MusicDhtService>,
|
||||
hub: Arc<PlayerDeviceHub>,
|
||||
transport_stats: Arc<TransportStats>,
|
||||
first_round: tokio::sync::watch::Sender<bool>,
|
||||
) {
|
||||
let mut interval = tokio::time::interval(DEVICE_SYNC_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut polls = tokio::task::JoinSet::new();
|
||||
let mut active = std::collections::HashMap::new();
|
||||
let mut first_round = FirstSyncRound::new(first_round);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
@@ -819,10 +828,15 @@ pub async fn sync_loop(
|
||||
Ok(users) => users,
|
||||
Err(error) => { tracing::warn!("device poll listing failed: {error:#}"); continue; }
|
||||
};
|
||||
let mut listed_all = true;
|
||||
for user_id in users {
|
||||
let devices = match active_remote_devices(&pool, user_id).await {
|
||||
Ok(devices) => devices,
|
||||
Err(error) => { let _ = set_last_error(&pool, user_id, Some(&format!("{error:#}"))).await; continue; }
|
||||
Err(error) => {
|
||||
listed_all = false;
|
||||
let _ = set_last_error(&pool, user_id, Some(&format!("{error:#}"))).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for device in devices {
|
||||
let key = (user_id, device.device_id.clone());
|
||||
@@ -838,12 +852,17 @@ pub async fn sync_loop(
|
||||
active.insert(handle.id(), key);
|
||||
}
|
||||
}
|
||||
if listed_all {
|
||||
// Includes exchanges still running from an earlier tick.
|
||||
first_round.started(active.keys().copied());
|
||||
}
|
||||
}
|
||||
Some(completed) = polls.join_next_with_id(), if !polls.is_empty() => {
|
||||
let (task, result) = match completed {
|
||||
Ok((task, result)) => (task, result),
|
||||
Err(error) => (error.id(), Err(anyhow::Error::from(error))),
|
||||
};
|
||||
first_round.finished(&task);
|
||||
if let Some((user_id, device_id)) = active.remove(&task)
|
||||
&& let Err(error) = result {
|
||||
tracing::debug!(device = %device_id, "web fed device sync failed: {error:#}");
|
||||
@@ -854,6 +873,50 @@ pub async fn sync_loop(
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks the first device-sync round of a federation run for
|
||||
/// [`sync_loop`]: the exchanges started by the first complete listing of the
|
||||
/// trusted devices (and those still running then). The watch turns `true`
|
||||
/// when the last of them ends, or at once when there are none, and stays
|
||||
/// `true`.
|
||||
pub(crate) struct FirstSyncRound<K> {
|
||||
pending: Option<std::collections::HashSet<K>>,
|
||||
done: tokio::sync::watch::Sender<bool>,
|
||||
}
|
||||
|
||||
impl<K: Eq + std::hash::Hash> FirstSyncRound<K> {
|
||||
pub(crate) fn new(done: tokio::sync::watch::Sender<bool>) -> Self {
|
||||
Self {
|
||||
pending: None,
|
||||
done,
|
||||
}
|
||||
}
|
||||
|
||||
/// The first complete listing started (or found running) `polls`;
|
||||
/// later listings are ignored.
|
||||
pub(crate) fn started(&mut self, polls: impl IntoIterator<Item = K>) {
|
||||
if self.pending.is_some() || *self.done.borrow() {
|
||||
return;
|
||||
}
|
||||
let pending: std::collections::HashSet<K> = polls.into_iter().collect();
|
||||
if pending.is_empty() {
|
||||
self.done.send_replace(true);
|
||||
} else {
|
||||
self.pending = Some(pending);
|
||||
}
|
||||
}
|
||||
|
||||
/// An exchange ended (answered, failed or timed out).
|
||||
pub(crate) fn finished(&mut self, poll: &K) {
|
||||
if let Some(pending) = &mut self.pending
|
||||
&& pending.remove(poll)
|
||||
&& pending.is_empty()
|
||||
{
|
||||
self.pending = None;
|
||||
self.done.send_replace(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn sync_once(
|
||||
pool: &sqlx::PgPool,
|
||||
service: Arc<MusicDhtService>,
|
||||
@@ -5081,6 +5144,8 @@ fn short_id(value: &str) -> String {
|
||||
value.chars().take(10).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod first_round_tests;
|
||||
#[cfg(test)]
|
||||
mod gating_tests;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
//! The first device-sync round that the podcast publication reconciler
|
||||
//! waits for ([`FirstSyncRound`]).
|
||||
|
||||
use super::FirstSyncRound;
|
||||
|
||||
#[test]
|
||||
fn the_first_round_ends_when_its_last_exchange_ends() {
|
||||
let (done, watch) = tokio::sync::watch::channel(false);
|
||||
let mut round = FirstSyncRound::new(done);
|
||||
// An exchange that ends before the first complete listing is not
|
||||
// waited for.
|
||||
round.finished(&7);
|
||||
assert!(!*watch.borrow());
|
||||
round.started([1, 2]);
|
||||
// A later listing (with exchanges it started) changes nothing.
|
||||
round.started([3]);
|
||||
round.finished(&1);
|
||||
round.finished(&3);
|
||||
assert!(!*watch.borrow());
|
||||
round.finished(&2);
|
||||
assert!(*watch.borrow());
|
||||
// Stays done.
|
||||
round.started([4]);
|
||||
round.finished(&4);
|
||||
assert!(*watch.borrow());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_trusted_devices_the_first_round_ends_at_once() {
|
||||
let (done, watch) = tokio::sync::watch::channel(false);
|
||||
let mut round = FirstSyncRound::<u32>::new(done);
|
||||
round.started([]);
|
||||
assert!(*watch.borrow());
|
||||
}
|
||||
@@ -551,9 +551,12 @@ impl Federation {
|
||||
));
|
||||
|
||||
// Public podcast discovery: the server node publishes the podcasts
|
||||
// accounts explicitly shared (see `crate::podcasts::discovery`).
|
||||
// accounts explicitly shared (see `crate::podcasts::discovery`),
|
||||
// after the first device-sync round of this run.
|
||||
let (first_device_sync_done, first_device_sync) = tokio::sync::watch::channel(false);
|
||||
let podcast_discovery =
|
||||
podcast_discovery::start(&service, &self.data_dir, pool.clone()).await;
|
||||
podcast_discovery::start(&service, &self.data_dir, pool.clone(), first_device_sync)
|
||||
.await;
|
||||
let podcast_dht = podcast_discovery
|
||||
.as_ref()
|
||||
.map(|discovery| Arc::clone(&discovery.dht));
|
||||
@@ -612,6 +615,7 @@ impl Federation {
|
||||
Arc::clone(&service),
|
||||
device_hub,
|
||||
Arc::clone(&self.transport_stats),
|
||||
first_device_sync_done,
|
||||
));
|
||||
let capabilities_acceptor = service
|
||||
.stream_acceptor(CAPABILITIES_ALPN)
|
||||
|
||||
@@ -18,6 +18,7 @@ use std::sync::Arc;
|
||||
use music_dht::MusicDhtService;
|
||||
use music_dht::podcast_dht::{PODCAST_DHT_ALPN, PodcastDht};
|
||||
use sqlx::PgPool;
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
/// Cache file of the overlay in the federation data directory.
|
||||
@@ -33,16 +34,24 @@ pub(crate) struct PodcastDiscovery {
|
||||
/// Opens the overlay on `service` (whose config registered
|
||||
/// [`PODCAST_DHT_ALPN`]) and starts its tasks. Returns `None`, with a
|
||||
/// warning, when the cache cannot be opened or the acceptor is unavailable.
|
||||
///
|
||||
/// `first_device_sync` turns `true` after the first device-sync round of
|
||||
/// this federation run; the publication reconciler waits for it.
|
||||
pub(crate) async fn start(
|
||||
service: &Arc<MusicDhtService>,
|
||||
data_dir: &Path,
|
||||
pool: PgPool,
|
||||
first_device_sync: watch::Receiver<bool>,
|
||||
) -> Option<PodcastDiscovery> {
|
||||
let (dht, serve_task) = open(service, data_dir).await?;
|
||||
let tasks = vec![
|
||||
serve_task,
|
||||
tokio::spawn(Arc::clone(&dht).maintenance()),
|
||||
tokio::spawn(crate::podcasts::discovery::run(pool, Arc::clone(&dht))),
|
||||
tokio::spawn(crate::podcasts::discovery::run(
|
||||
pool,
|
||||
Arc::clone(&dht),
|
||||
first_device_sync,
|
||||
)),
|
||||
tokio::spawn(super::capabilities::probe_loop(
|
||||
Arc::clone(service),
|
||||
Arc::clone(&dht),
|
||||
|
||||
@@ -51,7 +51,17 @@
|
||||
//! [`run`] recomputes the set and calls [`PodcastDht::sync_shares`] after
|
||||
//! every change that can affect it ([`request_reconcile`]: share toggles,
|
||||
//! subscribe and unsubscribe, received sync operations and snapshots, feed
|
||||
//! refreshes of shared podcasts) and every [`RECONCILE_INTERVAL`]. A
|
||||
//! refreshes of shared podcasts) and every [`RECONCILE_INTERVAL`].
|
||||
//!
|
||||
//! The first round of a federation run waits for the first device-sync
|
||||
//! round (every trusted device polled answered, failed or timed out; at
|
||||
//! most [`FIRST_DEVICE_SYNC_WAIT`]). The overlay withholds the cards it
|
||||
//! published before the restart until the first `sync_shares`, which
|
||||
//! re-sends the cards still shared. Without the wait that first call would
|
||||
//! use the share decisions stored when the server stopped, so a podcast
|
||||
//! whose sharing was turned off on a TUI meanwhile would be announced
|
||||
//! again until the share-off arrived. A device that cannot be reached at
|
||||
//! start still delivers its decision only when it syncs. A
|
||||
//! failing database connection or query skips the round instead of
|
||||
//! unpublishing everything (the overlay's own maintenance republishes the
|
||||
//! cards in the meantime). Stored data of one `(account, podcast)` that
|
||||
@@ -70,7 +80,7 @@ use music_dht::podcast_dht::{
|
||||
PodcastDht, PodcastDhtSyncStats, PodcastShareState, PublishedPodcast,
|
||||
};
|
||||
use sqlx::{PgConnection, PgPool, Row};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::{Notify, watch};
|
||||
|
||||
use super::{apply, feed_url_col, podcast_id_col};
|
||||
|
||||
@@ -81,6 +91,10 @@ pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
/// batch, a refresh run) becomes one round.
|
||||
pub(crate) const RECONCILE_DEBOUNCE: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Longest wait of the first round for the first device-sync round of the
|
||||
/// federation run (one exchange times out after 30 s).
|
||||
pub(crate) const FIRST_DEVICE_SYNC_WAIT: Duration = Duration::from_secs(45);
|
||||
|
||||
/// Podcasts the server node publishes at most.
|
||||
///
|
||||
/// A replica keeps at most `MAX_PODCAST_RECORDS_PER_PUBLISHER` (256) records
|
||||
@@ -804,7 +818,38 @@ fn unix_now_ms() -> u64 {
|
||||
}
|
||||
|
||||
/// Runs the reconciler until the task is aborted (federation stop).
|
||||
pub(crate) async fn run(pool: PgPool, dht: Arc<PodcastDht>) {
|
||||
///
|
||||
/// The first round waits for `first_device_sync`, the first device-sync
|
||||
/// round of this federation run (`federation::devices::sync_loop`), but at
|
||||
/// most [`FIRST_DEVICE_SYNC_WAIT`]: see the module documentation.
|
||||
pub(crate) async fn run(
|
||||
pool: PgPool,
|
||||
dht: Arc<PodcastDht>,
|
||||
first_device_sync: watch::Receiver<bool>,
|
||||
) {
|
||||
run_rounds(pool, dht, first_device_sync, FIRST_DEVICE_SYNC_WAIT, None).await;
|
||||
}
|
||||
|
||||
/// [`run`] with the longest first wait and the accounts (all when `None`)
|
||||
/// as parameters.
|
||||
pub(crate) async fn run_rounds(
|
||||
pool: PgPool,
|
||||
dht: Arc<PodcastDht>,
|
||||
mut first_device_sync: watch::Receiver<bool>,
|
||||
first_wait: Duration,
|
||||
users: Option<Vec<i64>>,
|
||||
) {
|
||||
// A closed channel (the device-sync loop ended) does not wait either.
|
||||
if tokio::time::timeout(first_wait, first_device_sync.wait_for(|done| *done))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
seconds = first_wait.as_secs(),
|
||||
"podcast publications: the first device-sync round did not end in time; \
|
||||
publishing the stored share decisions"
|
||||
);
|
||||
}
|
||||
let mut interval = tokio::time::interval(RECONCILE_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
@@ -813,7 +858,7 @@ pub(crate) async fn run(pool: PgPool, dht: Arc<PodcastDht>) {
|
||||
_ = notifier().notified() => tokio::time::sleep(RECONCILE_DEBOUNCE).await,
|
||||
}
|
||||
// Only counts are logged: never feed URLs, titles or accounts.
|
||||
match reconcile_once(&pool, &dht, None).await {
|
||||
match reconcile_once(&pool, &dht, users.as_deref()).await {
|
||||
Ok(report) => {
|
||||
let stats = &report.stats;
|
||||
let aggregation = &report.aggregation;
|
||||
|
||||
+152
-18
@@ -1,6 +1,7 @@
|
||||
//! Public podcast discovery against PostgreSQL: the share toggle and its
|
||||
//! privacy confirmation, the aggregation of several accounts and the
|
||||
//! publication from the web's federation node to another loopback node.
|
||||
//! privacy confirmation, the aggregation of several accounts, the
|
||||
//! publication from the web's federation node to another loopback node and
|
||||
//! the first publication after a restart.
|
||||
//!
|
||||
//! Ignored by default, like the other store tests: set
|
||||
//! `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a disposable database. The
|
||||
@@ -434,22 +435,7 @@ async fn loopback_scenario() {
|
||||
|
||||
// The stranger reaches the web node through the music-DHT transport
|
||||
// only: no device-sync pairing, another identity.
|
||||
stranger
|
||||
.service
|
||||
.connect(web.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for("music DHT contacts", async || {
|
||||
(!web.service.known_peers().is_empty() && !stranger.service.known_peers().is_empty())
|
||||
.then_some(())
|
||||
})
|
||||
.await;
|
||||
web.dht.bootstrap().await.unwrap();
|
||||
stranger.dht.bootstrap().await.unwrap();
|
||||
wait_for("overlay contacts", async || {
|
||||
(web.dht.known_peers() >= 1 && stranger.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
meet(&web, &stranger).await;
|
||||
|
||||
// Two accounts share one podcast; one of them also has a private one.
|
||||
let title = format!("Loopback Radio {tag}");
|
||||
@@ -583,6 +569,154 @@ async fn loopback_scenario() {
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
}
|
||||
|
||||
/// Connects `stranger` to `web` and fills both overlay routing tables.
|
||||
async fn meet(web: &Node, stranger: &Node) {
|
||||
stranger
|
||||
.service
|
||||
.connect(web.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for("music DHT contacts", async || {
|
||||
(!web.service.known_peers().is_empty() && !stranger.service.known_peers().is_empty())
|
||||
.then_some(())
|
||||
})
|
||||
.await;
|
||||
web.dht.bootstrap().await.unwrap();
|
||||
stranger.dht.bootstrap().await.unwrap();
|
||||
wait_for("overlay contacts", async || {
|
||||
(web.dht.known_peers() >= 1 && stranger.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
async fn postgres_a_restarted_server_publishes_only_after_the_first_device_sync() {
|
||||
tokio::time::timeout(Duration::from_secs(240), restart_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Interop scenario 7c with the web server as the device that was stopped:
|
||||
/// a TUI of the account turned sharing off meanwhile, and the server comes
|
||||
/// back with the old "shared" rows.
|
||||
async fn restart_scenario() {
|
||||
let pool = test_pool().await;
|
||||
let store = PodcastStore::new(pool.clone());
|
||||
let users = accounts::<1>(&pool).await;
|
||||
let [user] = users;
|
||||
let tag = random_hex(6);
|
||||
let directory = std::env::temp_dir().join(format!("furumusic-podcast-restart-{tag}"));
|
||||
let network = NetworkId::from_name(&format!("furumusic-podcast-restart-{tag}"));
|
||||
|
||||
let kept_title = format!("Kept Show {tag}");
|
||||
let revoked_title = format!("Revoked Show {tag}");
|
||||
let kept = subscribe(
|
||||
&store,
|
||||
user,
|
||||
&format!("https://podcasts.example/kept-{tag}.xml"),
|
||||
&kept_title,
|
||||
)
|
||||
.await;
|
||||
let revoked = subscribe(
|
||||
&store,
|
||||
user,
|
||||
&format!("https://podcasts.example/revoked-{tag}.xml"),
|
||||
&revoked_title,
|
||||
)
|
||||
.await;
|
||||
done(share(&store, user, &kept, json!({"shared": true})).await);
|
||||
done(share(&store, user, &revoked, json!({"shared": true})).await);
|
||||
let web = start_node(&directory.join("web"), network).await;
|
||||
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report.stats.published, 2);
|
||||
web.stop().await;
|
||||
|
||||
// The server restarts from the same directory (same key, same overlay
|
||||
// cache) and meets a stranger before its first device-sync round.
|
||||
let web = start_node(&directory.join("web"), network).await;
|
||||
let stranger = start_node(&directory.join("stranger"), network).await;
|
||||
meet(&web, &stranger).await;
|
||||
let (first_sync_done, first_sync) = tokio::sync::watch::channel(false);
|
||||
let reconciler = tokio::spawn(discovery::run_rounds(
|
||||
pool.clone(),
|
||||
Arc::clone(&web.dht),
|
||||
first_sync,
|
||||
Duration::from_secs(600),
|
||||
Some(users.to_vec()),
|
||||
));
|
||||
// Time for a round: nothing is announced from the stored decisions.
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
for podcast_id in [&kept, &revoked] {
|
||||
assert!(
|
||||
stranger
|
||||
.dht
|
||||
.lookup_podcast(podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"a card was announced before the first device-sync round"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
stranger
|
||||
.dht
|
||||
.search(&revoked_title, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// The first device-sync round brings the share-off the TUI made while
|
||||
// the server was stopped.
|
||||
let share_off = share_op("tui_while_stopped", 1, &revoked, false, 100);
|
||||
assert_eq!(
|
||||
store.apply_remote_op(user, &share_off).await.unwrap(),
|
||||
ApplyOutcome::Changed
|
||||
);
|
||||
first_sync_done.send_replace(true);
|
||||
|
||||
let hit = wait_for("the card still shared", async || {
|
||||
stranger.dht.lookup_podcast(&kept).await.unwrap()
|
||||
})
|
||||
.await;
|
||||
assert_eq!(hit.card.title, kept_title);
|
||||
assert_eq!(hit.publishers, vec![web.service.endpoint_id()]);
|
||||
assert!(
|
||||
stranger
|
||||
.dht
|
||||
.lookup_podcast(&revoked)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
stranger
|
||||
.dht
|
||||
.search(&revoked_title, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
let revoked_record = web
|
||||
.dht
|
||||
.published()
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|publication| publication.podcast_id == revoked)
|
||||
.unwrap();
|
||||
assert!(!revoked_record.active, "the revoked card is tombstoned");
|
||||
|
||||
reconciler.abort();
|
||||
web.stop().await;
|
||||
stranger.stop().await;
|
||||
remove_accounts(&pool, &users).await;
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
async fn postgres_an_unreadable_row_does_not_stop_other_revocations() {
|
||||
|
||||
Reference in new issue
Block a user