diff --git a/src/app/event.rs b/src/app/event.rs index a2628a9..a854f92 100644 --- a/src/app/event.rs +++ b/src/app/event.rs @@ -208,6 +208,13 @@ pub enum AppEvent { JamInvite(Result), /// Result of joining a Jam capability. JamJoined(Result), + /// Received podcast operations or snapshot sections changed the podcast + /// store: `progress` lists changed progress registers, `changed` means + /// anything else (subscriptions, feeds, sessions). + PodcastsSynced { + progress: Vec, + changed: bool, + }, /// The podcast store opened (with the feed client), or why it did not. PodcastStoreOpened(Result), /// Subscribed podcasts reloaded from the podcast store. diff --git a/src/app/mod.rs b/src/app/mod.rs index e6aad6b..6b10659 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -3442,6 +3442,7 @@ fn handle_app_event(state: &mut AppState, runtime: &mut Runtime, event: AppEvent match event { // Folded in by podcast_runtime::handle_event above. AppEvent::PodcastStoreOpened(_) + | AppEvent::PodcastsSynced { .. } | AppEvent::PodcastsLoaded(_) | AppEvent::PodcastSeasonsLoaded { .. } | AppEvent::PodcastEpisodesLoaded { .. } diff --git a/src/app/podcast_runtime.rs b/src/app/podcast_runtime.rs index 0b91216..df69eec 100644 --- a/src/app/podcast_runtime.rs +++ b/src/app/podcast_runtime.rs @@ -261,6 +261,16 @@ pub fn handle_event( }) .inspect_err(|err| tracing::warn!("{err:#}")) .ok(); + // Device sync applies received podcast operations and + // snapshot sections to the store from now on (and the ones + // that arrived before it opened). + let devices = Arc::clone(&runtime.devices); + let store = Arc::clone(&services.store); + tokio::task::spawn_blocking(move || { + if let Err(err) = devices.attach_podcast_store(store) { + tracing::warn!("podcast sync: {err:#}"); + } + }); runtime.podcasts = Some(PodcastRuntime { services, refresh_slots: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_REFRESHES)), @@ -302,6 +312,14 @@ pub fn handle_event( .into_iter() .collect(), AppEvent::PodcastProgress(event) => progress_event(state, runtime, event), + AppEvent::PodcastsSynced { progress, changed } => { + if changed { + // A subscription that came from another device has no + // episodes here until its feed is fetched: check now. + let _ = runtime.event_tx.send(AppEvent::PodcastRefreshTick); + } + podcasts::synced(state, progress, changed) + } AppEvent::PodcastEpisodeStarted { generation, episode, diff --git a/src/app/update/podcasts.rs b/src/app/update/podcasts.rs index 5498a25..5584889 100644 --- a/src/app/update/podcasts.rs +++ b/src/app/update/podcasts.rs @@ -270,7 +270,8 @@ pub fn restart_current(state: &mut AppState) -> Option { } /// The progress writer saved (or found) a progress register: show it in -/// the episode list without reloading it. +/// the episode list without reloading it, and carry its generation in the +/// queue items of that episode (a handoff passes it on). pub fn progress_saved(state: &mut AppState, record: ProgressRecord) -> Vec { if let Some(list) = state.podcasts.episodes.as_mut() { for item in &mut list.items { @@ -279,9 +280,46 @@ pub fn progress_saved(state: &mut AppState, record: ProgressRecord) -> Vec, changed: bool) -> Vec { + let mut effects = Vec::new(); + for record in progress { + effects.extend(progress_saved(state, record)); + } + if state.podcasts.store != PodcastStoreStatus::Ready { + return effects; + } + // Counts in the list (unplayed, in progress) follow both. + effects.push(Effect::PodcastsReload); + if changed { + if let Some(opened) = &state.podcasts.opened + && opened.season_cursor.is_some() + { + effects.push(Effect::PodcastLoadSeasons(opened.podcast_id.clone())); + } + effects.extend(reload_episodes(state)); + } + effects +} + fn open_podcast(state: &mut AppState, index: usize) -> Option { let row = state.podcasts.podcasts().get(index)?; let podcast_id = row.id().clone(); diff --git a/src/app/update_tests.rs b/src/app/update_tests.rs index 4bc18cc..6da4886 100644 --- a/src/app/update_tests.rs +++ b/src/app/update_tests.rs @@ -1843,6 +1843,7 @@ mod podcast_queue { use crate::app::podcast_fixtures::{self as fx, episode_track}; use crate::app::state::{PODCAST_ACTION_ROWS, Popup, RepeatMode, track_content_id}; use crate::app::update::PODCAST_NOT_MUSIC; + use music_dht::podcast::ProgressStatus; fn keys(queue: &[TrackItem]) -> Vec { queue.iter().map(track_key).collect() @@ -2147,4 +2148,49 @@ mod podcast_queue { assert_eq!(items[1].resume_position_ms(), 90_000); assert_eq!(items[1].progress.as_ref().unwrap().generation, 4); } + + #[test] + fn saved_progress_raises_the_generation_a_handoff_carries() { + let mut state = AppState::default(); + state.player.queue = vec![episode_track(1), episode_track(2), test_track(3)]; + state.player.current = Some(episode_track(2)); + let mut record = fx::episode(2, "Episode 2", Some((ProgressStatus::InProgress, 1))) + .progress + .unwrap(); + record.generation = 7; + crate::app::update::podcasts::progress_saved(&mut state, record.clone()); + let generation = |track: &TrackItem| track.podcast.as_ref().unwrap().progress_generation; + assert_eq!(generation(&state.player.queue[0]), None); + assert_eq!(generation(&state.player.queue[1]), Some(7)); + assert_eq!(generation(state.player.current.as_ref().unwrap()), Some(7)); + // Never lowered by an older register. + record.generation = 3; + crate::app::update::podcasts::progress_saved(&mut state, record); + assert_eq!(generation(&state.player.queue[1]), Some(7)); + } + + #[test] + fn podcast_changes_from_other_devices_reload_what_is_shown() { + let mut state = episodes_tab(&[None, None]); + let record = fx::episode(1, "Episode 1", Some((ProgressStatus::InProgress, 5_000))) + .progress + .unwrap(); + // A progress register is shown in place; the list counts reload. + let effects = crate::app::update::podcasts::synced(&mut state, vec![record], false); + assert_eq!(effects, [Effect::PodcastsReload]); + let items = &state.podcasts.episodes.as_ref().unwrap().items; + assert_eq!(items[0].resume_position_ms(), 5_000); + // Anything else reloads the opened podcast too. + let effects = crate::app::update::podcasts::synced(&mut state, Vec::new(), true); + assert_eq!(effects[0], Effect::PodcastsReload); + assert!( + effects + .iter() + .any(|effect| matches!(effect, Effect::PodcastLoadEpisodes { .. })), + "{effects:?}" + ); + // Before the store is ready there is nothing to reload. + let mut opening = AppState::default(); + assert!(crate::app::update::podcasts::synced(&mut opening, Vec::new(), true).is_empty()); + } } diff --git a/src/devices.rs b/src/devices.rs index 0e97320..4b076e3 100644 --- a/src/devices.rs +++ b/src/devices.rs @@ -4,6 +4,14 @@ //! byte-stream ALPN. Local state is stored as an append-only operation log plus //! materialized tables, so offline clients can merge likes, playlists and //! membership changes deterministically. +//! +//! The wire types are the shared `music_dht::device_sync` definitions. This +//! client speaks protocol v4: podcast operations stay in the log, are +//! applied to the podcast store (`furumi_library::podcasts::PodcastStore`) +//! and are relayed; podcast snapshot sections are exported from and merged +//! into that store. Everything sent to a peer is gated by the version that +//! peer reported itself (`op_allowed_for_peer`, `SyncSnapshot::for_peer`, +//! `playback_allowed_for_peer`). use music_dht::playback::{Checkpoint, CommandStamp, Config as PlaybackConfig, Engine}; use std::collections::{BTreeMap, BTreeSet}; @@ -13,7 +21,8 @@ use std::time::Duration; use anyhow::{Context as _, Result}; use furumi_library::podcasts::{ - OpStamp as PodcastOpStamp, PodcastLocalOp, StampSource as PodcastStampSource, + ApplyOutcome as PodcastApplyOutcome, OpStamp as PodcastOpStamp, PodcastLocalOp, PodcastStore, + StampSource as PodcastStampSource, }; use music_dht::device_sync::{ DeviceProfileWire, InviteWire, ListenEvent, ListenTrackMetadata, SnapshotLike, @@ -25,6 +34,7 @@ pub use music_dht::device_sync::{ PlaybackCommand, PlaybackRepeat, PlaybackSnapshot, PlaybackStateWire, PlaybackTrack, SyncOpPayload, SyncedFedTrack, }; +use music_dht::podcast::ProgressRecord; use music_dht::{ByteStream, MusicDhtService, NetworkId, PeerTicket, SecretKey, StreamAcceptor}; use rusqlite::{Connection, OptionalExtension, params}; @@ -34,7 +44,19 @@ use crate::library::models::{ArtistRef, TrackItem}; pub const SYNC_ALPN: &[u8] = b"furumi/sync/2"; const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const PROTOCOL_VERSION: u16 = 3; +/// Device-sync protocol version of this client. +/// +/// Version 4 is announced because podcast operations, podcast snapshot +/// sections and podcast queue items are applied, kept and relayed (see the +/// module documentation). Review that support before following a newer +/// shared protocol version. +pub const PROTOCOL_VERSION: u16 = 4; +const _: () = assert!( + PROTOCOL_VERSION == music_dht::device_sync::DEVICE_SYNC_PROTOCOL_VERSION, + "review device-sync support before announcing a new protocol version" +); +/// Lowest protocol version of a peer that speaks `furumi/sync/2`. +const SYNC_V2_PROTOCOL_VERSION: u16 = 2; const INVITE_TTL_MS: i64 = 10 * 60 * 1000; const PAIRING_WAIT_MS: i64 = 5 * 60 * 1000; const PAIRING_RETRY_DELAY: Duration = Duration::from_secs(1); @@ -100,6 +122,23 @@ pub struct DeviceSync { library: Arc, event_tx: Arc>>>, playback: Arc>, + podcasts: Arc>, +} + +/// The podcast store as device sync sees it. +/// +/// The store opens in the background after device sync started. Podcast +/// operations received before that (or whose application failed) are kept +/// in `sync_podcast_pending` and applied when the store is attached. +#[derive(Default)] +struct PodcastLink { + store: Option>, + /// Progress registers that received operations changed since the last + /// notification. + progress: Vec, + /// Other podcast state (subscriptions, shares, feed URLs, sessions) + /// changed since the last notification. + changed: bool, } /// Converts queue items to and from the shared wire form @@ -403,6 +442,7 @@ impl DeviceSync { library, event_tx: Arc::new(std::sync::Mutex::new(None)), playback: Arc::new(std::sync::Mutex::new(PlaybackShared::default())), + podcasts: Arc::default(), }); sync.ensure_identity()?; sync.repair_like_order_from_sync_state()?; @@ -569,6 +609,182 @@ impl DeviceSync { Ok(()) } + /// Protocol version a device reported itself, for version gating. + /// Unknown devices count as the oldest protocol. + pub fn peer_protocol_version(&self, device_id: &str) -> Result { + let stored = stored_protocol_version(&lock(&self.conn), device_id)?; + Ok(music_dht::device_sync::profile_protocol_version( + None, stored, + )) + } + + /// Connects the podcast store. Podcast operations received from now on + /// are applied to it, and those that arrived while it was unavailable + /// (or whose application failed) are applied first. Blocking. + pub fn attach_podcast_store(&self, store: Arc) -> Result<()> { + let pending = { + // Same lock order as `apply_podcast_op`: link, then log. An + // operation received concurrently is either in this list or + // sees the attached store. + let mut link = lock(&self.podcasts); + link.store = Some(Arc::clone(&store)); + let conn = lock(&self.conn); + // Rows whose operation was coalesced away have nothing to apply. + conn.execute( + "DELETE FROM sync_podcast_pending + WHERE op_id NOT IN (SELECT op_id FROM sync_ops)", + [], + )?; + let mut stmt = conn.prepare( + "SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json + FROM sync_podcast_pending p + JOIN sync_ops o ON o.op_id = p.op_id + ORDER BY o.hlc_ms, o.op_id", + )?; + stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + )) + })? + .collect::>>()? + }; + let mut applied = 0usize; + for (op_id, origin_device_id, seq, hlc_ms, payload_json) in pending { + let op = match serde_json::from_str(&payload_json) { + Ok(payload) => SyncOpWire { + op_id: op_id.clone(), + origin_device_id, + seq, + hlc_ms, + payload, + }, + Err(err) => { + tracing::warn!(%op_id, "dropping an unreadable podcast operation: {err}"); + lock(&self.conn).execute( + "DELETE FROM sync_podcast_pending WHERE op_id = ?1", + [&op_id], + )?; + continue; + } + }; + match self.apply_podcast_op_to(&store, &op) { + Ok(()) => { + applied += 1; + lock(&self.conn).execute( + "DELETE FROM sync_podcast_pending WHERE op_id = ?1", + [&op_id], + )?; + } + Err(err) => { + tracing::warn!(%op_id, "podcast operation still not applied: {err:#}"); + } + } + } + if applied > 0 { + tracing::info!(applied, "applied podcast operations received earlier"); + } + self.flush_podcast_changes(); + Ok(()) + } + + fn podcast_store(&self) -> Option> { + lock(&self.podcasts).store.clone() + } + + /// Merges a received podcast operation into the podcast store, or keeps + /// it for [`DeviceSync::attach_podcast_store`]. A podcast store problem + /// never fails music sync: the operation stays in the log either way. + fn apply_podcast_op(&self, op: &SyncOpWire) -> Result<()> { + let store = { + let link = lock(&self.podcasts); + match &link.store { + Some(store) => Arc::clone(store), + None => { + lock(&self.conn).execute( + "INSERT OR IGNORE INTO sync_podcast_pending (op_id) VALUES (?1)", + [&op.op_id], + )?; + return Ok(()); + } + } + }; + if let Err(err) = self.apply_podcast_op_to(&store, op) { + tracing::warn!(op_id = %op.op_id, "podcast operation not applied yet: {err:#}"); + lock(&self.conn).execute( + "INSERT OR IGNORE INTO sync_podcast_pending (op_id) VALUES (?1)", + [&op.op_id], + )?; + } + Ok(()) + } + + fn apply_podcast_op_to(&self, store: &PodcastStore, op: &SyncOpWire) -> Result<()> { + match store.apply_remote_op(op)? { + PodcastApplyOutcome::Changed => { + let mut link = lock(&self.podcasts); + match op.podcast_progress() { + Some(record) => link.progress.push(record), + None => link.changed = true, + } + } + // Acknowledged anyway so the log keeps moving (FRID contract). + PodcastApplyOutcome::Rejected(reason) => tracing::warn!( + op_id = %op.op_id, + kind = payload_kind(&op.payload), + %reason, + "skipped an invalid podcast operation" + ), + _ => {} + } + Ok(()) + } + + /// Merges the podcast sections of a received snapshot. + fn apply_podcast_snapshot(&self, snapshot: &SyncSnapshot) { + let Some(store) = self.podcast_store() else { + // Snapshots come with every sync; a later one is merged. + return; + }; + match store.apply_snapshot(snapshot) { + Ok(stats) => { + if stats.ignored_over_limits { + tracing::warn!("ignored podcast snapshot sections over the shared limits"); + } + if stats.rejected > 0 { + tracing::warn!( + rejected = stats.rejected, + "skipped invalid podcast snapshot records" + ); + } + if stats.changed > 0 { + lock(&self.podcasts).changed = true; + } + } + Err(err) => tracing::warn!("podcast snapshot not merged: {err:#}"), + } + } + + /// Tells the UI about podcast state that received operations or + /// snapshots changed (one event per batch). + fn flush_podcast_changes(&self) { + let (progress, changed) = { + let mut link = lock(&self.podcasts); + ( + std::mem::take(&mut link.progress), + std::mem::take(&mut link.changed), + ) + }; + if (changed || !progress.is_empty()) + && let Some(tx) = lock(&self.event_tx).as_ref() + { + let _ = tx.send(AppEvent::PodcastsSynced { progress, changed }); + } + } + pub fn status(&self) -> DeviceSyncStatus { match self.status_inner() { Ok(status) => status, @@ -661,9 +877,11 @@ impl DeviceSync { let profile = self.own_profile(&own_ticket)?; let devices = self.device_profiles()?; let vector = self.vector()?; - let ops = self.ops_for_peer(&invite.device_id)?; - let snapshot = self.snapshot()?; - let playback = self.local_playback_snapshot(); + // The inviter has not reported anything yet but its invite. + let peer_version = invite.inviter_protocol_version(); + let ops = self.ops_for_peer(&invite.device_id, peer_version)?; + let snapshot = self.snapshot_for(peer_version)?; + let playback = self.local_playback_snapshot_for(peer_version); let mut stream = service.open_stream(peer, SYNC_ALPN).await?; crate::federation::record_stream_transport( &transport_stats, @@ -723,8 +941,7 @@ impl DeviceSync { } } self.apply_device_profiles(&devices, Some(&invite.device_id))?; - self.apply_snapshot(snapshot)?; - self.apply_ops(ops)?; + self.apply_received(snapshot, ops)?; self.record_local_op(SyncOpPayload::DeviceTrusted { target_device_id: invite.device_id.clone(), })?; @@ -1176,9 +1393,10 @@ impl DeviceSync { let profile = self.own_profile(&own_ticket)?; let devices = self.device_profiles()?; let vector = self.vector()?; - let ops = self.ops_for_peer(&device.device_id)?; - let snapshot = self.snapshot()?; - let playback = self.local_playback_snapshot(); + let peer_version = self.peer_protocol_version(&device.device_id)?; + let ops = self.ops_for_peer(&device.device_id, peer_version)?; + let snapshot = self.snapshot_for(peer_version)?; + let playback = self.local_playback_snapshot_for(peer_version); let mut stream = service.open_stream(peer, SYNC_ALPN).await?; crate::federation::record_stream_transport( &transport_stats, @@ -1226,8 +1444,7 @@ impl DeviceSync { if let Some(playback) = playback { self.apply_playback_snapshot(&device.device_id, playback)?; } - self.apply_snapshot(snapshot)?; - self.apply_ops(ops)?; + self.apply_received(snapshot, ops)?; self.note_peer_vector(&device.device_id, &vector)?; self.mark_seen(&device.device_id, Some(peer.to_string()))?; self.set_last_sync(Some(format!("synced {}", short_id(&device.device_id))))?; @@ -1447,6 +1664,7 @@ impl DeviceSync { }; if inserted > 0 { changed |= self.apply_op(&op)?; + coalesce_progress_ops(&lock(&self.conn), &op)?; } let conn = lock(&self.conn); conn.execute( @@ -1460,6 +1678,7 @@ impl DeviceSync { if changed { self.notify_library_changed(); } + self.flush_podcast_changes(); Ok(()) } @@ -1591,13 +1810,16 @@ impl DeviceSync { SyncOpPayload::ListenRecorded { event } => self .library .apply_listen_event(event, &op.origin_device_id)?, - // Podcast operations (protocol v4) are kept in the log; this - // client does not apply or relay them yet. + // Podcast operations (protocol v4) stay in the log, so they are + // relayed, and are merged into the podcast store. SyncOpPayload::PodcastSubscriptionSet { .. } | SyncOpPayload::PodcastShareSet { .. } | SyncOpPayload::PodcastFeedAliasesAdded { .. } | SyncOpPayload::PodcastProgressSet { .. } - | SyncOpPayload::PodcastListenRecorded { .. } => false, + | SyncOpPayload::PodcastListenRecorded { .. } => { + self.apply_podcast_op(op)?; + false + } }; Ok(changed) } @@ -1983,6 +2205,28 @@ impl DeviceSync { Ok(visible_changed) } + /// Applies what a peer sent: the music snapshot, the operations, then + /// the podcast snapshot sections. Podcast merges are order-independent; + /// operations go first so a changed progress register reaches the UI as + /// such (it is shown in place) instead of as a generic change. + fn apply_received(&self, mut snapshot: SyncSnapshot, ops: Vec) -> Result<()> { + let podcast = SyncSnapshot { + podcast_subscriptions: std::mem::take(&mut snapshot.podcast_subscriptions), + podcast_shares: std::mem::take(&mut snapshot.podcast_shares), + podcast_progress: std::mem::take(&mut snapshot.podcast_progress), + podcast_listen_sessions: std::mem::take(&mut snapshot.podcast_listen_sessions), + ..SyncSnapshot::default() + }; + self.apply_snapshot(snapshot)?; + self.apply_ops(ops)?; + if podcast.has_podcast_state() { + self.apply_podcast_snapshot(&podcast); + self.flush_podcast_changes(); + } + Ok(()) + } + + /// Applies the music sections of a snapshot. fn apply_snapshot(&self, snapshot: SyncSnapshot) -> Result<()> { let mut changed = false; for like in snapshot.likes { @@ -2163,10 +2407,20 @@ impl DeviceSync { Ok(()) } - fn ops_for_peer(&self, peer_device_id: &str) -> Result> { - // Podcast operations (`podcast_*`, protocol v4) stay in the log but - // are not sent yet: this client still announces protocol v3 and its - // wire types cannot carry them. Gated sending comes with sync v4. + /// Operations the peer has not acknowledged and can parse. + /// + /// Kinds the peer's version does not know are excluded in SQL, so a + /// batch of hidden operations cannot stall the per-peer cursor; + /// `op_allowed_for_peer` stays the final check (playback commands that + /// carry podcast items). Hidden operations are never re-sent after the + /// peer upgrades (vectors only grow): snapshots are the recovery path. + fn ops_for_peer(&self, peer_device_id: &str, peer_version: u16) -> Result> { + // Every peer of `furumi/sync/2` keeps listen history, even when it + // never reported a version (an invite without one, a device known + // only from `device_trusted`). Listen events are not part of + // snapshots, so skipping them would lose them for good. + let peer_version = peer_version.max(SYNC_V2_PROTOCOL_VERSION); + let hide_podcasts = !music_dht::device_sync::supports_podcasts(peer_version); let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json @@ -2174,7 +2428,7 @@ impl DeviceSync { LEFT JOIN sync_peer_acks a ON a.peer_device_id = ?1 AND a.origin_device_id = o.origin_device_id WHERE o.seq > COALESCE(a.max_seq, 0) - AND substr(o.kind, 1, 8) != 'podcast_' + AND NOT (?4 AND substr(o.kind, 1, 8) = 'podcast_') AND ( o.kind != 'playback_command' OR ( @@ -2189,7 +2443,8 @@ impl DeviceSync { params![ peer_device_id, now_ms().saturating_sub(PLAYBACK_COMMAND_TTL_MS), - MAX_OPS_PER_BATCH as i64 + MAX_OPS_PER_BATCH as i64, + hide_podcasts, ], |row| { let payload_json: String = row.get(4)?; @@ -2208,9 +2463,30 @@ impl DeviceSync { }) }, )?; - Ok(rows.collect::>>()?) + let mut ops = rows.collect::>>()?; + ops.retain(|op| music_dht::device_sync::op_allowed_for_peer(&op.payload, peer_version)); + Ok(ops) } + /// The materialized snapshot for a peer: podcast sections (bounded by + /// the shared snapshot limits) only for v4 peers. + fn snapshot_for(&self, peer_version: u16) -> Result { + let mut snapshot = self.snapshot()?; + if music_dht::device_sync::supports_podcasts(peer_version) + && let Some(store) = self.podcast_store() + && let Err(err) = store.fill_snapshot( + &mut snapshot, + music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES, + ) + { + // Music sync goes on; podcast state comes with a later sync. + tracing::warn!("podcast snapshot export failed: {err:#}"); + snapshot.strip_podcast_state(); + } + Ok(snapshot.for_peer(peer_version)) + } + + /// Music sections of the materialized snapshot. fn snapshot(&self) -> Result { let like_rows = { let conn = lock(&self.conn); @@ -2593,8 +2869,14 @@ impl DeviceSync { } } - fn local_playback_snapshot(&self) -> Option { - lock(&self.playback).local.clone() + /// The playback snapshot for a peer, or `None` when the queue has + /// podcast items and the peer is below v4 (`playback_allowed_for_peer`). + /// Episodes are never stripped to make the state fit: an old client + /// would forward the shortened queue and lose them for every device. + fn local_playback_snapshot_for(&self, peer_version: u16) -> Option { + lock(&self.playback).local.clone().filter(|snapshot| { + music_dht::device_sync::playback_allowed_for_peer(&snapshot.state, peer_version) + }) } fn apply_playback_snapshot(&self, sender: &str, snapshot: PlaybackSnapshot) -> Result<()> { @@ -2740,6 +3022,7 @@ impl PodcastLogStamps<'_> { i64::from(op.payload.is_tombstone()), ], )?; + coalesce_progress_ops(conn, &op)?; conn.execute( "INSERT INTO sync_vectors (device_id, max_seq) VALUES (?1, ?2) @@ -3082,8 +3365,7 @@ async fn handle_pair_request( if let Some(playback) = playback { sync.apply_playback_snapshot(&profile.device_id, playback)?; } - sync.apply_snapshot(snapshot)?; - sync.apply_ops(ops)?; + sync.apply_received(snapshot, ops)?; sync.note_peer_vector(&profile.device_id, &vector)?; { let conn = lock(&sync.conn); @@ -3095,9 +3377,10 @@ async fn handle_pair_request( sync.set_last_sync(Some(format!("paired {}", short_id(&profile.device_id))))?; let devices = sync.device_profiles()?; let vector = sync.vector()?; - let ops = sync.ops_for_peer(&profile.device_id)?; - let snapshot = sync.snapshot()?; - let playback = sync.local_playback_snapshot(); + let peer_version = sync.peer_protocol_version(&profile.device_id)?; + let ops = sync.ops_for_peer(&profile.device_id, peer_version)?; + let snapshot = sync.snapshot_for(peer_version)?; + let playback = sync.local_playback_snapshot_for(peer_version); write_msg( &mut stream, &WireMessage::PairResponse { @@ -3176,8 +3459,7 @@ async fn handle_hello( if let Some(playback) = playback { sync.apply_playback_snapshot(&profile.device_id, playback)?; } - sync.apply_snapshot(snapshot)?; - sync.apply_ops(ops)?; + sync.apply_received(snapshot, ops)?; sync.note_peer_vector(&profile.device_id, &vector)?; sync.mark_seen(&profile.device_id, Some(stream.peer_id.to_string()))?; sync.set_last_sync(Some(format!("synced {}", short_id(&profile.device_id))))?; @@ -3186,9 +3468,10 @@ async fn handle_hello( let own_profile = sync.own_profile(&own_ticket)?; let devices = sync.device_profiles()?; let vector = sync.vector()?; - let ops = sync.ops_for_peer(&profile.device_id)?; - let snapshot = sync.snapshot()?; - let playback = sync.local_playback_snapshot(); + let peer_version = sync.peer_protocol_version(&profile.device_id)?; + let ops = sync.ops_for_peer(&profile.device_id, peer_version)?; + let snapshot = sync.snapshot_for(peer_version)?; + let playback = sync.local_playback_snapshot_for(peer_version); write_msg( &mut stream, &WireMessage::SyncResponse { @@ -3385,6 +3668,10 @@ CREATE TABLE IF NOT EXISTS sync_playback_applied ( op_id TEXT PRIMARY KEY, applied_at_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS sync_podcast_pending ( + op_id TEXT PRIMARY KEY +); +CREATE INDEX IF NOT EXISTS idx_sync_ops_kind ON sync_ops(kind); "#, )?; ensure_column( @@ -3466,6 +3753,30 @@ fn delete_meta(conn: &Connection, key: &str) -> Result<()> { Ok(()) } +/// Keeps only the newest `podcast_progress_set` of an origin per episode in +/// the log (the periodic checkpoints of a playing episode would grow it +/// without bound). +/// +/// Safe because the operations of one origin are ordered: a later one has +/// a later `hlc_ms` and never a lower progress generation, so it wins every +/// register comparison against the earlier ones. Peers that have not +/// received the removed operations get the later one, and their vector +/// skips the gap as usual. +fn coalesce_progress_ops(conn: &Connection, op: &SyncOpWire) -> Result<()> { + let SyncOpPayload::PodcastProgressSet { episode_id, .. } = &op.payload else { + return Ok(()); + }; + conn.execute( + "DELETE FROM sync_ops + WHERE origin_device_id = ?1 + AND kind = 'podcast_progress_set' + AND seq < ?2 + AND json_extract(payload_json, '$.episode_id') = ?3", + params![op.origin_device_id, op.seq, episode_id.as_str()], + )?; + Ok(()) +} + fn payload_kind(payload: &SyncOpPayload) -> &'static str { match payload { SyncOpPayload::TrackLikeSet { .. } => "track_like_set", @@ -3644,10 +3955,36 @@ fn peer_ack_floor_label(conn: &Connection) -> Result { } async fn write_msg(stream: &mut ByteStream, message: &WireMessage) -> Result<()> { - music_dht::device_sync::write_msg(stream, message).await?; + let mut message = message.clone(); + fit_podcast_snapshot(&mut message); + music_dht::device_sync::write_msg(stream, &message).await?; Ok(()) } +fn message_snapshot(message: &mut WireMessage) -> &mut SyncSnapshot { + match message { + WireMessage::PairRequest { snapshot, .. } + | WireMessage::PairResponse { snapshot, .. } + | WireMessage::Hello { snapshot, .. } + | WireMessage::SyncResponse { snapshot, .. } => snapshot, + } +} + +/// Bounds the podcast snapshot sections so the whole line stays under the +/// protocol limit (`podcast_snapshot_budget`): the operations, music +/// snapshot and playback queue of the same message come first. +fn fit_podcast_snapshot(message: &mut WireMessage) { + if !message_snapshot(message).has_podcast_state() { + return; + } + let podcast = message_snapshot(message).clone(); + message_snapshot(message).strip_podcast_state(); + let other_bytes = serde_json::to_vec(message).map_or(usize::MAX, |bytes| bytes.len()); + let snapshot = message_snapshot(message); + *snapshot = podcast; + snapshot.bound_podcast_state_to(music_dht::device_sync::podcast_snapshot_budget(other_bytes)); +} + async fn finish_response(stream: &mut ByteStream) -> Result<()> { music_dht::device_sync::finish_response(stream, RESPONSE_DRAIN_TIMEOUT).await?; Ok(()) @@ -3697,6 +4034,9 @@ mod interop_tests; #[cfg(test)] mod wire_compat_tests; +#[cfg(test)] +mod podcast_sync_tests; + fn playback_clock() -> u64 { static START: std::sync::OnceLock = std::sync::OnceLock::new(); START diff --git a/src/devices/interop_tests.rs b/src/devices/interop_tests.rs index 95bf5ca..60fc2a2 100644 --- a/src/devices/interop_tests.rs +++ b/src/devices/interop_tests.rs @@ -25,6 +25,7 @@ async fn localhost_web_peer() { library: Arc::new(Library::open(&dir.join("interop-library.sqlite3")).unwrap()), event_tx: Default::default(), playback: Default::default(), + podcasts: Default::default(), }; let id = sync.ensure_identity().unwrap(); let (tx, mut events) = tokio::sync::mpsc::unbounded_channel(); @@ -55,7 +56,7 @@ async fn localhost_web_peer() { vector: BTreeMap::new(), ops: vec![], snapshot: SyncSnapshot::default(), - playback: sync.local_playback_snapshot(), + playback: sync.local_playback_snapshot_for(PROTOCOL_VERSION), }; let mut bytes = serde_json::to_vec(&hello).unwrap(); bytes.push(b'\n'); diff --git a/src/devices/podcast_sync_tests.rs b/src/devices/podcast_sync_tests.rs new file mode 100644 index 0000000..789c6b8 --- /dev/null +++ b/src/devices/podcast_sync_tests.rs @@ -0,0 +1,614 @@ +//! Podcast sync between two (or three) TUI device stores, with the same +//! functions the stream handlers use (`ops_for_peer`, `snapshot_for`, +//! `local_playback_snapshot_for`, `apply_*`). Every message goes through its +//! JSON line, like on the wire. The network round trip itself is covered by +//! `localhost_devices_sync_podcasts` in `tests.rs`. + +use super::*; +use furumi_library::podcasts::{ + CheckpointOutcome, PodcastListFilter, ProgressSession, SubscribeOutcome, +}; +use music_dht::podcast::{PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressStatus}; +use tokio::sync::mpsc::UnboundedReceiver; + +pub(super) struct Device { + pub(super) sync: DeviceSync, + pub(super) store: Arc, + pub(super) events: UnboundedReceiver, + pub(super) id: String, + _library_dir: tempfile::TempDir, +} + +/// A device with an in-memory sync log and its podcast store attached. +pub(super) fn device() -> Device { + device_with_store(Arc::new(PodcastStore::open_in_memory().unwrap()), true) +} + +pub(super) fn device_with_store(store: Arc, attach: bool) -> Device { + let library_dir = tempfile::tempdir().unwrap(); + let conn = Connection::open_in_memory().unwrap(); + init_schema(&conn).unwrap(); + let sync = DeviceSync { + _identity_lock: None, + conn: Arc::new(std::sync::Mutex::new(conn)), + library: Arc::new(Library::open(&library_dir.path().join("library.sqlite3")).unwrap()), + event_tx: Default::default(), + playback: Default::default(), + podcasts: Default::default(), + }; + let (tx, events) = tokio::sync::mpsc::unbounded_channel(); + sync.set_event_tx(tx); + let id = sync.ensure_identity().unwrap().device_id; + if attach { + sync.attach_podcast_store(Arc::clone(&store)).unwrap(); + } + Device { + sync, + store, + events, + id, + _library_dir: library_dir, + } +} + +/// Trusts `b` on `a` and the other way round, in `a`'s group, with the +/// versions their profiles report. +pub(super) fn pair(a: &Device, b: &Device) { + b.sync + .set_group_id(&a.sync.ensure_identity().unwrap().group_id) + .unwrap(); + for (left, right) in [(a, b), (b, a)] { + let profile = right.sync.own_profile("").unwrap(); + left.sync + .apply_device_profile( + &profile, + true, + VersionReport::SelfReported(Some(profile.protocol_version)), + ) + .unwrap(); + } +} + +/// The `hello` that `sync_device` sends from `from` to `to`, as JSON. +pub(super) fn hello_json(from: &Device, to_id: &str) -> String { + let version = from.sync.peer_protocol_version(to_id).unwrap(); + let mut message = WireMessage::Hello { + group_id: from.sync.ensure_identity().unwrap().group_id, + profile: from.sync.own_profile("").unwrap(), + devices: from.sync.device_profiles().unwrap(), + vector: from.sync.vector().unwrap(), + ops: from.sync.ops_for_peer(to_id, version).unwrap(), + snapshot: from.sync.snapshot_for(version).unwrap(), + playback: from.sync.local_playback_snapshot_for(version), + }; + fit_podcast_snapshot(&mut message); + serde_json::to_string(&message).unwrap() +} + +/// What `handle_hello` does with a `hello` line, minus the transport. +pub(super) fn receive(to: &DeviceSync, json: &str) { + let WireMessage::Hello { + profile, + devices, + vector, + ops, + snapshot, + playback, + .. + } = serde_json::from_str(json).unwrap() + else { + panic!("expected a hello"); + }; + to.apply_device_profile( + &profile, + false, + VersionReport::SelfReported(Some(profile.protocol_version)), + ) + .unwrap(); + to.apply_device_profiles(&devices, Some(&profile.device_id)) + .unwrap(); + if let Some(playback) = playback { + to.apply_playback_snapshot(&profile.device_id, playback) + .unwrap(); + } + to.apply_received(snapshot, ops).unwrap(); + to.note_peer_vector(&profile.device_id, &vector).unwrap(); +} + +pub(super) fn deliver(from: &Device, to: &Device) { + receive(&to.sync, &hello_json(from, &to.id)); +} + +/// One sync in each direction. +pub(super) fn sync_round(a: &Device, b: &Device) { + deliver(a, b); + deliver(b, a); +} + +pub(super) fn meta() -> PodcastMeta { + crate::app::podcast_fixtures::row("Show", 0, 0, 0) + .summary + .meta +} + +pub(super) fn episode_ref(n: u32) -> PodcastEpisodeRef { + crate::app::podcast_fixtures::episode(n, &format!("Episode {n}"), None) + .episode + .to_ref(&meta()) +} + +pub(super) fn subscribe(device: &Device) -> SubscribeOutcome { + let store = Arc::clone(&device.store); + device + .sync + .run_podcast_action(|stamps| { + let outcome = store.subscribe(stamps, &meta())?; + let ops = outcome.ops.clone(); + Ok((outcome, ops)) + }) + .unwrap() +} + +pub(super) fn start(device: &Device, episode: &PodcastEpisodeRef, at_ms: i64) -> ProgressSession { + let store = Arc::clone(&device.store); + let session_id = device.sync.new_listen_id(); + device + .sync + .run_podcast_action(|stamps| { + let write = store.start_session(stamps, episode, &session_id, at_ms, None)?; + Ok((write.session.clone(), vec![write.op])) + }) + .unwrap() +} + +pub(super) fn checkpoint( + device: &Device, + session: &ProgressSession, + at_ms: i64, +) -> CheckpointOutcome { + let store = Arc::clone(&device.store); + device + .sync + .run_podcast_action(|stamps| { + let outcome = store.checkpoint(stamps, session, at_ms)?; + let ops = match &outcome { + CheckpointOutcome::Written(write) => vec![write.op.clone()], + _ => Vec::new(), + }; + Ok((outcome, ops)) + }) + .unwrap() +} + +fn restart(device: &Device, episode: &PodcastEpisodeRef) -> ProgressSession { + let store = Arc::clone(&device.store); + let session_id = device.sync.new_listen_id(); + device + .sync + .run_podcast_action(|stamps| { + let write = store.restart(stamps, episode, &session_id)?; + Ok((write.session.clone(), vec![write.op])) + }) + .unwrap() +} + +fn synced_events(device: &mut Device) -> Vec<(Vec, bool)> { + let mut events = Vec::new(); + while let Ok(event) = device.events.try_recv() { + if let AppEvent::PodcastsSynced { progress, changed } = event { + events.push((progress, changed)); + } + } + events +} + +fn subscribed_ids(store: &PodcastStore) -> Vec { + store + .list_podcasts(PodcastListFilter::Subscribed) + .unwrap() + .into_iter() + .map(|summary| summary.podcast_id) + .collect() +} + +fn logged_kinds(device: &Device) -> Vec { + let conn = lock(&device.sync.conn); + let mut stmt = conn + .prepare("SELECT kind FROM sync_ops ORDER BY seq") + .unwrap(); + stmt.query_map([], |row| row.get(0)) + .unwrap() + .collect::>>() + .unwrap() +} + +#[test] +fn a_subscription_on_one_device_appears_on_the_other_without_its_url() { + let a = device(); + let mut b = device(); + pair(&a, &b); + let outcome = subscribe(&a); + assert!(!outcome.already_subscribed); + + deliver(&a, &b); + assert_eq!( + subscribed_ids(&b.store), + std::slice::from_ref(&outcome.podcast_id) + ); + let podcast = b.store.podcast(&outcome.podcast_id).unwrap().unwrap(); + assert_eq!(podcast.meta.title, "Show"); + assert!(!podcast.shared, "synced subscriptions stay private"); + // B fetches the feed itself, from the synced URL. + assert_eq!( + b.store.podcasts_due_for_refresh(now_ms(), 16).unwrap(), + [(outcome.podcast_id.clone(), meta().feed_url)] + ); + // The UI hears about it once. + assert_eq!(synced_events(&mut b), [(Vec::new(), true)]); + + // Delivering the same state again (operations and snapshot) changes + // nothing and creates no duplicate. + receive(&b.sync, &hello_json(&a, &b.id)); + deliver(&a, &b); + assert_eq!(subscribed_ids(&b.store).len(), 1); + assert!(synced_events(&mut b).is_empty()); +} + +#[test] +fn progress_saved_on_one_device_resumes_on_the_other_after_a_restart() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("podcasts.sqlite3"); + let a = device(); + let mut b = device_with_store(Arc::new(PodcastStore::open(&path).unwrap()), true); + pair(&a, &b); + subscribe(&a); + let episode = episode_ref(1); + let session = start(&a, &episode, 0); + checkpoint(&a, &session, 600_000); + checkpoint(&a, &session, 1_800_000); + // Periodic checkpoints are coalesced: one progress operation remains. + assert_eq!( + logged_kinds(&a) + .iter() + .filter(|kind| *kind == "podcast_progress_set") + .count(), + 1 + ); + + deliver(&a, &b); + assert_eq!( + b.store.resume_position_ms(&episode.episode_id).unwrap(), + 1_800_000 + ); + assert!(!synced_events(&mut b).is_empty()); + // A later checkpoint arrives as an operation only: the UI gets the + // register to show in place. + checkpoint(&a, &session, 1_830_000); + deliver(&a, &b); + let events = synced_events(&mut b); + assert!( + events + .iter() + .flat_map(|(progress, _)| progress) + .any(|record| record.position_ms == 1_830_000), + "{events:?}" + ); + + // Days later: B starts again with the same files. + drop(b); + let store = Arc::new(PodcastStore::open(&path).unwrap()); + let b = device_with_store(Arc::clone(&store), true); + let queued = TrackItem::podcast_episode(episode.clone()); + let (resolved, start_ms) = crate::podcasts::playback::resolve( + Some(&b.store), + queued.podcast_episode_ref().unwrap(), + crate::podcasts::playback::StartAt::Resume, + ) + .unwrap(); + assert_eq!(start_ms, 1_830_000); + assert_eq!(resolved.episode_id, episode.episode_id); + assert!(resolved.progress_generation >= Some(1)); +} + +#[test] +fn start_over_on_one_device_beats_a_late_checkpoint_from_the_other() { + let a = device(); + let b = device(); + pair(&a, &b); + subscribe(&a); + let episode = episode_ref(2); + let session_a = start(&a, &episode, 0); + checkpoint(&a, &session_a, 2_400_000); + sync_round(&a, &b); + assert_eq!( + b.store.resume_position_ms(&episode.episode_id).unwrap(), + 2_400_000 + ); + + // B starts over while A (offline) still plays and saves a checkpoint + // later in wall-clock time. + restart(&b, &episode); + std::thread::sleep(Duration::from_millis(5)); + assert!(matches!( + checkpoint(&a, &session_a, 2_460_000), + CheckpointOutcome::Written(_) + )); + // Delivery in both orders converges on the start over. + deliver(&a, &b); + deliver(&b, &a); + for device in [&a, &b] { + let progress = device.store.progress(&episode.episode_id).unwrap().unwrap(); + assert_eq!(progress.position_ms, 0, "{progress:?}"); + assert_eq!(progress.status, ProgressStatus::InProgress); + assert_eq!(progress.device_id, b.id); + } + // A's session stops saving: its next checkpoint is superseded. + assert!(matches!( + checkpoint(&a, &session_a, 2_470_000), + CheckpointOutcome::Superseded { .. } + )); +} + +#[test] +fn unsubscribing_keeps_progress_and_resubscribing_continues() { + let a = device(); + let b = device(); + pair(&a, &b); + let podcast_id = subscribe(&a).podcast_id; + let episode = episode_ref(3); + let session = start(&a, &episode, 0); + checkpoint(&a, &session, 900_000); + sync_round(&a, &b); + + let store = Arc::clone(&a.store); + a.sync + .run_podcast_action(|stamps| { + let ops = store.unsubscribe(stamps, &podcast_id)?; + Ok(((), ops)) + }) + .unwrap(); + sync_round(&a, &b); + for device in [&a, &b] { + assert!(subscribed_ids(&device.store).is_empty()); + assert!( + !device + .store + .subscription(&podcast_id) + .unwrap() + .unwrap() + .subscribed + ); + assert_eq!( + device + .store + .resume_position_ms(&episode.episode_id) + .unwrap(), + 900_000 + ); + } + + // Subscribing again on B reuses the podcast and its progress, and the + // later decision wins on A as well. + let again = subscribe(&b); + assert_eq!(again.podcast_id, podcast_id); + sync_round(&b, &a); + for device in [&a, &b] { + assert_eq!( + subscribed_ids(&device.store), + std::slice::from_ref(&podcast_id) + ); + assert_eq!( + device + .store + .resume_position_ms(&episode.episode_id) + .unwrap(), + 900_000 + ); + } +} + +#[test] +fn operations_received_before_the_store_opens_are_applied_when_it_does() { + let a = device(); + let mut b = device_with_store(Arc::new(PodcastStore::open_in_memory().unwrap()), false); + pair(&a, &b); + let podcast_id = subscribe(&a).podcast_id; + let episode = episode_ref(4); + let session = start(&a, &episode, 0); + checkpoint(&a, &session, 120_000); + deliver(&a, &b); + assert!(subscribed_ids(&b.store).is_empty()); + // The operations are in B's log (and acknowledged) already: nothing + // would deliver them again. + assert!( + logged_kinds(&b) + .iter() + .any(|kind| kind == "podcast_subscription_set") + ); + + b.sync.attach_podcast_store(Arc::clone(&b.store)).unwrap(); + assert_eq!(subscribed_ids(&b.store), [podcast_id]); + assert_eq!( + b.store.resume_position_ms(&episode.episode_id).unwrap(), + 120_000 + ); + assert!(!synced_events(&mut b).is_empty()); + let pending: i64 = lock(&b.sync.conn) + .query_row("SELECT COUNT(*) FROM sync_podcast_pending", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(pending, 0); +} + +#[test] +fn podcast_operations_are_relayed_to_devices_that_were_not_online() { + let a = device(); + let b = device(); + let c = device(); + pair(&a, &b); + pair(&b, &c); + let podcast_id = subscribe(&a).podcast_id; + deliver(&a, &b); + // C never talks to A. + deliver(&b, &c); + assert_eq!(subscribed_ids(&c.store), [podcast_id]); +} + +/// A peer that still runs protocol v3 (furumi-tui 0.3.x, furumi-web +/// 0.12.x), simulated with its exact JSON types. +#[test] +fn a_v3_peer_gets_no_podcast_state_and_catches_up_from_the_snapshot_after_upgrading() { + use super::wire_compat_tests::legacy; + + let mut a = device(); + let old = device(); + a.sync.set_group_id("grp_test").unwrap(); + let old_profile = legacy::DeviceProfileWire { + device_id: old.id.clone(), + name: "old".into(), + client_version: "0.3.5".into(), + protocol_version: 3, + endpoint_id: String::new(), + endpoint_ticket: String::new(), + revoked: false, + revoke_cutoff_seq: None, + updated_at_ms: 1, + }; + // Its hello, as a 0.3.x client writes it, with a profile announcement + // that has no version field. + let old_hello = legacy::WireMessage::Hello { + group_id: "grp_test".into(), + profile: old_profile.clone(), + devices: vec![old_profile.clone()], + vector: BTreeMap::from([(old.id.clone(), 1)]), + ops: vec![legacy::SyncOpWire { + op_id: format!("{}:1", old.id), + origin_device_id: old.id.clone(), + seq: 1, + hlc_ms: 1, + payload: legacy::SyncOpPayload::DeviceProfileSet { + name: "old".into(), + client_version: "0.3.5".into(), + endpoint_ticket: String::new(), + endpoint_id: String::new(), + }, + }], + snapshot: legacy::SyncSnapshot::default(), + playback: None, + }; + a.sync.apply_device_trusted(&old.id, 1).unwrap(); + receive(&a.sync, &serde_json::to_string(&old_hello).unwrap()); + assert_eq!(a.sync.peer_protocol_version(&old.id).unwrap(), 3); + + let podcast_id = subscribe(&a).podcast_id; + let episode = episode_ref(5); + let session = start(&a, &episode, 0); + checkpoint(&a, &session, 300_000); + a.sync + .record_content_like(&format!("b3:{}", "c".repeat(64)), true) + .unwrap(); + a.sync.publish_playback(PlaybackSnapshot { + device_id: a.id.clone(), + device_name: "A".into(), + active: true, + updated_at_ms: now_ms(), + state: PlaybackStateWire { + queue: vec![PlaybackTrack::podcast_episode(episode.clone())], + queue_pos: 0, + playing: true, + paused: false, + idle_since_ms: None, + position_secs: 300.0, + volume: 80, + shuffle: false, + repeat: PlaybackRepeat::Off, + playback_rate: None, + }, + coordination: None, + }); + + // What A sends the v3 peer parses with the v3 types and has no podcast + // operation, podcast section or podcast-bearing playback state. + let json = hello_json(&a, &old.id); + let parsed: legacy::WireMessage = serde_json::from_str(&json).unwrap(); + let legacy::WireMessage::Hello { + ops, + snapshot, + playback, + vector, + .. + } = parsed + else { + panic!("expected a hello"); + }; + assert!(!json.contains("podcast"), "{json}"); + assert_eq!(ops.len(), 1, "only the like: {ops:?}"); + assert_eq!(snapshot.likes.len(), 1); + assert!(playback.is_none()); + // The old peer acknowledges everything it was sent. + a.sync.note_peer_vector(&old.id, &vector).unwrap(); + + // After upgrading, its hello reports v4: the skipped operations are not + // sent again, but the snapshot carries the podcast state. + receive(&a.sync, &hello_json(&old, &a.id)); + assert_eq!( + a.sync.peer_protocol_version(&old.id).unwrap(), + PROTOCOL_VERSION + ); + let json = hello_json(&a, &old.id); + let WireMessage::Hello { + ops, + snapshot, + playback, + .. + } = serde_json::from_str(&json).unwrap() + else { + panic!("expected a hello"); + }; + assert!(ops.iter().all(|op| !op.payload.is_podcast()), "{ops:?}"); + assert_eq!(snapshot.podcast_subscriptions.len(), 1); + assert_eq!(snapshot.podcast_progress.len(), 1); + assert!(playback.is_some()); + receive(&old.sync, &json); + assert_eq!(subscribed_ids(&old.store), [podcast_id]); + assert_eq!( + old.store.resume_position_ms(&episode.episode_id).unwrap(), + 300_000 + ); + let _ = synced_events(&mut a); +} + +#[test] +fn podcast_snapshot_sections_shrink_to_fit_the_line_limit() { + let a = device(); + let b = device(); + pair(&a, &b); + subscribe(&a); + let mut message = WireMessage::Hello { + group_id: String::new(), + profile: a.sync.own_profile("").unwrap(), + devices: Vec::new(), + vector: BTreeMap::new(), + ops: Vec::new(), + snapshot: a.sync.snapshot_for(PROTOCOL_VERSION).unwrap(), + playback: None, + }; + assert_eq!( + message_snapshot(&mut message).podcast_subscriptions.len(), + 1 + ); + // A music part that leaves no room for podcast state. + message_snapshot(&mut message).likes = (0..70_000) + .map(|n| SnapshotLike { + content_id: format!("b3:{n:064x}"), + hlc_ms: n, + op_id: format!("x:{n}"), + fed: None, + }) + .collect(); + fit_podcast_snapshot(&mut message); + assert!(!message_snapshot(&mut message).has_podcast_state()); + // The music part is never cut for podcast state. + assert_eq!(message_snapshot(&mut message).likes.len(), 70_000); +} diff --git a/src/devices/tests.rs b/src/devices/tests.rs index a0ca162..b858a80 100644 --- a/src/devices/tests.rs +++ b/src/devices/tests.rs @@ -30,6 +30,7 @@ async fn localhost_devices_exchange_state_and_handoff() { library: Arc::new(Library::open(&dir.path().join("library.sqlite3")).unwrap()), event_tx: Default::default(), playback: Default::default(), + podcasts: Default::default(), }); let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); sync.set_event_tx(tx); @@ -233,6 +234,121 @@ async fn localhost_devices_exchange_state_and_handoff() { .expect("localhost sync must complete within 30 seconds"); } +/// Podcast state over the production stream handlers: a subscription and +/// progress from A reach B through `sync_device` → `handle_hello`, and B's +/// start over reaches A through the response of B's own sync. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn localhost_devices_sync_podcasts() { + use super::podcast_sync_tests::{checkpoint, device_with_store, episode_ref, start, subscribe}; + tokio::time::timeout(Duration::from_secs(30), async { + let dirs = [tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()]; + let network = NetworkId::from_name(&format!("podcast-sync-test-{}", random_hex(16))); + let mut peers = Vec::new(); + let mut servers = Vec::new(); + for dir in &dirs { + std::fs::create_dir_all(dir.path().join("network")).unwrap(); + let config = music_dht::MusicDhtConfig::builder() + .data_dir(dir.path().join("network")) + .network_id(network) + .stream_protocol(SYNC_ALPN) + .build() + .unwrap(); + let (service, _events) = MusicDhtService::start(config).await.unwrap(); + let service = Arc::new(service); + let store = Arc::new( + furumi_library::podcasts::PodcastStore::open(&dir.path().join("podcasts.sqlite3")) + .unwrap(), + ); + let device = device_with_store(store, true); + let sync = Arc::new(device.sync.clone()); + let stats = Arc::new(crate::federation::TransportStats::default()); + servers.push(tokio::spawn(serve_peers( + service.stream_acceptor(SYNC_ALPN).unwrap(), + sync.clone(), + service.clone(), + stats.clone(), + ))); + peers.push((device, sync, service, stats)); + } + let (a, sync_a, service_a, stats_a) = &peers[0]; + let (b, sync_b, service_b, stats_b) = &peers[1]; + sync_b + .set_group_id(&sync_a.ensure_identity().unwrap().group_id) + .unwrap(); + let profile_a = sync_a + .own_profile(&service_a.ticket().await.unwrap().to_string()) + .unwrap(); + let profile_b = sync_b + .own_profile(&service_b.ticket().await.unwrap().to_string()) + .unwrap(); + for (sync, profile) in [(sync_a, &profile_b), (sync_b, &profile_a)] { + sync.apply_device_profile( + profile, + true, + VersionReport::SelfReported(Some(profile.protocol_version)), + ) + .unwrap(); + } + + let podcast_id = subscribe(a).podcast_id; + let episode = episode_ref(1); + let session = start(a, &episode, 0); + checkpoint(a, &session, 1_200_000); + sync_a + .sync_device( + service_a.clone(), + &sync_a.active_remote_devices().unwrap()[0], + stats_a.clone(), + ) + .await + .unwrap(); + let subscribed = b + .store + .list_podcasts(furumi_library::podcasts::PodcastListFilter::Subscribed) + .unwrap(); + assert_eq!(subscribed.len(), 1); + assert_eq!(subscribed[0].podcast_id, podcast_id); + assert_eq!( + b.store.resume_position_ms(&episode.episode_id).unwrap(), + 1_200_000 + ); + + // B starts over; its next sync carries the start over to A, which + // then refuses A's older session. + let store = Arc::clone(&b.store); + sync_b + .run_podcast_action(|stamps| { + let write = store.restart(stamps, &episode, "restart-on-b")?; + Ok(((), vec![write.op])) + }) + .unwrap(); + sync_b + .sync_device( + service_b.clone(), + &sync_b.active_remote_devices().unwrap()[0], + stats_b.clone(), + ) + .await + .unwrap(); + let progress = a.store.progress(&episode.episode_id).unwrap().unwrap(); + assert_eq!(progress.position_ms, 0); + assert_eq!(progress.session_id, "restart-on-b"); + assert!(matches!( + checkpoint(a, &session, 1_230_000), + furumi_library::podcasts::CheckpointOutcome::Superseded { .. } + )); + + for server in servers { + server.abort(); + } + for (_, _, service, _) in &peers { + service.shutdown().await.unwrap(); + } + }) + .await + .expect("localhost podcast sync must complete within 30 seconds"); +} + fn empty_playback_state() -> PlaybackStateWire { PlaybackStateWire { queue: vec![], @@ -298,7 +414,7 @@ fn coordination_ignores_legacy_commands_and_wrong_snapshot_sender() { coordination: None, }); let gossip = sync - .local_playback_snapshot() + .local_playback_snapshot_for(PROTOCOL_VERSION) .unwrap() .coordination .unwrap(); @@ -358,6 +474,7 @@ fn test_sync() -> DeviceSync { library: Arc::new(Library::open(&library_path).unwrap()), event_tx: Arc::new(std::sync::Mutex::new(None)), playback: Arc::new(std::sync::Mutex::new(PlaybackShared::default())), + podcasts: Default::default(), }; sync.ensure_identity().unwrap(); sync @@ -481,6 +598,35 @@ fn tombstone_detection() { ); } +fn music_track(id: i64, content_id: &str) -> TrackItem { + TrackItem { + id, + title: "Local Song".to_string(), + track_number: Some(1), + disc_number: Some(1), + duration_seconds: 180.0, + artists: vec![ArtistRef { + id: 1, + name: "Local Artist".to_string(), + }], + featured_artists: Vec::new(), + release_id: 2, + release_title: "Local Release".to_string(), + release_year: Some(2026), + file_path: format!("/music/{id}.mp3"), + content_id: Some(content_id.to_string()), + cover_path: None, + audio_format: Some("mp3".to_string()), + audio_bitrate: Some(320), + audio_sample_rate: Some(44_100), + audio_bit_depth: None, + file_size_bytes: Some(123_456), + play_count: 3, + fed: None, + podcast: None, + } +} + #[test] fn playback_tracks_do_not_sync_device_local_paths() { let source = TrackItem { @@ -661,7 +807,7 @@ fn playback_commands_are_caught_up_only_by_their_target_while_fresh() { .unwrap(); sync.record_playback_command("dev_other", command).unwrap(); - let target_ops = sync.ops_for_peer("dev_target").unwrap(); + let target_ops = sync.ops_for_peer("dev_target", PROTOCOL_VERSION).unwrap(); assert_eq!( target_ops .iter() @@ -670,7 +816,7 @@ fn playback_commands_are_caught_up_only_by_their_target_while_fresh() { 1 ); assert!( - sync.ops_for_peer("dev_unknown") + sync.ops_for_peer("dev_unknown", PROTOCOL_VERSION) .unwrap() .iter() .all(|op| !matches!(op.payload, SyncOpPayload::PlaybackCommand { .. })) @@ -685,7 +831,7 @@ fn playback_commands_are_caught_up_only_by_their_target_while_fresh() { ) .unwrap(); assert!( - sync.ops_for_peer("dev_target") + sync.ops_for_peer("dev_target", PROTOCOL_VERSION) .unwrap() .iter() .all(|op| !matches!(op.payload, SyncOpPayload::PlaybackCommand { .. })) @@ -1070,7 +1216,7 @@ fn podcast_actions_are_appended_to_the_sync_log_with_their_stamps() { } #[test] -fn podcast_ops_are_not_sent_to_peers_before_sync_v4() { +fn podcast_ops_are_sent_only_to_v4_peers() { let sync = test_sync(); let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap(); sync.run_podcast_action(|stamps| { @@ -1081,11 +1227,36 @@ fn podcast_ops_are_not_sent_to_peers_before_sync_v4() { .unwrap(); sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true) .unwrap(); - // This client still speaks protocol v3; its wire types cannot carry the - // podcast payload, so the batch keeps only the music operation. - let ops = sync.ops_for_peer("dev_peer").unwrap(); + // A v3 peer cannot parse the podcast payload (an unknown `kind` breaks + // its whole message), so its batch keeps only the music operation. + let ops = sync.ops_for_peer("dev_peer", 3).unwrap(); assert_eq!(ops.len(), 1); assert!(matches!(ops[0].payload, SyncOpPayload::TrackLikeSet { .. })); + let ops = sync.ops_for_peer("dev_peer", 4).unwrap(); + assert_eq!(ops.len(), 2); + assert!(ops[0].payload.is_podcast()); + // Unreported versions are the oldest protocol, except that every + // `furumi/sync/2` peer keeps listen history. + let event = sync + .listen_event_for_track( + sync.new_listen_id(), + &music_track(7, &format!("b3:{}", "b".repeat(64))), + 1, + 120_000, + music_dht::device_sync::ListenEndReason::Finished, + ) + .unwrap(); + sync.record_listen(event).unwrap(); + let kinds: Vec<_> = sync + .ops_for_peer( + "dev_peer", + music_dht::device_sync::UNREPORTED_PROTOCOL_VERSION, + ) + .unwrap() + .into_iter() + .map(|op| payload_kind(&op.payload)) + .collect(); + assert_eq!(kinds, ["track_like_set", "listen_recorded"]); } #[test]