//! Personal-device sync: trusted furumi clients that belong to one listener. //! //! The transport is a small JSON-lines protocol over a dedicated frid/iroh //! byte-stream ALPN. Local state is stored as an append-only operation log plus //! materialized tables, so offline clients can merge likes, playlists and //! membership changes deterministically. //! //! The wire types are the shared `music_dht::device_sync` definitions. This //! client speaks protocol v4: podcast operations stay in the log, are //! applied to the podcast store (`furumi_library::podcasts::PodcastStore`) //! and are relayed; podcast snapshot sections are exported from and merged //! into that store. Everything sent to a peer is gated by the version that //! peer reported itself (`op_allowed_for_peer`, `SyncSnapshot::for_peer`, //! `playback_allowed_for_peer`; a queue with podcast items reaches an older //! peer only redacted, for its ownership engine). use music_dht::playback::{Checkpoint, CommandStamp, Config as PlaybackConfig, Engine}; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use anyhow::{Context as _, Result}; use furumi_library::podcasts::{ ApplyOutcome as PodcastApplyOutcome, OpStamp as PodcastOpStamp, PodcastLocalOp, PodcastStore, StampSource as PodcastStampSource, }; use music_dht::device_sync::{ DeviceProfileWire, InviteWire, ListenEvent, ListenTrackMetadata, SnapshotLike, SnapshotLikeTombstone, SnapshotPlaylist, SnapshotPlaylistItem, SnapshotPlaylistItemTombstone, SnapshotPlaylistTombstone, SyncOpWire, SyncSnapshot, WireMessage, encode_invite, finish_send, hash_secret, random_hex, read_msg, ticket_endpoint_id, }; pub use music_dht::device_sync::{ PlaybackCommand, PlaybackRepeat, PlaybackSnapshot, PlaybackStateWire, PlaybackTrack, SyncOpPayload, SyncedFedTrack, }; use music_dht::podcast::ProgressRecord; use music_dht::{ByteStream, MusicDhtService, NetworkId, PeerTicket, SecretKey, StreamAcceptor}; use rusqlite::{Connection, OptionalExtension, params}; use crate::app::event::AppEvent; use crate::library::Library; use crate::library::models::{ArtistRef, TrackItem}; pub const SYNC_ALPN: &[u8] = b"furumi/sync/2"; const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// Device-sync protocol version of this client. /// /// Version 4 is announced because podcast operations, podcast snapshot /// sections and podcast queue items are applied, kept and relayed (see the /// module documentation). Review that support before following a newer /// shared protocol version. pub const PROTOCOL_VERSION: u16 = 4; const _: () = assert!( PROTOCOL_VERSION == music_dht::device_sync::DEVICE_SYNC_PROTOCOL_VERSION, "review device-sync support before announcing a new protocol version" ); /// Lowest protocol version of a peer that speaks `furumi/sync/2`. const SYNC_V2_PROTOCOL_VERSION: u16 = 2; const INVITE_TTL_MS: i64 = 10 * 60 * 1000; const PAIRING_WAIT_MS: i64 = 5 * 60 * 1000; const PAIRING_RETRY_DELAY: Duration = Duration::from_secs(1); const RESPONSE_DRAIN_TIMEOUT: Duration = Duration::from_secs(2); const SYNC_INTERVAL: Duration = Duration::from_secs(2); /// Playback control is ephemeral. Keeping old full-queue commands in a new /// peer's catch-up batch can make the initial pairing frame arbitrarily large. const PLAYBACK_COMMAND_TTL_MS: i64 = 5 * 60 * 1_000; const MAX_OPS_PER_BATCH: usize = 1000; #[derive(Debug, Clone, PartialEq, Eq)] pub struct PendingPairing { pub request_id: String, pub device_id: String, pub name: String, pub client_version: String, pub requester_group_id: Option, pub requester_group_active_devices: usize, } #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct DeviceStatusRow { pub device_id: String, pub name: String, pub client_version: String, /// Device-sync protocol version the device reported itself. pub protocol_version: u16, pub endpoint_id: String, pub last_seen_ms: Option, pub revoked: bool, pub is_self: bool, } #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct DeviceSyncStatus { pub this_device_id: String, pub this_device_name: String, pub group_id: String, pub active_devices: usize, pub revoked_devices: usize, pub pending_requests: usize, pub ops_total: usize, pub tombstone_ops: usize, pub compactable_tombstones: usize, pub outbox_ops: usize, pub snapshot_likes: usize, pub snapshot_playlists: usize, pub snapshot_items: usize, pub unresolved_playlist_items: usize, pub peer_ack_floor: String, pub last_sync: Option, pub last_error: Option, pub devices: Vec, } #[derive(Clone)] pub struct DeviceSync { // A device identity has exactly one coordinator, including across binaries // launched from different installation directories. The OS releases this // lock on crashes; the file itself is deliberately never removed. _identity_lock: Option>, conn: Arc>, library: Arc, event_tx: Arc>>>, playback: Arc>, podcasts: Arc>, } /// The podcast store as device sync sees it. /// /// The store opens in the background after device sync started. Podcast /// operations received before that (or whose application failed) are kept /// in `sync_podcast_pending` and applied when the store is attached. #[derive(Default)] struct PodcastLink { store: Option>, /// Progress registers that received operations changed since the last /// notification. progress: Vec, /// Other podcast state (subscriptions, shares, feed URLs, sessions) /// changed since the last notification. changed: bool, } /// Converts queue items to and from the shared wire form /// ([`PlaybackTrack`]). The wire types themselves are the shared /// `music_dht::device_sync` definitions; this client keeps no copies. /// /// A podcast episode travels with its explicit source /// (`PlaybackSource::PodcastEpisode`), never as a music item. Peers below /// protocol v4 never get the episodes (`playback_allowed_for_peer`): they /// see the queue redacted to "Private item" placeholders, only so that their /// ownership engine keeps hearing from the owner. pub trait PlaybackTrackExt { /// Wire form of a queue item. Device-local paths are never sent. fn from_track(track: &TrackItem) -> Self; /// Queue item for a wire track received from another device. A Jam /// private placeholder becomes [`private_track_item`]`(0)`; callers /// that keep a queue number them ([`track_items_from_wire`]). fn to_track_item(&self) -> TrackItem; } /// Ids of the "private item" queue entries that stand for a Jam host's /// podcast episodes on a guest: `PRIVATE_ITEM_ID_BASE - k` for the k-th, so /// each has its own queue key. Process-local; never sent (they go back on /// the wire as `PlaybackSource::Private`). const PRIVATE_ITEM_ID_BASE: i64 = -(1 << 62); const PRIVATE_ITEM_ID_SPAN: i64 = 1 << 32; /// Queue entry for the k-th Jam private placeholder: no title but /// [`music_dht::device_sync::PRIVATE_ITEM_TITLE`], no URL, not playable. pub fn private_track_item(k: usize) -> TrackItem { TrackItem { id: PRIVATE_ITEM_ID_BASE - (k as i64 % PRIVATE_ITEM_ID_SPAN), title: music_dht::device_sync::PRIVATE_ITEM_TITLE.to_string(), track_number: None, disc_number: None, duration_seconds: 0.0, artists: Vec::new(), featured_artists: Vec::new(), release_id: TrackItem::NOT_A_LIBRARY_ROW, release_title: String::new(), release_year: None, file_path: String::new(), content_id: None, cover_path: None, audio_format: None, audio_bitrate: None, audio_sample_rate: None, audio_bit_depth: None, file_size_bytes: None, play_count: 0, fed: None, podcast: None, } } /// Whether a queue entry is a Jam private placeholder. pub fn is_private_track_item(track: &TrackItem) -> bool { track.podcast.is_none() && track.fed.is_none() && track.content_id.is_none() && track.id <= PRIVATE_ITEM_ID_BASE && track.id > PRIVATE_ITEM_ID_BASE - PRIVATE_ITEM_ID_SPAN && track.title == music_dht::device_sync::PRIVATE_ITEM_TITLE } /// Queue entries for a received queue, with `resolve_music` deciding music /// items (a library copy, for example). Podcast episodes keep their typed /// reference; Jam private placeholders are numbered in queue order. pub fn track_items_from_wire( queue: &[PlaybackTrack], mut resolve_music: impl FnMut(&PlaybackTrack) -> TrackItem, ) -> Vec { let mut private = 0; queue .iter() .map(|track| match TrackItem::from_playback_track(track) { furumi_library::models::PlaybackItem::Music => resolve_music(track), furumi_library::models::PlaybackItem::PodcastEpisode(item) => *item, furumi_library::models::PlaybackItem::Private => { private += 1; private_track_item(private - 1) } }) .collect() } impl PlaybackTrackExt for PlaybackTrack { fn from_track(track: &TrackItem) -> Self { if let Some(episode) = &track.podcast { return Self::podcast_episode(episode.clone()); } if is_private_track_item(track) { return Self::private_placeholder(); } Self { id: track.id, title: track.title.clone(), track_number: track.track_number, disc_number: track.disc_number, duration_seconds: track.duration_seconds, artist_names: track .artists .iter() .map(|artist| artist.name.clone()) .collect(), featured_artist_names: track .featured_artists .iter() .map(|artist| artist.name.clone()) .collect(), release_id: track.release_id, release_title: track.release_title.clone(), release_year: track.release_year, file_path: String::new(), content_id: track .content_id .clone() .or_else(|| track.fed.as_ref().and_then(|fed| fed.content_id.clone())), audio_format: track.audio_format.clone(), audio_bitrate: track.audio_bitrate, audio_sample_rate: track.audio_sample_rate, audio_bit_depth: track.audio_bit_depth, file_size_bytes: track.file_size_bytes, play_count: track.play_count, fed: track.fed.as_ref().and_then(SyncedFedTrack::from_fed), source: None, } } fn to_track_item(&self) -> TrackItem { match TrackItem::from_playback_track(self) { furumi_library::models::PlaybackItem::Music => {} furumi_library::models::PlaybackItem::PodcastEpisode(item) => return *item, furumi_library::models::PlaybackItem::Private => return private_track_item(0), } let refs = |names: &[String]| -> Vec { names .iter() .map(|name| ArtistRef { id: -1, name: name.clone(), }) .collect() }; TrackItem { id: portable_placeholder_id(self), title: self.title.clone(), track_number: self.track_number, disc_number: self.disc_number, duration_seconds: self.duration_seconds, artists: refs(&self.artist_names), featured_artists: refs(&self.featured_artist_names), release_id: self.release_id, release_title: self.release_title.clone(), release_year: self.release_year, file_path: String::new(), content_id: self.content_id.clone(), cover_path: None, audio_format: self.audio_format.clone(), audio_bitrate: self.audio_bitrate, audio_sample_rate: self.audio_sample_rate, audio_bit_depth: self.audio_bit_depth, file_size_bytes: self.file_size_bytes, play_count: self.play_count, fed: self.fed.as_ref().map(SyncedFedTrack::to_fed_track), podcast: None, } } } /// Process-local negative id of a music item received from another device, /// derived from its portable identity (content id, then federated source). fn portable_placeholder_id(track: &PlaybackTrack) -> i64 { let key = track .content_id .as_deref() .and_then(music_dht::normalize_content_id) .map(|content_id| format!("content:{content_id}")) .or_else(|| { track .fed .as_ref() .map(|fed| fed.content_id.as_str()) .and_then(music_dht::normalize_content_id) .map(|content_id| format!("content:{content_id}")) }) .or_else(|| { track .fed .as_ref() .map(|fed| format!("fed:{}:{}", fed.owner, fed.item_id)) }) .unwrap_or_else(|| { format!( "remote:{}:{}:{}:{}", track.id, track.title, track.release_title, track.duration_seconds ) }); let hash = blake3::hash(key.as_bytes()); let mut bytes = [0u8; 8]; bytes.copy_from_slice(&hash.as_bytes()[..8]); let positive = (i64::from_be_bytes(bytes) & i64::MAX).max(1); -positive } /// Converts federated tracks to and from the synced metadata form. trait SyncedFedTrackExt: Sized { fn from_fed(fed: &crate::federation::FedTrack) -> Option; fn to_fed_track(&self) -> crate::federation::FedTrack; } impl SyncedFedTrackExt for SyncedFedTrack { fn from_fed(fed: &crate::federation::FedTrack) -> Option { let content_id = fed .content_id .as_deref() .and_then(music_dht::normalize_content_id)?; Some(Self { item_id: fed.item_id.clone(), owner: fed.owner.clone(), title: fed.title.clone(), artist_names: fed.artist_names.clone(), featured_artist_names: fed.featured_artist_names.clone(), year: fed.year, duration_seconds: fed.duration_seconds, content_id, release_title: fed.release_title.clone(), track_number: fed.track_number, disc_number: fed.disc_number, }) } fn to_fed_track(&self) -> crate::federation::FedTrack { crate::federation::FedTrack { item_id: self.item_id.clone(), owner: self.owner.clone(), own: false, title: self.title.clone(), artist_names: self.artist_names.clone(), featured_artist_names: self.featured_artist_names.clone(), year: self.year, duration_seconds: self.duration_seconds, content_id: Some(self.content_id.clone()), release_title: self.release_title.clone(), track_number: self.track_number, disc_number: self.disc_number, } } } #[derive(Debug, Clone, Default)] struct PlaybackShared { engine: Option, engine_group: String, local: Option, remote: BTreeMap, } #[derive(Debug, Clone)] struct LocalIdentity { device_id: String, group_id: String, name: String, } enum PairAttempt { Accepted(String), Pending, Denied(String), } struct PairingStatus { status: String, use_requester_group: bool, } /// How the protocol version in a device profile became known. /// /// Version gating (podcast operations, snapshot sections and playback /// state need v4) depends on these values, so this device never stores its /// own [`PROTOCOL_VERSION`] for another device. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum VersionReport { /// The device reported its own version (`None`: an older client left it /// out). Replaces the stored version, so a downgrade is noticed. SelfReported(Option), /// Another device relayed the profile. Only raises the stored version: /// v1–v3 relays substitute their own constant. Relayed(u16), } impl VersionReport { fn resolve(self, stored: Option) -> u16 { match self { VersionReport::SelfReported(reported) => { music_dht::device_sync::profile_protocol_version(reported, stored) } VersionReport::Relayed(relayed) => { music_dht::device_sync::relayed_protocol_version(relayed, stored) } } } /// Report carried by an entry of a `devices` list sent by `sender`. fn for_listed(profile: &DeviceProfileWire, sender: Option<&str>) -> Self { if sender == Some(profile.device_id.as_str()) { VersionReport::SelfReported(Some(profile.protocol_version)) } else { VersionReport::Relayed(profile.protocol_version) } } } /// The stored protocol version of a device, if any row exists. fn stored_protocol_version(conn: &Connection, device_id: &str) -> Result> { Ok(conn .query_row( "SELECT protocol_version FROM sync_devices WHERE device_id = ?1", [device_id], |row| row.get::<_, i64>(0), ) .optional()? .and_then(|version| u16::try_from(version).ok())) } fn lock(mutex: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { mutex .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } fn now_ms() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_millis() as i64) .unwrap_or(0) } fn now_label() -> String { let secs = (now_ms() / 1000).max(0); format!( "{:02}:{:02}:{:02} UTC", secs / 3600 % 24, secs / 60 % 60, secs % 60 ) } fn default_db_path() -> PathBuf { crate::config::project_dirs() .map(|dirs| dirs.data_dir().join("devices").join("sync.sqlite3")) .unwrap_or_else(|| PathBuf::from("devices").join("sync.sqlite3")) } fn acquire_identity_lock(database: &std::path::Path) -> Result { let path = database.with_extension("lock"); let file = std::fs::OpenOptions::new() .read(true) .write(true) .create(true) .truncate(false) .open(&path) .with_context(|| format!("opening device identity lock {}", path.display()))?; file.try_lock().with_context(|| format!( "Cannot acquire device identity lock {}. Another Furumi instance may already be using this device. Close it before launching another binary; two players must not share one device identity.", path.display() ))?; Ok(file) } impl DeviceSync { pub fn new(library: Arc) -> Result> { let path = default_db_path(); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } let identity_lock = acquire_identity_lock(&path)?; let conn = Connection::open(&path).with_context(|| format!("opening {}", path.display()))?; init_schema(&conn)?; let sync = Arc::new(Self { _identity_lock: Some(Arc::new(identity_lock)), conn: Arc::new(std::sync::Mutex::new(conn)), library, event_tx: Arc::new(std::sync::Mutex::new(None)), playback: Arc::new(std::sync::Mutex::new(PlaybackShared::default())), podcasts: Arc::default(), }); sync.ensure_identity()?; sync.repair_like_order_from_sync_state()?; Ok(sync) } pub fn set_event_tx(&self, tx: tokio::sync::mpsc::UnboundedSender) { *lock(&self.event_tx) = Some(tx); } pub fn identity_summary(&self) -> Result<(String, String)> { let identity = self.ensure_identity()?; Ok((identity.device_id, identity.name)) } pub fn new_listen_id(&self) -> String { format!("{}-{}", now_ms(), random_hex(12)) } pub fn configure_playback(&self, config: PlaybackConfig) -> Result<()> { set_meta( &lock(&self.conn), "playback_config_v1", &serde_json::to_string(&config)?, )?; lock(&self.playback).engine = None; Ok(()) } fn with_playback_engine(&self, f: impl FnOnce(&mut Engine) -> R) -> Result { let identity = self.ensure_identity()?; let mut shared = lock(&self.playback); if shared.engine.is_none() || shared.engine_group != identity.group_id { shared.engine_group = identity.group_id.clone(); let conn = lock(&self.conn); let durable = get_meta(&conn, "playback_coordination_v1")? .map(|json| serde_json::from_str::(&json)) .transpose()? .filter(|checkpoint| checkpoint.scope == identity.group_id) .map(|checkpoint| checkpoint.state) .unwrap_or_default(); let config = get_meta(&conn, "playback_config_v1")? .map(|json| serde_json::from_str::(&json)) .transpose()? .unwrap_or_default(); shared.engine = Some(Engine::new( identity.device_id, config, durable, playback_clock(), )); } let engine = shared.engine.as_mut().expect("initialized playback engine"); let previous = engine.clone(); let result = f(engine); if previous.durable() != engine.durable() { let saved = set_meta( &lock(&self.conn), "playback_coordination_v1", &serde_json::to_string(&Checkpoint { scope: identity.group_id, state: engine.durable().clone(), })?, ); if let Err(error) = saved { *engine = previous; return Err(error); } } Ok(result) } /// Evaluate shared ownership independently of the UI's online-device list. pub fn playback_tick(&self, available: bool, playing: bool) -> Result> { self.with_playback_engine(|engine| { engine.set_output(available, playing); engine.heartbeat(playback_clock()); engine.tick(playback_clock()); engine.owner().map(str::to_string) }) } pub fn playback_command_is_current(&self, origin: &str, stamp: &CommandStamp) -> bool { self.with_playback_engine(|engine| engine.command_is_current(origin, stamp)) .unwrap_or(false) } pub fn claim_playback(&self, target: &str) -> Result<()> { self.with_playback_engine(|engine| engine.transfer(target, playback_clock()))? .then_some(()) .context("cannot allocate playback ownership term") } pub fn publish_playback(&self, mut snapshot: PlaybackSnapshot) { if snapshot.updated_at_ms <= 0 { snapshot.updated_at_ms = now_ms(); } let Ok(coordination) = self.with_playback_engine(|engine| engine.announcement()) else { return; }; snapshot.active = coordination .claim .as_ref() .is_some_and(|claim| claim.owner == snapshot.device_id); snapshot.coordination = Some(coordination); lock(&self.playback).local = Some(snapshot); } pub fn record_playback_command( &self, target_device_id: &str, command: PlaybackCommand, ) -> Result<()> { if target_device_id.trim().is_empty() { return Ok(()); } let authority = self .with_playback_engine(|engine| { if let PlaybackCommand::ActiveChanged { active_device_id, .. } = &command && engine.owner() != Some(active_device_id.as_str()) { engine.transfer(active_device_id, playback_clock()); } engine.stamp() })? .context("no playback owner; wait for discovery or select an output")?; self.record_local_op(SyncOpPayload::PlaybackCommand { target_device_id: target_device_id.to_string(), command, authority: Some(authority), }) } pub fn set_device_name(&self, name: &str, endpoint_ticket: Option<&str>) -> Result<()> { let name = if name.trim().is_empty() { "furumi".to_string() } else { name.trim().to_string() }; { let conn = lock(&self.conn); set_meta(&conn, "device_name", &name)?; if let Some(device_id) = get_meta(&conn, "device_id")? { conn.execute( "UPDATE sync_devices SET name = ?2, client_version = ?3 WHERE device_id = ?1", params![device_id, name, CLIENT_VERSION], )?; } } if let Some(ticket) = endpoint_ticket { let endpoint_id = ticket_endpoint_id(ticket).unwrap_or_default(); self.record_local_op(SyncOpPayload::DeviceProfileSet { name, client_version: CLIENT_VERSION.to_string(), endpoint_ticket: ticket.to_string(), endpoint_id, protocol_version: Some(PROTOCOL_VERSION), })?; } Ok(()) } /// Protocol version a device reported itself, for version gating. /// Unknown devices count as the oldest protocol. pub fn peer_protocol_version(&self, device_id: &str) -> Result { let stored = stored_protocol_version(&lock(&self.conn), device_id)?; Ok(music_dht::device_sync::profile_protocol_version( None, stored, )) } /// Connects the podcast store. Podcast operations received from now on /// are applied to it, and those that arrived while it was unavailable /// (or whose application failed) are applied first. Blocking. pub fn attach_podcast_store(&self, store: Arc) -> Result<()> { let pending = { // Same lock order as `apply_podcast_op`: link, then log. An // operation received concurrently is either in this list or // sees the attached store. let mut link = lock(&self.podcasts); link.store = Some(Arc::clone(&store)); let conn = lock(&self.conn); // Rows whose operation was coalesced away have nothing to apply. conn.execute( "DELETE FROM sync_podcast_pending WHERE op_id NOT IN (SELECT op_id FROM sync_ops)", [], )?; let mut stmt = conn.prepare( "SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json FROM sync_podcast_pending p JOIN sync_ops o ON o.op_id = p.op_id ORDER BY o.hlc_ms, o.op_id", )?; stmt.query_map([], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, i64>(2)?, row.get::<_, i64>(3)?, row.get::<_, String>(4)?, )) })? .collect::>>()? }; let mut applied = 0usize; for (op_id, origin_device_id, seq, hlc_ms, payload_json) in pending { let op = match serde_json::from_str(&payload_json) { Ok(payload) => SyncOpWire { op_id: op_id.clone(), origin_device_id, seq, hlc_ms, payload, }, Err(err) => { tracing::warn!(%op_id, "dropping an unreadable podcast operation: {err}"); lock(&self.conn).execute( "DELETE FROM sync_podcast_pending WHERE op_id = ?1", [&op_id], )?; continue; } }; match self.apply_podcast_op_to(&store, &op) { Ok(()) => { applied += 1; lock(&self.conn).execute( "DELETE FROM sync_podcast_pending WHERE op_id = ?1", [&op_id], )?; } Err(err) => { tracing::warn!(%op_id, "podcast operation still not applied: {err:#}"); } } } if applied > 0 { tracing::info!(applied, "applied podcast operations received earlier"); } self.flush_podcast_changes(); Ok(()) } fn podcast_store(&self) -> Option> { lock(&self.podcasts).store.clone() } /// Merges a received podcast operation into the podcast store, or keeps /// it for [`DeviceSync::attach_podcast_store`]. A podcast store problem /// never fails music sync: the operation stays in the log either way. fn apply_podcast_op(&self, op: &SyncOpWire) -> Result<()> { let store = { let link = lock(&self.podcasts); match &link.store { Some(store) => Arc::clone(store), None => { lock(&self.conn).execute( "INSERT OR IGNORE INTO sync_podcast_pending (op_id) VALUES (?1)", [&op.op_id], )?; return Ok(()); } } }; if let Err(err) = self.apply_podcast_op_to(&store, op) { tracing::warn!(op_id = %op.op_id, "podcast operation not applied yet: {err:#}"); lock(&self.conn).execute( "INSERT OR IGNORE INTO sync_podcast_pending (op_id) VALUES (?1)", [&op.op_id], )?; } Ok(()) } fn apply_podcast_op_to(&self, store: &PodcastStore, op: &SyncOpWire) -> Result<()> { match store.apply_remote_op(op)? { PodcastApplyOutcome::Changed => { let mut link = lock(&self.podcasts); match op.podcast_progress() { Some(record) => link.progress.push(record), None => link.changed = true, } } // Acknowledged anyway so the log keeps moving (FRID contract). PodcastApplyOutcome::Rejected(reason) => tracing::warn!( op_id = %op.op_id, kind = payload_kind(&op.payload), %reason, "skipped an invalid podcast operation" ), _ => {} } Ok(()) } /// Merges the podcast sections of a received snapshot. fn apply_podcast_snapshot(&self, snapshot: &SyncSnapshot) { let Some(store) = self.podcast_store() else { // Snapshots come with every sync; a later one is merged. return; }; match store.apply_snapshot(snapshot) { Ok(stats) => { if stats.ignored_over_limits { tracing::warn!("ignored podcast snapshot sections over the shared limits"); } if stats.rejected > 0 { tracing::warn!( rejected = stats.rejected, "skipped invalid podcast snapshot records" ); } if stats.changed > 0 { lock(&self.podcasts).changed = true; } } Err(err) => tracing::warn!("podcast snapshot not merged: {err:#}"), } } /// Tells the UI about podcast state that received operations or /// snapshots changed (one event per batch). fn flush_podcast_changes(&self) { let (progress, changed) = { let mut link = lock(&self.podcasts); ( std::mem::take(&mut link.progress), std::mem::take(&mut link.changed), ) }; if (changed || !progress.is_empty()) && let Some(tx) = lock(&self.event_tx).as_ref() { let _ = tx.send(AppEvent::PodcastsSynced { progress, changed }); } } pub fn status(&self) -> DeviceSyncStatus { match self.status_inner() { Ok(status) => status, Err(err) => DeviceSyncStatus { last_error: Some(format!("{err:#}")), ..DeviceSyncStatus::default() }, } } pub async fn create_invite(&self, service: Arc) -> Result { let identity = self.ensure_identity()?; let ticket = service.ticket().await?.to_string(); let secret = random_hex(16); let invite_id = random_hex(8); let expires_at_ms = now_ms() + INVITE_TTL_MS; { let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_invites (invite_id, secret_hash, expires_at_ms, created_at_ms) VALUES (?1, ?2, ?3, ?4)", params![invite_id, hash_secret(&secret), expires_at_ms, now_ms()], )?; } let payload = InviteWire { v: 1, ticket, device_id: identity.device_id, invite_id, secret, expires_at_ms, protocol_version: Some(PROTOCOL_VERSION), }; Ok(encode_invite(&payload)?) } pub async fn connect_invite( &self, service: Arc, invite_link: &str, transport_stats: Arc, ) -> Result { let invite = parse_invite(invite_link)?; anyhow::ensure!(invite.expires_at_ms >= now_ms(), "invite expired"); let ticket: PeerTicket = invite .ticket .parse() .map_err(|err| anyhow::anyhow!("malformed invite ticket: {err}"))?; let deadline = (now_ms() + PAIRING_WAIT_MS).min(invite.expires_at_ms); let mut last_error: Option; loop { match self .try_connect_invite( Arc::clone(&service), &invite, ticket.clone(), Arc::clone(&transport_stats), ) .await { Ok(PairAttempt::Accepted(message)) => return Ok(message), Ok(PairAttempt::Pending) => last_error = None, Ok(PairAttempt::Denied(message)) => anyhow::bail!(message), Err(err) => { tracing::debug!("pairing poll failed: {err:#}"); last_error = Some(format!("{err:#}")); } } if now_ms() >= deadline { if let Some(error) = last_error { anyhow::bail!("pairing timed out; last error: {error}"); } anyhow::bail!("pairing timed out"); } tokio::time::sleep(PAIRING_RETRY_DELAY).await; } } async fn try_connect_invite( &self, service: Arc, invite: &InviteWire, ticket: PeerTicket, transport_stats: Arc, ) -> Result { let peer = service.connect(ticket).await?; let own_ticket = service.ticket().await?.to_string(); let identity = self.ensure_identity()?; let group_active_devices = self.active_device_count()?; let profile = self.own_profile(&own_ticket)?; let devices = self.device_profiles()?; let vector = self.vector()?; // The inviter has not reported anything yet but its invite. let peer_version = invite.inviter_protocol_version(); let ops = self.ops_for_peer(&invite.device_id, peer_version)?; let snapshot = self.snapshot_for(peer_version)?; let playback = self.local_playback_snapshot_for(peer_version); let mut stream = service.open_stream(peer, SYNC_ALPN).await?; crate::federation::record_stream_transport( &transport_stats, "device-sync", "outbound", "pair-open", &stream, ); write_msg( &mut stream, &WireMessage::PairRequest { invite_id: invite.invite_id.clone(), secret: invite.secret.clone(), profile, group_id: Some(identity.group_id), group_active_devices, devices, vector, ops, snapshot, playback, }, ) .await?; finish_send(&mut stream).await?; let response = read_msg(&mut stream) .await .context("pairing response was not received")?; crate::federation::record_stream_transport( &transport_stats, "device-sync", "outbound", "pair-done", &stream, ); match response { WireMessage::PairResponse { accepted: true, group_id: Some(group_id), profile, devices, vector, ops, snapshot, playback, .. } => { self.set_group_id(&group_id)?; if let Some(profile) = &profile { self.apply_device_profile( profile, true, VersionReport::SelfReported(Some(profile.protocol_version)), )?; if let Some(playback) = playback { self.apply_playback_snapshot(&profile.device_id, playback)?; } } self.apply_device_profiles(&devices, Some(&invite.device_id))?; self.apply_received(snapshot, ops)?; self.record_local_op(SyncOpPayload::DeviceTrusted { target_device_id: invite.device_id.clone(), })?; self.note_peer_vector(&invite.device_id, &vector)?; self.set_last_sync(Some(format!("paired with {}", short_id(&invite.device_id))))?; self.gc_tombstones()?; Ok(PairAttempt::Accepted(format!( "connected device {}", short_id(&invite.device_id) ))) } WireMessage::PairResponse { accepted: false, pending: true, .. } => Ok(PairAttempt::Pending), WireMessage::PairResponse { accepted: false, error, .. } => Ok(PairAttempt::Denied( error.unwrap_or_else(|| "pairing denied".to_string()), )), _ => anyhow::bail!("unexpected pairing response"), } } pub fn answer_pairing( &self, request_id: &str, accept: bool, use_requester_group: bool, ) -> Result<()> { let pending = { let conn = lock(&self.conn); conn.query_row( "SELECT device_id, name, client_version, endpoint_id, endpoint_ticket, created_at_ms, requester_group_id, requester_group_devices_json, protocol_version FROM sync_pending_pairing WHERE request_id = ?1", [request_id], |row| { Ok(( DeviceProfileWire { device_id: row.get(0)?, name: row.get(1)?, client_version: row.get(2)?, protocol_version: music_dht::device_sync::UNREPORTED_PROTOCOL_VERSION, endpoint_id: row.get(3)?, endpoint_ticket: row.get(4)?, revoked: false, revoke_cutoff_seq: None, updated_at_ms: row.get(5)?, }, row.get::<_, Option>(6)?, row.get::<_, String>(7)?, row.get::<_, Option>(8)? .and_then(|version| u16::try_from(version).ok()), )) }, ) .optional()? }; let changed = { let conn = lock(&self.conn); conn.execute( "UPDATE sync_pending_pairing SET status = ?2, answered_at_ms = ?3, use_requester_group = ?4 WHERE request_id = ?1 AND status = 'pending'", params![ request_id, if accept { "accepted" } else { "denied" }, now_ms(), i64::from(use_requester_group), ], )? }; if accept && changed > 0 { if let Some(( profile, requester_group_id, requester_group_devices_json, requester_version, )) = pending { if use_requester_group { if let Some(group_id) = requester_group_id .as_deref() .filter(|id| !id.trim().is_empty()) { self.set_group_id(group_id)?; } let requester_devices: Vec = serde_json::from_str(&requester_group_devices_json)?; self.apply_device_profiles(&requester_devices, Some(&profile.device_id))?; } // The version the requester reported in its own pairing // request (requests stored by older builds have none). self.apply_device_profile( &profile, true, VersionReport::SelfReported(requester_version), )?; self.record_local_op(SyncOpPayload::DeviceTrusted { target_device_id: profile.device_id, })?; } } Ok(()) } pub fn revoke_device(&self, device_id: &str) -> Result<()> { let own = self.ensure_identity()?.device_id; anyhow::ensure!(device_id != own, "cannot revoke this device from itself"); let cutoff = { let conn = lock(&self.conn); conn.query_row( "SELECT COALESCE(MAX(seq), 0) FROM sync_ops WHERE origin_device_id = ?1", [device_id], |row| row.get::<_, i64>(0), )? }; self.record_local_op(SyncOpPayload::DeviceRevoked { target_device_id: device_id.to_string(), target_max_seq_seen: cutoff, })?; self.gc_tombstones()?; Ok(()) } pub fn record_leave_group_revoke(&self) -> Result { let op = self.record_local_op_generated( |identity, seq| { Ok(SyncOpPayload::DeviceRevoked { target_device_id: identity.device_id.clone(), target_max_seq_seen: seq, }) }, false, )?; Ok(op.op_id) } pub fn cancel_leave_group_revoke(&self, op_id: &str) -> Result<()> { let identity = self.ensure_identity()?; let conn = lock(&self.conn); conn.execute("DELETE FROM sync_ops WHERE op_id = ?1", [op_id])?; conn.execute( "UPDATE sync_devices SET revoked_at_ms = NULL, revoked_by = NULL, revoke_cutoff_seq = NULL WHERE device_id = ?1", [&identity.device_id], )?; Ok(()) } pub fn finish_leave_group_reset(&self) -> Result { let identity = self.ensure_identity()?; let new_group_id = format!( "grp_{}", &blake3::hash(random_hex(32).as_bytes()).to_hex()[..24] ); let now = now_ms(); { let conn = lock(&self.conn); set_meta(&conn, "group_id", &new_group_id)?; set_meta(&conn, "last_sync", "left previous device group")?; delete_meta(&conn, "last_error")?; conn.execute("DELETE FROM sync_invites", [])?; conn.execute("DELETE FROM sync_pending_pairing", [])?; conn.execute("DELETE FROM sync_ops", [])?; conn.execute("DELETE FROM sync_vectors", [])?; conn.execute("DELETE FROM sync_peer_acks", [])?; conn.execute("DELETE FROM sync_compacted", [])?; conn.execute("DELETE FROM sync_playback_applied", [])?; conn.execute( "DELETE FROM sync_devices WHERE device_id != ?1", [&identity.device_id], )?; conn.execute( "INSERT INTO sync_devices (device_id, name, client_version, protocol_version, endpoint_id, endpoint_ticket, trusted_at_ms, last_seen_ms, revoked_at_ms, revoked_by, revoke_cutoff_seq) VALUES (?1, ?2, ?3, ?4, '', '', ?5, ?5, NULL, NULL, NULL) ON CONFLICT(device_id) DO UPDATE SET name = excluded.name, client_version = excluded.client_version, protocol_version = excluded.protocol_version, trusted_at_ms = excluded.trusted_at_ms, last_seen_ms = excluded.last_seen_ms, revoked_at_ms = NULL, revoked_by = NULL, revoke_cutoff_seq = NULL", params![ identity.device_id, identity.name, CLIENT_VERSION, PROTOCOL_VERSION, now, ], )?; let local_seq = get_meta(&conn, "local_seq")? .and_then(|value| value.parse::().ok()) .unwrap_or(0); conn.execute( "INSERT INTO sync_vectors (device_id, max_seq) VALUES (?1, ?2) ON CONFLICT(device_id) DO UPDATE SET max_seq = excluded.max_seq", params![identity.device_id, local_seq], )?; } lock(&self.playback).remote.clear(); Ok(new_group_id) } pub fn record_content_like(&self, content_id: &str, liked: bool) -> Result<()> { let Some(content_id) = music_dht::normalize_content_id(content_id) else { return Ok(()); }; self.record_local_op(SyncOpPayload::TrackLikeSet { content_id, liked, fed: None, })?; Ok(()) } pub fn record_fed_like(&self, fed: &crate::federation::FedTrack, liked: bool) -> Result<()> { let Some(content_id) = fed .content_id .as_deref() .and_then(music_dht::normalize_content_id) else { return Ok(()); }; self.record_local_op(SyncOpPayload::TrackLikeSet { content_id, liked, fed: liked.then(|| SyncedFedTrack::from_fed(fed)).flatten(), })?; Ok(()) } pub fn record_listen(&self, event: ListenEvent) -> Result<()> { if !event.should_record() { return Ok(()); } self.record_local_op(SyncOpPayload::ListenRecorded { event }) } pub fn listen_event_for_track( &self, listen_id: String, track: &TrackItem, started_at_ms: i64, listened_ms: i64, ended_reason: music_dht::device_sync::ListenEndReason, ) -> Option { // Music listening history only: episodes record podcast listening // sessions in the podcast store instead. if track.is_podcast() { return None; } let content_id = track .content_id .as_deref() .or_else(|| track.fed.as_ref()?.content_id.as_deref()) .and_then(music_dht::normalize_content_id)?; Some(ListenEvent { listen_id, content_id, started_at_ms, listened_ms, track_duration_ms: (track.duration_seconds > 0.0) .then_some((track.duration_seconds * 1_000.0).round() as i64), ended_reason, track: ListenTrackMetadata { title: track.title.clone(), artist_names: track .artists .iter() .map(|artist| artist.name.clone()) .collect(), featured_artist_names: track .featured_artists .iter() .map(|artist| artist.name.clone()) .collect(), release_title: (!track.release_title.trim().is_empty()) .then(|| track.release_title.clone()), }, }) } pub fn record_playlist_created(&self, playlist_id: i64, title: &str) -> Result<()> { let playlist_id = self.library.ensure_playlist_sync_id(playlist_id)?; self.record_local_op(SyncOpPayload::PlaylistCreated { playlist_id, title: title.to_string(), })?; Ok(()) } pub fn record_playlist_renamed(&self, playlist_id: i64, title: &str) -> Result<()> { let playlist_id = self.library.ensure_playlist_sync_id(playlist_id)?; self.record_local_op(SyncOpPayload::PlaylistRenamed { playlist_id, title: title.to_string(), })?; Ok(()) } pub fn record_playlist_deleted(&self, playlist_id: i64) -> Result<()> { let Some(playlist_id) = self.library.playlist_sync_id(playlist_id)? else { return Ok(()); }; self.record_local_op(SyncOpPayload::PlaylistDeleted { playlist_id })?; Ok(()) } pub fn record_playlist_tracks_added(&self, playlist_id: i64, track_ids: &[i64]) -> Result<()> { let playlist_sync_id = self.library.ensure_playlist_sync_id(playlist_id)?; for (content_id, position) in self .library .playlist_track_content_positions(playlist_id, track_ids)? { self.record_local_op(SyncOpPayload::PlaylistTrackAdded { playlist_id: playlist_sync_id.clone(), content_id, position, fed: None, })?; } Ok(()) } pub fn record_playlist_fed_tracks_added( &self, playlist_id: i64, tracks: &[crate::federation::FedTrack], ) -> Result<()> { let playlist_sync_id = self.library.ensure_playlist_sync_id(playlist_id)?; for (fallback_position, fed) in tracks.iter().enumerate() { let Some(content_id) = fed .content_id .as_deref() .and_then(music_dht::normalize_content_id) else { continue; }; let position = self .library .playlist_content_position(playlist_id, &content_id)? .unwrap_or(fallback_position as i64); self.record_local_op(SyncOpPayload::PlaylistTrackAdded { playlist_id: playlist_sync_id.clone(), content_id, position, fed: SyncedFedTrack::from_fed(fed), })?; } Ok(()) } pub fn record_playlist_content_removed( &self, playlist_id: i64, content_ids: &[String], ) -> Result<()> { let Some(playlist_id) = self.library.playlist_sync_id(playlist_id)? else { return Ok(()); }; let mut seen = BTreeSet::new(); for content_id in content_ids { let Some(content_id) = music_dht::normalize_content_id(content_id) else { continue; }; if !seen.insert(content_id.clone()) { continue; } self.record_local_op(SyncOpPayload::PlaylistTrackRemoved { playlist_id: playlist_id.clone(), content_id, })?; } Ok(()) } /// Runs a `PodcastStore` local action with stamps reserved in this /// device's sync log and appends the operations it returns with exactly /// those stamps (`op_id`, `seq`, `hlc_ms`). /// /// The log connection stays locked from the first stamp until the /// operations are appended, so no sync batch can read past a reserved /// `seq` before its operation exists. Lock order is podcast store, then /// this log: never call the podcast store while holding `self.conn`. pub fn run_podcast_action( &self, action: impl FnOnce(&mut dyn PodcastStampSource) -> Result<(T, Vec)>, ) -> Result { let identity = self.ensure_identity()?; let mut log = PodcastLogStamps { conn: &self.conn, guard: None, device_id: identity.device_id, reserved: BTreeMap::new(), }; let (value, ops) = action(&mut log)?; log.append(ops)?; Ok(value) } pub async fn sync_once( &self, service: Arc, transport_stats: Arc, ) -> Result<()> { let devices = self.active_remote_devices()?; for device in devices { if device.endpoint_ticket.trim().is_empty() { continue; } if let Err(err) = self .sync_device(Arc::clone(&service), &device, Arc::clone(&transport_stats)) .await { tracing::debug!(device = %device.device_id, "device sync failed: {err:#}"); self.set_last_error(Some(format!("{}: {err:#}", short_id(&device.device_id))))?; } } self.gc_tombstones()?; Ok(()) } async fn sync_device( &self, service: Arc, device: &StoredDevice, transport_stats: Arc, ) -> Result<()> { let peer = resolve_device_peer(&service, device).await?; let own_ticket = service.ticket().await?.to_string(); let identity = self.ensure_identity()?; let profile = self.own_profile(&own_ticket)?; let devices = self.device_profiles()?; let vector = self.vector()?; let peer_version = self.peer_protocol_version(&device.device_id)?; let ops = self.ops_for_peer(&device.device_id, peer_version)?; let snapshot = self.snapshot_for(peer_version)?; let playback = self.local_playback_snapshot_for(peer_version); let mut stream = service.open_stream(peer, SYNC_ALPN).await?; crate::federation::record_stream_transport( &transport_stats, "device-sync", "outbound", "sync-open", &stream, ); write_msg( &mut stream, &WireMessage::Hello { group_id: identity.group_id, profile, devices, vector, ops, snapshot, playback, }, ) .await?; finish_send(&mut stream).await?; let response = read_msg(&mut stream) .await .context("device sync response was not received")?; crate::federation::record_stream_transport( &transport_stats, "device-sync", "outbound", "sync-done", &stream, ); match response { WireMessage::SyncResponse { accepted: true, devices, vector, ops, snapshot, playback, .. } => { // The responder lists itself: that entry is its own report. self.apply_device_profiles(&devices, Some(&device.device_id))?; if let Some(playback) = playback { self.apply_playback_snapshot(&device.device_id, playback)?; } self.apply_received(snapshot, ops)?; self.note_peer_vector(&device.device_id, &vector)?; self.mark_seen(&device.device_id, Some(peer.to_string()))?; self.set_last_sync(Some(format!("synced {}", short_id(&device.device_id))))?; self.set_last_error(None)?; Ok(()) } WireMessage::SyncResponse { accepted: false, error, .. } => anyhow::bail!(error.unwrap_or_else(|| "sync refused".to_string())), _ => anyhow::bail!("unexpected sync response"), } } fn ensure_identity(&self) -> Result { let conn = lock(&self.conn); if let (Some(device_id), Some(group_id), Some(name)) = ( get_meta(&conn, "device_id")?, get_meta(&conn, "group_id")?, get_meta(&conn, "device_name")?, ) { return Ok(LocalIdentity { device_id, group_id, name, }); } let key = SecretKey::generate(); let secret_hex = hex_encode(&key.to_bytes()); let device_id = format!( "dev_{}", &blake3::hash(secret_hex.as_bytes()).to_hex()[..24] ); let group_id = format!("grp_{}", &blake3::hash(device_id.as_bytes()).to_hex()[..24]); let name = default_device_name(&device_id); set_meta(&conn, "device_secret", &secret_hex)?; set_meta(&conn, "device_id", &device_id)?; set_meta(&conn, "group_id", &group_id)?; set_meta(&conn, "device_name", &name)?; set_meta(&conn, "local_seq", "0")?; set_meta(&conn, "last_hlc_ms", "0")?; conn.execute( "INSERT OR IGNORE INTO sync_devices (device_id, name, client_version, protocol_version, endpoint_id, endpoint_ticket, trusted_at_ms, last_seen_ms) VALUES (?1, ?2, ?3, ?4, '', '', ?5, ?5)", params![device_id, name, CLIENT_VERSION, PROTOCOL_VERSION, now_ms()], )?; Ok(LocalIdentity { device_id, group_id, name, }) } fn set_group_id(&self, group_id: &str) -> Result<()> { let conn = lock(&self.conn); set_meta(&conn, "group_id", group_id) } fn own_profile(&self, endpoint_ticket: &str) -> Result { let identity = self.ensure_identity()?; let endpoint_id = ticket_endpoint_id(endpoint_ticket).unwrap_or_default(); let previous_version = { let conn = lock(&self.conn); let previous_version = stored_protocol_version(&conn, &identity.device_id)?; conn.execute( "INSERT INTO sync_devices (device_id, name, client_version, protocol_version, endpoint_id, endpoint_ticket, trusted_at_ms, last_seen_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) ON CONFLICT(device_id) DO UPDATE SET name = excluded.name, client_version = excluded.client_version, protocol_version = excluded.protocol_version, endpoint_id = excluded.endpoint_id, endpoint_ticket = excluded.endpoint_ticket, last_seen_ms = excluded.last_seen_ms", params![ identity.device_id, identity.name, CLIENT_VERSION, PROTOCOL_VERSION, endpoint_id, endpoint_ticket, now_ms(), ], )?; previous_version }; if previous_version.is_some_and(|version| version < PROTOCOL_VERSION) { // First start of a new protocol version: announce it in the log // too, so devices that only hear of this one through relays // learn its version (`profile_protocol_version`). self.record_local_op(SyncOpPayload::DeviceProfileSet { name: identity.name.clone(), client_version: CLIENT_VERSION.to_string(), endpoint_ticket: endpoint_ticket.to_string(), endpoint_id: endpoint_id.clone(), protocol_version: Some(PROTOCOL_VERSION), })?; } Ok(DeviceProfileWire { device_id: identity.device_id, name: identity.name, client_version: CLIENT_VERSION.to_string(), protocol_version: PROTOCOL_VERSION, endpoint_id, endpoint_ticket: endpoint_ticket.to_string(), revoked: false, revoke_cutoff_seq: None, updated_at_ms: now_ms(), }) } fn record_local_op(&self, payload: SyncOpPayload) -> Result<()> { self.record_local_op_generated(|_, _| Ok(payload), true) .map(|_| ()) } fn record_local_op_generated( &self, make_payload: impl FnOnce(&LocalIdentity, i64) -> Result, gc_after_record: bool, ) -> Result { let identity = self.ensure_identity()?; let (op, payload_json, tombstone) = { let conn = lock(&self.conn); let seq = get_meta(&conn, "local_seq")? .and_then(|v| v.parse::().ok()) .unwrap_or(0) + 1; let last_hlc = get_meta(&conn, "last_hlc_ms")? .and_then(|v| v.parse::().ok()) .unwrap_or(0); let hlc_ms = now_ms().max(last_hlc + 1); let op_id = format!("{}:{seq}", identity.device_id); let payload = make_payload(&identity, seq)?; let payload_json = serde_json::to_string(&payload)?; let tombstone = payload.is_tombstone(); conn.execute( "INSERT OR IGNORE INTO sync_ops (op_id, origin_device_id, seq, kind, payload_json, hlc_ms, received_at_ms, tombstone) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ op_id, identity.device_id, seq, payload_kind(&payload), payload_json, hlc_ms, now_ms(), i64::from(tombstone), ], )?; set_meta(&conn, "local_seq", &seq.to_string())?; set_meta(&conn, "last_hlc_ms", &hlc_ms.to_string())?; conn.execute( "INSERT INTO sync_vectors (device_id, max_seq) VALUES (?1, ?2) ON CONFLICT(device_id) DO UPDATE SET max_seq = MAX(max_seq, excluded.max_seq)", params![identity.device_id, seq], )?; ( SyncOpWire { op_id, origin_device_id: identity.device_id, seq, hlc_ms, payload, }, payload_json, tombstone, ) }; let _ = self.apply_op(&op)?; if payload_may_hold_private_urls(&op.payload) { // Feed and audio URLs may carry private tokens, and the Logs tab // keeps this crate's debug output: log the kind only. tracing::debug!( op_id = %op.op_id, tombstone, kind = payload_kind(&op.payload), "recorded personal-sync op" ); } else { tracing::debug!( op_id = %op.op_id, tombstone, payload = %payload_json, "recorded personal-sync op" ); } if gc_after_record { let _ = self.gc_tombstones(); } Ok(op) } fn apply_ops(&self, ops: Vec) -> Result<()> { let mut changed = false; for op in ops { if !self.should_accept_op(&op)? { continue; } let inserted = { let conn = lock(&self.conn); conn.execute( "INSERT OR IGNORE INTO sync_ops (op_id, origin_device_id, seq, kind, payload_json, hlc_ms, received_at_ms, tombstone) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ op.op_id, op.origin_device_id, op.seq, payload_kind(&op.payload), serde_json::to_string(&op.payload)?, op.hlc_ms, now_ms(), i64::from(op.payload.is_tombstone()), ], )? }; if inserted > 0 { changed |= self.apply_op(&op)?; coalesce_progress_ops(&lock(&self.conn), &op)?; } let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_vectors (device_id, max_seq) VALUES (?1, ?2) ON CONFLICT(device_id) DO UPDATE SET max_seq = MAX(max_seq, excluded.max_seq)", params![op.origin_device_id, op.seq], )?; } if changed { self.notify_library_changed(); } self.flush_podcast_changes(); Ok(()) } fn should_accept_op(&self, op: &SyncOpWire) -> Result { let identity = self.ensure_identity()?; if op.origin_device_id == identity.device_id { return Ok(true); } let conn = lock(&self.conn); let revoked: Option<(Option,)> = conn .query_row( "SELECT revoke_cutoff_seq FROM sync_devices WHERE device_id = ?1 AND revoked_at_ms IS NOT NULL", [&op.origin_device_id], |row| Ok((row.get(0)?,)), ) .optional()?; if let Some((cutoff,)) = revoked { return Ok(op.seq <= cutoff.unwrap_or(0)); } let known: Option = conn .query_row( "SELECT 1 FROM sync_devices WHERE device_id = ?1 AND trusted_at_ms IS NOT NULL", [&op.origin_device_id], |row| row.get(0), ) .optional()?; Ok(known.is_some()) } fn apply_op(&self, op: &SyncOpWire) -> Result { let changed = match &op.payload { SyncOpPayload::TrackLikeSet { content_id, liked, fed, } => self.apply_like_state(content_id, *liked, fed.as_ref(), op.hlc_ms, &op.op_id)?, SyncOpPayload::PlaylistCreated { playlist_id, title } => { self.apply_playlist_state(playlist_id, title, false, op.hlc_ms, &op.op_id)? } SyncOpPayload::PlaylistRenamed { playlist_id, title } => { self.apply_playlist_state(playlist_id, title, false, op.hlc_ms, &op.op_id)? } SyncOpPayload::PlaylistDeleted { playlist_id } => { self.apply_playlist_state(playlist_id, "", true, op.hlc_ms, &op.op_id)? } SyncOpPayload::PlaylistTrackAdded { playlist_id, content_id, position, fed, } => self.apply_playlist_item_state( playlist_id, content_id, true, *position, fed.as_ref(), op.hlc_ms, &op.op_id, )?, SyncOpPayload::PlaylistTrackRemoved { playlist_id, content_id, } => self.apply_playlist_item_state( playlist_id, content_id, false, 0, None, op.hlc_ms, &op.op_id, )?, SyncOpPayload::DeviceProfileSet { name, client_version, endpoint_ticket, endpoint_id, protocol_version, } => { let profile = DeviceProfileWire { device_id: op.origin_device_id.clone(), name: name.clone(), client_version: client_version.clone(), protocol_version: protocol_version .unwrap_or(music_dht::device_sync::UNREPORTED_PROTOCOL_VERSION), endpoint_id: endpoint_id.clone(), endpoint_ticket: endpoint_ticket.clone(), revoked: false, revoke_cutoff_seq: None, updated_at_ms: op.hlc_ms, }; // The origin announces its own profile; profiles before v4 // carry no version and keep the stored one. self.apply_device_profile( &profile, false, VersionReport::SelfReported(*protocol_version), )?; false } SyncOpPayload::DeviceTrusted { target_device_id } => { self.apply_device_trusted(target_device_id, op.hlc_ms)? } SyncOpPayload::DeviceRevoked { target_device_id, target_max_seq_seen, } => self.apply_device_revoked( target_device_id, op.hlc_ms, &op.origin_device_id, *target_max_seq_seen, )?, SyncOpPayload::PlaybackCommand { target_device_id, command, authority, } => { self.apply_playback_command( target_device_id, command, authority.as_ref(), &op.origin_device_id, &op.op_id, )?; false } SyncOpPayload::ListenRecorded { event } => self .library .apply_listen_event(event, &op.origin_device_id)?, // Podcast operations (protocol v4) stay in the log, so they are // relayed, and are merged into the podcast store. SyncOpPayload::PodcastSubscriptionSet { .. } | SyncOpPayload::PodcastShareSet { .. } | SyncOpPayload::PodcastFeedAliasesAdded { .. } | SyncOpPayload::PodcastProgressSet { .. } | SyncOpPayload::PodcastListenRecorded { .. } => { self.apply_podcast_op(op)?; false } }; Ok(changed) } fn apply_playback_command( &self, target_device_id: &str, command: &PlaybackCommand, authority: Option<&CommandStamp>, origin: &str, op_id: &str, ) -> Result<()> { let identity = self.ensure_identity()?; // Legacy commands still replicate with the library log, but cannot // override the versioned personal-playback protocol. let Some(authority) = authority else { return Ok(()); }; // A peer below v4 does not know podcast items: while this queue has // any, its state would silently drop them. Checked before the // ownership engine sees the command. let peer_version = self.peer_protocol_version(origin)?; let local = lock(&self.playback) .local .as_ref() .map(|snapshot| snapshot.state.clone()); if let Some(local) = local && !music_dht::device_sync::playback_command_allowed_from_peer(&local, peer_version) { tracing::info!( origin = %short_id(origin), peer_version, "ignored a playback command from a device without podcast support" ); if let Some(tx) = lock(&self.event_tx).as_ref() { let _ = tx.send(AppEvent::StatusMessage(format!( "ignored playback control from {}: it needs a Furumi update to keep the \ podcast episodes in this queue", self.device_label(origin) ))); } return Ok(()); } let handoff = matches!(command, PlaybackCommand::ActiveChanged { .. }); if !handoff && target_device_id != identity.device_id { return Ok(()); } if let PlaybackCommand::ActiveChanged { active_device_id, .. } = command && active_device_id != &authority.claim.owner { return Ok(()); } let accepted = self.with_playback_engine(|engine| { engine.accept_command(origin, authority, handoff, playback_clock()) && (handoff || engine.is_owner()) })?; if !accepted || target_device_id != identity.device_id { return Ok(()); } let inserted = { let conn = lock(&self.conn); conn.execute( "INSERT OR IGNORE INTO sync_playback_applied (op_id, applied_at_ms) VALUES (?1, ?2)", params![op_id, now_ms()], )? }; if inserted == 0 { return Ok(()); } if let Some(tx) = lock(&self.event_tx).as_ref() { let _ = tx.send(AppEvent::PlaybackCommand { command: command.clone(), authority: authority.clone(), origin: origin.to_string(), }); } Ok(()) } fn apply_device_trusted(&self, target_device_id: &str, hlc_ms: i64) -> Result { let was_revoked = { let conn = lock(&self.conn); conn.query_row( "SELECT revoked_at_ms IS NOT NULL FROM sync_devices WHERE device_id = ?1", [target_device_id], |row| row.get::<_, i64>(0), ) .optional()? .unwrap_or(0) != 0 }; let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_devices (device_id, trusted_at_ms, last_seen_ms) VALUES (?1, ?2, ?2) ON CONFLICT(device_id) DO UPDATE SET trusted_at_ms = MAX(COALESCE(sync_devices.trusted_at_ms, 0), excluded.trusted_at_ms), last_seen_ms = MAX(COALESCE(sync_devices.last_seen_ms, 0), excluded.last_seen_ms), revoked_at_ms = CASE WHEN sync_devices.revoked_at_ms IS NOT NULL AND sync_devices.revoked_at_ms <= excluded.trusted_at_ms THEN NULL ELSE sync_devices.revoked_at_ms END, revoked_by = CASE WHEN sync_devices.revoked_at_ms IS NOT NULL AND sync_devices.revoked_at_ms <= excluded.trusted_at_ms THEN NULL ELSE sync_devices.revoked_by END, revoke_cutoff_seq = CASE WHEN sync_devices.revoked_at_ms IS NOT NULL AND sync_devices.revoked_at_ms <= excluded.trusted_at_ms THEN NULL ELSE sync_devices.revoke_cutoff_seq END", params![target_device_id, hlc_ms], )?; Ok(was_revoked) } fn apply_device_revoked( &self, target_device_id: &str, hlc_ms: i64, revoked_by: &str, target_max_seq_seen: i64, ) -> Result { let was_active = { let conn = lock(&self.conn); conn.query_row( "SELECT trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL FROM sync_devices WHERE device_id = ?1", [target_device_id], |row| row.get::<_, i64>(0), ) .optional()? .unwrap_or(0) != 0 }; let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_devices (device_id, revoked_at_ms, revoked_by, revoke_cutoff_seq) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(device_id) DO UPDATE SET revoked_at_ms = CASE WHEN COALESCE(sync_devices.trusted_at_ms, 0) <= excluded.revoked_at_ms AND COALESCE(sync_devices.revoked_at_ms, 0) <= excluded.revoked_at_ms THEN excluded.revoked_at_ms ELSE sync_devices.revoked_at_ms END, revoked_by = CASE WHEN COALESCE(sync_devices.trusted_at_ms, 0) <= excluded.revoked_at_ms AND COALESCE(sync_devices.revoked_at_ms, 0) <= excluded.revoked_at_ms THEN excluded.revoked_by ELSE sync_devices.revoked_by END, revoke_cutoff_seq = CASE WHEN COALESCE(sync_devices.trusted_at_ms, 0) <= excluded.revoked_at_ms AND COALESCE(sync_devices.revoked_at_ms, 0) <= excluded.revoked_at_ms THEN excluded.revoke_cutoff_seq ELSE sync_devices.revoke_cutoff_seq END", params![target_device_id, hlc_ms, revoked_by, target_max_seq_seen,], )?; Ok(was_active) } fn apply_like_state( &self, content_id: &str, liked: bool, fed: Option<&SyncedFedTrack>, hlc_ms: i64, op_id: &str, ) -> Result { let Some(content_id) = music_dht::normalize_content_id(content_id) else { return Ok(false); }; let current = { let conn = lock(&self.conn); conn.query_row( "SELECT liked, hlc_ms, op_id FROM sync_state_likes WHERE content_id = ?1", [&content_id], |row| { Ok(( row.get::<_, i64>(0)? != 0, row.get::<_, i64>(1)?, row.get::<_, String>(2)?, )) }, ) .optional()? }; let apply = current.as_ref().is_none_or(|(_, current_hlc, current_op)| { (hlc_ms, op_id) > (*current_hlc, current_op.as_str()) }); if apply { let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_state_likes (content_id, liked, hlc_ms, op_id) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(content_id) DO UPDATE SET liked = excluded.liked, hlc_ms = excluded.hlc_ms, op_id = excluded.op_id", params![content_id, i64::from(liked), hlc_ms, op_id], )?; } let effective_liked = if apply { liked } else { current .as_ref() .map(|(liked, _, _)| *liked) .unwrap_or(false) }; let effective_hlc_ms = if apply { hlc_ms } else { current .as_ref() .map(|(_, current_hlc, _)| *current_hlc) .unwrap_or(hlc_ms) }; let mut changed = apply; if let Some(track_id) = self.library.track_id_by_content_id(&content_id)? { changed |= self .library .set_synced_like(track_id, effective_liked, effective_hlc_ms)?; if effective_liked { changed |= self.library.remove_fed_like_by_content_id(&content_id)?; } } else if effective_liked { if let Some(fed) = fed { changed |= self .library .upsert_synced_fed_like(&fed.to_fed_track(), effective_hlc_ms)?; } } else if apply { changed |= self.library.remove_fed_like_by_content_id(&content_id)?; } Ok(changed) } fn repair_like_order_from_sync_state(&self) -> Result<()> { let rows = { let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT content_id, hlc_ms FROM sync_state_likes WHERE liked = 1", )?; stmt.query_map([], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) })? .collect::>>()? }; for (content_id, hlc_ms) in rows { if let Some(track_id) = self.library.track_id_by_content_id(&content_id)? { let _ = self.library.set_synced_like(track_id, true, hlc_ms)?; } else if let Some(fed) = self.library.fed_like_by_content_id(&content_id)? { let _ = self.library.upsert_synced_fed_like(&fed, hlc_ms)?; } } Ok(()) } fn apply_playlist_state( &self, playlist_id: &str, title: &str, deleted: bool, hlc_ms: i64, op_id: &str, ) -> Result { let apply = { let conn = lock(&self.conn); let current: Option<(i64, String)> = conn .query_row( "SELECT hlc_ms, op_id FROM sync_state_playlists WHERE playlist_id = ?1", [playlist_id], |row| Ok((row.get(0)?, row.get(1)?)), ) .optional()?; current.as_ref().is_none_or(|(current_hlc, current_op)| { (hlc_ms, op_id) > (*current_hlc, current_op.as_str()) }) }; if !apply { return Ok(false); } { let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_state_playlists (playlist_id, title, deleted, hlc_ms, op_id) VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT(playlist_id) DO UPDATE SET title = excluded.title, deleted = excluded.deleted, hlc_ms = excluded.hlc_ms, op_id = excluded.op_id", params![playlist_id, title, i64::from(deleted), hlc_ms, op_id], )?; } if deleted { self.library.delete_playlist_by_sync_id(playlist_id)?; } else if !title.trim().is_empty() { self.library.upsert_synced_playlist(playlist_id, title)?; } Ok(true) } fn apply_playlist_item_state( &self, playlist_id: &str, content_id: &str, present: bool, position: i64, fed: Option<&SyncedFedTrack>, hlc_ms: i64, op_id: &str, ) -> Result { let current = { let conn = lock(&self.conn); conn.query_row( "SELECT present, position, hlc_ms, op_id FROM sync_state_playlist_items WHERE playlist_id = ?1 AND content_id = ?2", params![playlist_id, content_id], |row| { Ok(( row.get::<_, i64>(0)? != 0, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?, row.get::<_, String>(3)?, )) }, ) .optional()? }; let apply = current .as_ref() .is_none_or(|(_, _, current_hlc, current_op)| { (hlc_ms, op_id) > (*current_hlc, current_op.as_str()) }); if apply { let conn = lock(&self.conn); conn.execute( "INSERT INTO sync_state_playlist_items (playlist_id, content_id, present, position, hlc_ms, op_id) VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT(playlist_id, content_id) DO UPDATE SET present = excluded.present, position = excluded.position, hlc_ms = excluded.hlc_ms, op_id = excluded.op_id", params![ playlist_id, content_id, i64::from(present), position, hlc_ms, op_id ], )?; } let (effective_present, effective_position) = if apply { (present, position) } else { current .as_ref() .map(|(present, position, _, _)| (*present, *position)) .unwrap_or((present, position)) }; let mut visible_changed = apply; if effective_present { visible_changed |= self .library .add_content_id_to_synced_playlist(playlist_id, content_id)?; if let Some(fed) = fed { visible_changed |= self.library.upsert_fed_playlist_track( playlist_id, &fed.to_fed_track(), effective_position, )?; } else if let Some(fed) = self.library.fed_like_by_content_id(content_id)? { visible_changed |= self.library.upsert_fed_playlist_track( playlist_id, &fed, effective_position, )?; } } else if apply { self.library .remove_content_id_from_synced_playlist(playlist_id, content_id)?; } Ok(visible_changed) } /// Applies what a peer sent: the music snapshot, the operations, then /// the podcast snapshot sections. Podcast merges are order-independent; /// operations go first so a changed progress register reaches the UI as /// such (it is shown in place) instead of as a generic change. fn apply_received(&self, mut snapshot: SyncSnapshot, ops: Vec) -> Result<()> { let podcast = SyncSnapshot { podcast_subscriptions: std::mem::take(&mut snapshot.podcast_subscriptions), podcast_shares: std::mem::take(&mut snapshot.podcast_shares), podcast_progress: std::mem::take(&mut snapshot.podcast_progress), podcast_listen_sessions: std::mem::take(&mut snapshot.podcast_listen_sessions), ..SyncSnapshot::default() }; self.apply_snapshot(snapshot)?; self.apply_ops(ops)?; if podcast.has_podcast_state() { self.apply_podcast_snapshot(&podcast); self.flush_podcast_changes(); } Ok(()) } /// Applies the music sections of a snapshot. fn apply_snapshot(&self, snapshot: SyncSnapshot) -> Result<()> { let mut changed = false; for like in snapshot.likes { changed |= self.apply_like_state( &like.content_id, true, like.fed.as_ref(), like.hlc_ms, &like.op_id, )?; } for like in snapshot.unlikes { changed |= self.apply_like_state(&like.content_id, false, None, like.hlc_ms, &like.op_id)?; } for playlist in snapshot.playlists { changed |= self.apply_playlist_state( &playlist.playlist_id, &playlist.title, false, playlist.hlc_ms, &playlist.op_id, )?; for item in playlist.items { changed |= self.apply_playlist_item_state( &playlist.playlist_id, &item.content_id, true, item.position, item.fed.as_ref(), item.hlc_ms, &item.op_id, )?; } } for playlist in snapshot.deleted_playlists { changed |= self.apply_playlist_state( &playlist.playlist_id, "", true, playlist.hlc_ms, &playlist.op_id, )?; } for item in snapshot.removed_playlist_items { changed |= self.apply_playlist_item_state( &item.playlist_id, &item.content_id, false, 0, None, item.hlc_ms, &item.op_id, )?; } if changed { self.notify_library_changed(); } Ok(()) } /// Stores the profiles of a `devices` list sent by `sender`. The entry /// of the sender itself is its own version report; the others are /// relayed. fn apply_device_profiles( &self, devices: &[DeviceProfileWire], sender: Option<&str>, ) -> Result<()> { for profile in devices { self.apply_device_profile(profile, false, VersionReport::for_listed(profile, sender))?; } Ok(()) } fn apply_device_profile( &self, profile: &DeviceProfileWire, trusted: bool, version: VersionReport, ) -> Result<()> { let own = self.ensure_identity()?.device_id; if profile.device_id == own { return Ok(()); } let trusted_at_ms = if trusted { now_ms() } else { profile.updated_at_ms }; { let conn = lock(&self.conn); let protocol_version = version.resolve(stored_protocol_version(&conn, &profile.device_id)?); conn.execute( "INSERT INTO sync_devices (device_id, name, client_version, protocol_version, endpoint_id, endpoint_ticket, trusted_at_ms, last_seen_ms, revoked_at_ms, revoke_cutoff_seq) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) ON CONFLICT(device_id) DO UPDATE SET name = excluded.name, client_version = excluded.client_version, protocol_version = excluded.protocol_version, endpoint_id = excluded.endpoint_id, endpoint_ticket = CASE WHEN excluded.endpoint_ticket != '' THEN excluded.endpoint_ticket ELSE sync_devices.endpoint_ticket END, trusted_at_ms = MAX(COALESCE(sync_devices.trusted_at_ms, 0), excluded.trusted_at_ms), last_seen_ms = COALESCE(excluded.last_seen_ms, sync_devices.last_seen_ms), revoked_at_ms = CASE WHEN excluded.revoked_at_ms IS NOT NULL AND COALESCE(sync_devices.trusted_at_ms, 0) <= excluded.revoked_at_ms AND COALESCE(sync_devices.revoked_at_ms, 0) <= excluded.revoked_at_ms THEN excluded.revoked_at_ms ELSE sync_devices.revoked_at_ms END, revoke_cutoff_seq = CASE WHEN excluded.revoked_at_ms IS NOT NULL AND COALESCE(sync_devices.trusted_at_ms, 0) <= excluded.revoked_at_ms AND COALESCE(sync_devices.revoked_at_ms, 0) <= excluded.revoked_at_ms THEN excluded.revoke_cutoff_seq ELSE sync_devices.revoke_cutoff_seq END", params![ profile.device_id, profile.name, profile.client_version, protocol_version, profile.endpoint_id, profile.endpoint_ticket, trusted_at_ms, profile.updated_at_ms, if profile.revoked { Some(profile.updated_at_ms) } else { None }, profile.revoke_cutoff_seq, ], )?; } if trusted { let _ = self.apply_device_trusted(&profile.device_id, trusted_at_ms)?; } Ok(()) } fn mark_seen(&self, device_id: &str, endpoint_id: Option) -> Result<()> { let conn = lock(&self.conn); conn.execute( "UPDATE sync_devices SET last_seen_ms = ?2, endpoint_id = COALESCE(?3, endpoint_id) WHERE device_id = ?1", params![device_id, now_ms(), endpoint_id], )?; Ok(()) } fn vector(&self) -> Result> { let conn = lock(&self.conn); let mut stmt = conn.prepare("SELECT device_id, max_seq FROM sync_vectors")?; let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?; Ok(rows.collect::>>()?) } fn note_peer_vector(&self, peer_device_id: &str, vector: &BTreeMap) -> Result<()> { let conn = lock(&self.conn); for (origin, seq) in vector { conn.execute( "INSERT INTO sync_peer_acks (peer_device_id, origin_device_id, max_seq, updated_at_ms) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(peer_device_id, origin_device_id) DO UPDATE SET max_seq = MAX(max_seq, excluded.max_seq), updated_at_ms = excluded.updated_at_ms", params![peer_device_id, origin, seq, now_ms()], )?; } Ok(()) } /// Operations the peer has not acknowledged and can parse. /// /// Kinds the peer's version does not know are excluded in SQL, so a /// batch of hidden operations cannot stall the per-peer cursor; /// `op_allowed_for_peer` stays the final check (playback commands that /// carry podcast items). Hidden operations are never re-sent after the /// peer upgrades (vectors only grow): snapshots are the recovery path. fn ops_for_peer(&self, peer_device_id: &str, peer_version: u16) -> Result> { // Every peer of `furumi/sync/2` keeps listen history, even when it // never reported a version (an invite without one, a device known // only from `device_trusted`). Listen events are not part of // snapshots, so skipping them would lose them for good. let peer_version = peer_version.max(SYNC_V2_PROTOCOL_VERSION); let hide_podcasts = !music_dht::device_sync::supports_podcasts(peer_version); let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json FROM sync_ops o LEFT JOIN sync_peer_acks a ON a.peer_device_id = ?1 AND a.origin_device_id = o.origin_device_id WHERE o.seq > COALESCE(a.max_seq, 0) AND NOT (?4 AND substr(o.kind, 1, 8) = 'podcast_') AND ( o.kind != 'playback_command' OR ( o.hlc_ms >= ?2 AND json_extract(o.payload_json, '$.target_device_id') = ?1 ) ) ORDER BY o.hlc_ms, o.op_id LIMIT ?3", )?; let rows = stmt.query_map( params![ peer_device_id, now_ms().saturating_sub(PLAYBACK_COMMAND_TTL_MS), MAX_OPS_PER_BATCH as i64, hide_podcasts, ], |row| { let payload_json: String = row.get(4)?; Ok(SyncOpWire { op_id: row.get(0)?, origin_device_id: row.get(1)?, seq: row.get(2)?, hlc_ms: row.get(3)?, payload: serde_json::from_str(&payload_json).map_err(|err| { rusqlite::Error::FromSqlConversionFailure( 4, rusqlite::types::Type::Text, Box::new(err), ) })?, }) }, )?; let mut ops = rows.collect::>>()?; ops.retain(|op| music_dht::device_sync::op_allowed_for_peer(&op.payload, peer_version)); Ok(ops) } /// The materialized snapshot for a peer: podcast sections (bounded by /// the shared snapshot limits) only for v4 peers. fn snapshot_for(&self, peer_version: u16) -> Result { let mut snapshot = self.snapshot()?; if music_dht::device_sync::supports_podcasts(peer_version) && let Some(store) = self.podcast_store() && let Err(err) = store.fill_snapshot( &mut snapshot, music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES, ) { // Music sync goes on; podcast state comes with a later sync. tracing::warn!("podcast snapshot export failed: {err:#}"); snapshot.strip_podcast_state(); } Ok(snapshot.for_peer(peer_version)) } /// Music sections of the materialized snapshot. fn snapshot(&self) -> Result { let like_rows = { let conn = lock(&self.conn); let mut likes_stmt = conn.prepare( "SELECT content_id, hlc_ms, op_id FROM sync_state_likes WHERE liked = 1 ORDER BY content_id", )?; likes_stmt .query_map([], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, i64>(1)?, row.get::<_, String>(2)?, )) })? .collect::>>()? }; let mut likes = Vec::with_capacity(like_rows.len()); for (content_id, hlc_ms, op_id) in like_rows { let fed = self .library .fed_like_by_content_id(&content_id)? .as_ref() .and_then(SyncedFedTrack::from_fed); likes.push(SnapshotLike { content_id, hlc_ms, op_id, fed, }); } let unlikes = { let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT content_id, hlc_ms, op_id FROM sync_state_likes WHERE liked = 0 ORDER BY content_id", )?; stmt.query_map([], |row| { Ok(SnapshotLikeTombstone { content_id: row.get(0)?, hlc_ms: row.get(1)?, op_id: row.get(2)?, }) })? .collect::>>()? }; let conn = lock(&self.conn); let mut playlist_stmt = conn.prepare( "SELECT playlist_id, title, hlc_ms, op_id FROM sync_state_playlists WHERE deleted = 0 ORDER BY title COLLATE NOCASE", )?; let playlist_rows = playlist_stmt .query_map([], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, i64>(2)?, row.get::<_, String>(3)?, )) })? .collect::>>()?; let mut playlists = Vec::new(); for (playlist_id, title, hlc_ms, op_id) in playlist_rows { let mut item_stmt = conn.prepare( "SELECT content_id, position, hlc_ms, op_id FROM sync_state_playlist_items WHERE playlist_id = ?1 AND present = 1 ORDER BY position, content_id", )?; let item_rows = item_stmt .query_map([&playlist_id], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?, row.get::<_, String>(3)?, )) })? .collect::>>()?; let mut items = Vec::new(); for (content_id, position, hlc_ms, op_id) in item_rows { let fed_track = match self .library .fed_playlist_track_by_content_id(&playlist_id, &content_id)? { Some(fed) => Some(fed), None => self.library.fed_like_by_content_id(&content_id)?, }; let fed = fed_track.as_ref().and_then(SyncedFedTrack::from_fed); items.push(SnapshotPlaylistItem { content_id, position, hlc_ms, op_id, fed, }); } playlists.push(SnapshotPlaylist { playlist_id, title, hlc_ms, op_id, items, }); } let deleted_playlists = { let mut stmt = conn.prepare( "SELECT playlist_id, hlc_ms, op_id FROM sync_state_playlists WHERE deleted = 1 ORDER BY playlist_id", )?; stmt.query_map([], |row| { Ok(SnapshotPlaylistTombstone { playlist_id: row.get(0)?, hlc_ms: row.get(1)?, op_id: row.get(2)?, }) })? .collect::>>()? }; let removed_playlist_items = { let mut stmt = conn.prepare( "SELECT playlist_id, content_id, hlc_ms, op_id FROM sync_state_playlist_items WHERE present = 0 ORDER BY playlist_id, content_id", )?; stmt.query_map([], |row| { Ok(SnapshotPlaylistItemTombstone { playlist_id: row.get(0)?, content_id: row.get(1)?, hlc_ms: row.get(2)?, op_id: row.get(3)?, }) })? .collect::>>()? }; Ok(SyncSnapshot { likes, unlikes, playlists, deleted_playlists, removed_playlist_items, ..SyncSnapshot::default() }) } fn device_profiles(&self) -> Result> { let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT device_id, name, client_version, protocol_version, endpoint_id, endpoint_ticket, revoked_at_ms IS NOT NULL, revoke_cutoff_seq, COALESCE(last_seen_ms, trusted_at_ms, 0) FROM sync_devices WHERE trusted_at_ms IS NOT NULL ORDER BY device_id", )?; let rows = stmt.query_map([], |row| { Ok(DeviceProfileWire { device_id: row.get(0)?, name: row.get(1)?, client_version: row.get(2)?, protocol_version: row.get(3)?, endpoint_id: row.get(4)?, endpoint_ticket: row.get(5)?, revoked: row.get::<_, i64>(6)? != 0, revoke_cutoff_seq: row.get(7)?, updated_at_ms: row.get(8)?, }) })?; Ok(rows.collect::>>()?) } fn active_remote_devices(&self) -> Result> { let own = self.ensure_identity()?.device_id; let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT device_id, endpoint_id, endpoint_ticket FROM sync_devices WHERE trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL AND device_id != ?1 ORDER BY last_seen_ms DESC", )?; let rows = stmt.query_map([own], |row| { Ok(StoredDevice { device_id: row.get(0)?, endpoint_id: row.get(1)?, endpoint_ticket: row.get(2)?, }) })?; Ok(rows.collect::>>()?) } pub fn active_remote_endpoint_ids(&self) -> Result> { let own = self.ensure_identity()?.device_id; let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT endpoint_id FROM sync_devices WHERE trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL AND device_id != ?1 AND endpoint_id != '' ORDER BY last_seen_ms DESC", )?; let rows = stmt.query_map([own], |row| row.get::<_, String>(0))?; Ok(rows.collect::>>()?) } fn active_device_count(&self) -> Result { let conn = lock(&self.conn); Ok(conn.query_row( "SELECT COUNT(*) FROM sync_devices WHERE trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL", [], |row| row.get::<_, i64>(0), )? as usize) } fn status_inner(&self) -> Result { let identity = self.ensure_identity()?; let conn = lock(&self.conn); let active_devices: usize = conn.query_row( "SELECT COUNT(*) FROM sync_devices WHERE trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL", [], |row| row.get::<_, i64>(0), )? as usize; let revoked_devices: usize = conn.query_row( "SELECT COUNT(*) FROM sync_devices WHERE revoked_at_ms IS NOT NULL", [], |row| row.get::<_, i64>(0), )? as usize; let pending_requests: usize = conn.query_row( "SELECT COUNT(*) FROM sync_pending_pairing WHERE status = 'pending'", [], |row| row.get::<_, i64>(0), )? as usize; let ops_total: usize = conn.query_row("SELECT COUNT(*) FROM sync_ops", [], |row| { row.get::<_, i64>(0) })? as usize; let tombstone_ops: usize = conn.query_row( "SELECT COUNT(*) FROM sync_ops WHERE tombstone = 1", [], |row| row.get::<_, i64>(0), )? as usize; let compactable_tombstones = self.compactable_tombstone_count(&conn)?; let outbox_ops: usize = conn.query_row( "SELECT COUNT(*) FROM sync_ops o WHERE EXISTS ( SELECT 1 FROM sync_devices d LEFT JOIN sync_peer_acks a ON a.peer_device_id = d.device_id AND a.origin_device_id = o.origin_device_id WHERE d.trusted_at_ms IS NOT NULL AND d.revoked_at_ms IS NULL AND d.device_id != ?1 AND o.seq > COALESCE(a.max_seq, 0) )", [&identity.device_id], |row| row.get::<_, i64>(0), )? as usize; let snapshot_likes: usize = conn.query_row( "SELECT COUNT(*) FROM sync_state_likes WHERE liked = 1", [], |row| row.get::<_, i64>(0), )? as usize; let snapshot_playlists: usize = conn.query_row( "SELECT COUNT(*) FROM sync_state_playlists WHERE deleted = 0", [], |row| row.get::<_, i64>(0), )? as usize; let snapshot_items: usize = conn.query_row( "SELECT COUNT(*) FROM sync_state_playlist_items WHERE present = 1", [], |row| row.get::<_, i64>(0), )? as usize; let unresolved_playlist_items = self.unresolved_playlist_item_count_with_conn(&conn)? as usize; let peer_ack_floor = peer_ack_floor_label(&conn)?; let last_sync = get_meta(&conn, "last_sync")?; let last_error = get_meta(&conn, "last_error")?; let mut stmt = conn.prepare( "SELECT device_id, name, client_version, endpoint_id, last_seen_ms, revoked_at_ms IS NOT NULL, protocol_version FROM sync_devices WHERE trusted_at_ms IS NOT NULL ORDER BY revoked_at_ms IS NOT NULL, name COLLATE NOCASE, device_id", )?; let devices = stmt .query_map([], |row| { let device_id: String = row.get(0)?; Ok(DeviceStatusRow { is_self: device_id == identity.device_id, device_id, name: row.get(1)?, client_version: row.get(2)?, endpoint_id: row.get(3)?, last_seen_ms: row.get(4)?, revoked: row.get::<_, i64>(5)? != 0, protocol_version: u16::try_from(row.get::<_, i64>(6)?) .unwrap_or(music_dht::device_sync::UNREPORTED_PROTOCOL_VERSION), }) })? .collect::>>()?; Ok(DeviceSyncStatus { this_device_id: identity.device_id, this_device_name: identity.name, group_id: identity.group_id, active_devices, revoked_devices, pending_requests, ops_total, tombstone_ops, compactable_tombstones, outbox_ops, snapshot_likes, snapshot_playlists, snapshot_items, unresolved_playlist_items, peer_ack_floor, last_sync, last_error, devices, }) } fn compactable_tombstone_count(&self, conn: &Connection) -> Result { let rows = compactable_tombstone_ids(conn)?; Ok(rows.len()) } fn unresolved_playlist_item_count_with_conn(&self, conn: &Connection) -> Result { let mut stmt = conn.prepare( "SELECT playlist_id, content_id FROM sync_state_playlist_items WHERE present = 1", )?; let ids = stmt .query_map([], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) })? .collect::>>()?; drop(stmt); let mut unresolved = 0; for (playlist_id, content_id) in ids { if !self .library .has_playlist_content_reference(&playlist_id, &content_id)? { unresolved += 1; } } Ok(unresolved) } fn set_last_sync(&self, message: Option) -> Result<()> { let conn = lock(&self.conn); match message { Some(message) => set_meta(&conn, "last_sync", &format!("{} · {message}", now_label())), None => delete_meta(&conn, "last_sync"), } } fn set_last_error(&self, message: Option) -> Result<()> { let conn = lock(&self.conn); match message { Some(message) => set_meta(&conn, "last_error", &message), None => delete_meta(&conn, "last_error"), } } /// Refuses to hand a queue with podcast items to a device below v4 /// ("that device needs an update"), before ownership changes. pub fn ensure_playback_supported_by( &self, device_id: &str, state: &PlaybackStateWire, ) -> Result<()> { let version = self.peer_protocol_version(device_id)?; anyhow::ensure!( music_dht::device_sync::playback_allowed_for_peer(state, version), "{} needs a Furumi update to play podcast episodes (device sync v{}); update \ Furumi there, or remove the episodes from the queue", self.device_label(device_id), music_dht::device_sync::PODCAST_PROTOCOL_VERSION ); Ok(()) } /// Display name of a device, or a short id. fn device_label(&self, device_id: &str) -> String { lock(&self.conn) .query_row( "SELECT name FROM sync_devices WHERE device_id = ?1", [device_id], |row| row.get::<_, String>(0), ) .optional() .ok() .flatten() .filter(|name| !name.trim().is_empty()) .unwrap_or_else(|| short_id(device_id)) } /// The playback snapshot for a peer. /// /// A peer below v4 cannot carry a queue with podcast items /// (`playback_allowed_for_peer`), but it still needs the coordination /// envelope: an engine renews the owner's liveness only from the /// owner's own announcements, so an old device that hears nothing takes /// playback over by automatic failover after `owner_timeout_ms` (two /// minutes by default) and this device would then adopt its queue, /// losing the episodes. Such a peer therefore gets the announcement with /// the queue redacted like for Jam (`redact_for_jam`: every episode /// becomes an opaque "Private item", order and transport kept). The /// episodes are never stripped, and the commands the peer derives from /// that state are refused (`playback_command_allowed_from_peer`). /// FRID's PLAYBACK_PROTOCOL.md rule 1 still says "send `None`"; this is /// reported as a contract gap. fn local_playback_snapshot_for(&self, peer_version: u16) -> Option { let mut snapshot = lock(&self.playback).local.clone()?; if !music_dht::device_sync::playback_allowed_for_peer(&snapshot.state, peer_version) { snapshot.state = music_dht::device_sync::redact_for_jam(snapshot.state); } Some(snapshot) } fn apply_playback_snapshot(&self, sender: &str, snapshot: PlaybackSnapshot) -> Result<()> { let identity = self.ensure_identity()?; if snapshot.device_id != sender || snapshot.device_id == identity.device_id { return Ok(()); } let Some(coordination) = &snapshot.coordination else { return Ok(()); }; if !self.with_playback_engine(|engine| { engine.observe(&snapshot.device_id, coordination, playback_clock()) })? { return Ok(()); } let should_send = { let mut shared = lock(&self.playback); let changed = shared.remote.get(&snapshot.device_id).is_none_or(|old| { old.coordination.as_ref().is_none_or(|c| { coordination.claim > c.claim || (coordination.claim == c.claim && coordination.heartbeat > c.heartbeat) }) }); if changed { shared .remote .insert(snapshot.device_id.clone(), snapshot.clone()); } changed }; if should_send && let Some(tx) = lock(&self.event_tx).as_ref() { let _ = tx.send(AppEvent::DevicePlayback(snapshot)); } Ok(()) } fn notify_library_changed(&self) { if let Some(tx) = lock(&self.event_tx).as_ref() { let _ = tx.send(AppEvent::LibraryChanged { message: None }); let _ = tx.send(AppEvent::DeviceSyncStatus(self.status())); } } fn gc_tombstones(&self) -> Result<()> { let conn = lock(&self.conn); let compactable = compactable_tombstone_ids(&conn)?; for (op_id, origin, seq) in compactable { let revoked_target = tombstone_revoke_target(&conn, &op_id)?; conn.execute( "INSERT INTO sync_compacted (origin_device_id, max_seq) VALUES (?1, ?2) ON CONFLICT(origin_device_id) DO UPDATE SET max_seq = MAX(max_seq, excluded.max_seq)", params![origin, seq], )?; conn.execute("DELETE FROM sync_ops WHERE op_id = ?1", [op_id])?; if let Some(device_id) = revoked_target { delete_revoked_device_if_fully_compacted(&conn, &device_id)?; } } conn.execute( "DELETE FROM sync_state_likes WHERE liked = 0 AND EXISTS ( SELECT 1 FROM sync_compacted c WHERE c.origin_device_id = substr(sync_state_likes.op_id, 1, instr(sync_state_likes.op_id, ':') - 1) )", [], )?; Ok(()) } } /// Stamp source over the sync log for [`DeviceSync::run_podcast_action`]. struct PodcastLogStamps<'a> { conn: &'a std::sync::Mutex, /// Taken at the first stamp and held until the operations are appended. guard: Option>, device_id: String, /// `op_id` -> reserved `seq`. reserved: BTreeMap, } impl PodcastStampSource for PodcastLogStamps<'_> { fn device_id(&self) -> &str { &self.device_id } fn next_stamp(&mut self) -> Result { let conn = self.guard.get_or_insert_with(|| lock(self.conn)); let seq = get_meta(conn, "local_seq")? .and_then(|v| v.parse::().ok()) .unwrap_or(0) + 1; let last_hlc = get_meta(conn, "last_hlc_ms")? .and_then(|v| v.parse::().ok()) .unwrap_or(0); let hlc_ms = now_ms().max(last_hlc + 1); set_meta(conn, "local_seq", &seq.to_string())?; set_meta(conn, "last_hlc_ms", &hlc_ms.to_string())?; let op_id = format!("{}:{seq}", self.device_id); self.reserved.insert(op_id.clone(), seq); Ok(PodcastOpStamp::new(op_id, self.device_id.clone(), hlc_ms)) } } impl PodcastLogStamps<'_> { fn append(mut self, ops: Vec) -> Result<()> { if ops.len() != self.reserved.len() { tracing::warn!( reserved = self.reserved.len(), returned = ops.len(), "podcast action reserved stamps it did not return operations for" ); } if ops.is_empty() { return Ok(()); } let conn = self.guard.get_or_insert_with(|| lock(self.conn)); for op in ops { let Some(&seq) = self.reserved.get(&op.stamp.op_id) else { anyhow::bail!( "podcast operation {} was not stamped by this sync log", op.stamp.op_id ); }; let op = op.into_wire(seq); let payload = serde_json::to_value(&op.payload)?; let kind = payload_kind(&op.payload); conn.execute( "INSERT OR IGNORE INTO sync_ops (op_id, origin_device_id, seq, kind, payload_json, hlc_ms, received_at_ms, tombstone) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ op.op_id, op.origin_device_id, op.seq, kind, payload.to_string(), op.hlc_ms, now_ms(), i64::from(op.payload.is_tombstone()), ], )?; coalesce_progress_ops(conn, &op)?; conn.execute( "INSERT INTO sync_vectors (device_id, max_seq) VALUES (?1, ?2) ON CONFLICT(device_id) DO UPDATE SET max_seq = MAX(max_seq, excluded.max_seq)", params![op.origin_device_id, op.seq], )?; tracing::debug!(op_id = %op.op_id, kind, "recorded podcast sync op"); } Ok(()) } } impl crate::podcasts::OpLog for DeviceSync { fn record( &self, action: &mut dyn FnMut(&mut dyn PodcastStampSource) -> Result>, ) -> Result<()> { self.run_podcast_action(|stamps| Ok(((), action(stamps)?))) } } #[derive(Debug)] struct StoredDevice { device_id: String, endpoint_id: String, endpoint_ticket: String, } async fn resolve_device_peer( service: &MusicDhtService, device: &StoredDevice, ) -> Result { if let Ok(peer) = device.endpoint_id.parse::() && (service.connected_peers().contains(&peer) || service .known_peers() .iter() .any(|contact| contact.peer_id == peer)) { // The live DHT contact carries a ticket for the current schema. The // persisted device ticket may predate a schema upgrade. return Ok(peer); } let ticket: PeerTicket = device.endpoint_ticket.parse()?; service.connect(ticket).await.map_err(Into::into) } pub async fn serve_peers( mut acceptor: StreamAcceptor, sync: Arc, service: Arc, transport_stats: Arc, ) { while let Some(stream) = acceptor.accept().await { let sync = Arc::clone(&sync); let service = Arc::clone(&service); let transport_stats = Arc::clone(&transport_stats); tokio::spawn(async move { let peer = stream.peer_id; if let Err(err) = serve_one(stream, sync, service, transport_stats).await { tracing::warn!(peer = %peer, "personal sync stream failed: {err:#}"); } }); } } pub async fn sync_loop( sync: Arc, service: Arc, transport_stats: Arc, ) { let mut interval = tokio::time::interval(SYNC_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); // Independent polls: an offline device must not hold up live outputs. // JoinSet aborts outstanding IO when federation stops or restarts. let mut polls = tokio::task::JoinSet::new(); let mut active = std::collections::HashMap::new(); loop { tokio::select! { _ = interval.tick() => { match sync.active_remote_devices() { Ok(devices) => for device in devices { if device.endpoint_ticket.trim().is_empty() || active.values().any(|id| id == &device.device_id) { continue; } let device_id = device.device_id.clone(); let sync = Arc::clone(&sync); let service = Arc::clone(&service); let stats = Arc::clone(&transport_stats); let handle = polls.spawn(async move { tokio::time::timeout(Duration::from_secs(30), sync.sync_device(service, &device, stats)) .await.context("device sync exchange timed out")? }); active.insert(handle.id(), device_id); }, Err(err) => { let _ = sync.set_last_error(Some(format!("{err:#}"))); } } if let Err(err) = sync.gc_tombstones() { tracing::debug!("personal sync cleanup failed: {err:#}"); } } Some(completed) = polls.join_next_with_id(), if !polls.is_empty() => { let (task, result) = match completed { Ok((task, result)) => (task, result), Err(error) => (error.id(), Err(anyhow::Error::from(error))), }; if let Some(device_id) = active.remove(&task) && let Err(err) = result { tracing::debug!(device = %device_id, "device sync failed: {err:#}"); let _ = sync.set_last_error(Some(format!("{}: {err:#}", short_id(&device_id)))); } } } if let Some(tx) = lock(&sync.event_tx).as_ref() { let _ = tx.send(AppEvent::DeviceSyncStatus(sync.status())); } } } async fn serve_one( mut stream: ByteStream, sync: Arc, service: Arc, transport_stats: Arc, ) -> Result<()> { crate::federation::record_stream_transport( &transport_stats, "device-sync", "inbound", "open", &stream, ); match read_msg(&mut stream).await? { WireMessage::PairRequest { invite_id, secret, profile, group_id, group_active_devices, devices, vector, ops, snapshot, playback, } => { handle_pair_request( stream, sync, service, invite_id, secret, profile, group_id, group_active_devices, devices, vector, ops, snapshot, playback, ) .await } WireMessage::Hello { group_id, profile, devices, vector, ops, snapshot, playback, } => { handle_hello( stream, sync, service, group_id, profile, devices, vector, ops, snapshot, playback, ) .await } _ => anyhow::bail!("unexpected first message"), } } #[allow(clippy::too_many_arguments)] async fn handle_pair_request( mut stream: ByteStream, sync: Arc, service: Arc, invite_id: String, secret: String, mut profile: DeviceProfileWire, requester_group_id: Option, requester_group_active_devices: usize, requester_group_devices: Vec, vector: BTreeMap, ops: Vec, snapshot: SyncSnapshot, playback: Option, ) -> Result<()> { profile.endpoint_id = stream.peer_id.to_string(); let request_id = pair_request_id(&invite_id, &profile.device_id); if !valid_pair_request(&sync, &invite_id, &secret, &request_id)? { tracing::warn!( peer = %stream.peer_id, invite_id, "ignored pairing request with invalid invite secret" ); write_msg( &mut stream, &WireMessage::PairResponse { accepted: false, pending: false, error: Some("invalid or expired invite".to_string()), group_id: None, profile: None, devices: Vec::new(), vector: BTreeMap::new(), ops: Vec::new(), snapshot: SyncSnapshot::default(), playback: None, }, ) .await?; finish_response(&mut stream).await?; return Ok(()); } let identity = sync.ensure_identity()?; let requester_group_id = requester_group_id.filter(|id| !id.trim().is_empty()); let requester_group_active_devices = requester_group_active_devices.max(1); let requester_group_conflict = requester_group_id .as_deref() .is_some_and(|group_id| group_id != identity.group_id) && requester_group_active_devices > 1; let requester_group_devices_json = serde_json::to_string(&requester_group_devices)?; let inserted = { let conn = lock(&sync.conn); conn.execute( "INSERT OR IGNORE INTO sync_pending_pairing (request_id, device_id, name, client_version, endpoint_id, endpoint_ticket, invite_id, created_at_ms, status, requester_group_id, requester_group_active_devices, requester_group_devices_json, use_requester_group, protocol_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'pending', ?9, ?10, ?11, 0, ?12)", params![ request_id, profile.device_id, profile.name, profile.client_version, profile.endpoint_id, profile.endpoint_ticket, invite_id, now_ms(), requester_group_id.as_deref(), requester_group_active_devices as i64, requester_group_devices_json, profile.protocol_version, ], )? }; if inserted > 0 && let Some(tx) = lock(&sync.event_tx).as_ref() { let _ = tx.send(AppEvent::DevicePairingRequest(PendingPairing { request_id: request_id.clone(), device_id: profile.device_id.clone(), name: profile.name.clone(), client_version: profile.client_version.clone(), requester_group_id: requester_group_conflict .then(|| requester_group_id.clone()) .flatten(), requester_group_active_devices: requester_group_conflict .then_some(requester_group_active_devices) .unwrap_or(0), })); } let pairing = pairing_status(&sync, &request_id)?; match pairing.as_ref().map(|status| status.status.as_str()) { Some("pending") => { write_msg( &mut stream, &WireMessage::PairResponse { accepted: false, pending: true, error: Some("pairing pending".to_string()), group_id: None, profile: None, devices: Vec::new(), vector: BTreeMap::new(), ops: Vec::new(), snapshot: SyncSnapshot::default(), playback: None, }, ) .await?; finish_response(&mut stream).await?; return Ok(()); } Some("accepted") => {} _ => { write_msg( &mut stream, &WireMessage::PairResponse { accepted: false, pending: false, error: Some("pairing denied".to_string()), group_id: None, profile: None, devices: Vec::new(), vector: BTreeMap::new(), ops: Vec::new(), snapshot: SyncSnapshot::default(), playback: None, }, ) .await?; finish_response(&mut stream).await?; return Ok(()); } } let use_requester_group = pairing .as_ref() .is_some_and(|status| status.use_requester_group); let own_ticket = service.ticket().await?.to_string(); let own_profile = sync.own_profile(&own_ticket)?; let mut response_group_id = identity.group_id; if use_requester_group && let Some(group_id) = requester_group_id .as_deref() .filter(|group_id| !group_id.trim().is_empty()) { sync.set_group_id(group_id)?; response_group_id = group_id.to_string(); sync.apply_device_profiles(&requester_group_devices, Some(&profile.device_id))?; } sync.apply_device_profile( &profile, true, VersionReport::SelfReported(Some(profile.protocol_version)), )?; if let Some(playback) = playback { sync.apply_playback_snapshot(&profile.device_id, playback)?; } sync.apply_received(snapshot, ops)?; sync.note_peer_vector(&profile.device_id, &vector)?; { let conn = lock(&sync.conn); conn.execute( "UPDATE sync_invites SET used_at_ms = ?2 WHERE invite_id = ?1", params![invite_id, now_ms()], )?; } sync.set_last_sync(Some(format!("paired {}", short_id(&profile.device_id))))?; let devices = sync.device_profiles()?; let vector = sync.vector()?; let peer_version = sync.peer_protocol_version(&profile.device_id)?; let ops = sync.ops_for_peer(&profile.device_id, peer_version)?; let snapshot = sync.snapshot_for(peer_version)?; let playback = sync.local_playback_snapshot_for(peer_version); write_msg( &mut stream, &WireMessage::PairResponse { accepted: true, pending: false, error: None, group_id: Some(response_group_id), profile: Some(own_profile), devices, vector, ops, snapshot, playback, }, ) .await?; finish_response(&mut stream).await?; Ok(()) } #[allow(clippy::too_many_arguments)] async fn handle_hello( mut stream: ByteStream, sync: Arc, service: Arc, group_id: String, mut profile: DeviceProfileWire, devices: Vec, vector: BTreeMap, ops: Vec, snapshot: SyncSnapshot, playback: Option, ) -> Result<()> { let identity = sync.ensure_identity()?; if group_id != identity.group_id { write_msg( &mut stream, &WireMessage::SyncResponse { accepted: false, error: Some("sync group mismatch".to_string()), devices: Vec::new(), vector: BTreeMap::new(), ops: Vec::new(), snapshot: SyncSnapshot::default(), playback: None, }, ) .await?; finish_response(&mut stream).await?; return Ok(()); } if !is_active_trusted(&sync, &profile.device_id)? { write_msg( &mut stream, &WireMessage::SyncResponse { accepted: false, error: Some("device is not trusted".to_string()), devices: Vec::new(), vector: BTreeMap::new(), ops: Vec::new(), snapshot: SyncSnapshot::default(), playback: None, }, ) .await?; finish_response(&mut stream).await?; return Ok(()); } profile.endpoint_id = stream.peer_id.to_string(); sync.apply_device_profile( &profile, false, VersionReport::SelfReported(Some(profile.protocol_version)), )?; sync.apply_device_profiles(&devices, Some(&profile.device_id))?; if let Some(playback) = playback { sync.apply_playback_snapshot(&profile.device_id, playback)?; } sync.apply_received(snapshot, ops)?; sync.note_peer_vector(&profile.device_id, &vector)?; sync.mark_seen(&profile.device_id, Some(stream.peer_id.to_string()))?; sync.set_last_sync(Some(format!("synced {}", short_id(&profile.device_id))))?; let own_ticket = service.ticket().await?.to_string(); let own_profile = sync.own_profile(&own_ticket)?; let devices = sync.device_profiles()?; let vector = sync.vector()?; let peer_version = sync.peer_protocol_version(&profile.device_id)?; let ops = sync.ops_for_peer(&profile.device_id, peer_version)?; let snapshot = sync.snapshot_for(peer_version)?; let playback = sync.local_playback_snapshot_for(peer_version); write_msg( &mut stream, &WireMessage::SyncResponse { accepted: true, error: None, devices: { let mut profiles = devices; profiles.push(own_profile); profiles }, vector, ops, snapshot, playback, }, ) .await?; finish_response(&mut stream).await?; sync.gc_tombstones()?; Ok(()) } fn pairing_status(sync: &DeviceSync, request_id: &str) -> Result> { let conn = lock(&sync.conn); Ok(conn .query_row( "SELECT status, use_requester_group FROM sync_pending_pairing WHERE request_id = ?1", [request_id], |row| { Ok(PairingStatus { status: row.get(0)?, use_requester_group: row.get::<_, i64>(1)? != 0, }) }, ) .optional()?) } fn pair_request_id(invite_id: &str, device_id: &str) -> String { let digest = blake3::hash(format!("{invite_id}:{device_id}").as_bytes()); format!("pair_{}", &digest.to_hex()[..16]) } fn valid_pair_request( sync: &DeviceSync, invite_id: &str, secret: &str, request_id: &str, ) -> Result { let conn = lock(&sync.conn); let expected: Option<(String, i64, Option)> = conn .query_row( "SELECT secret_hash, expires_at_ms, used_at_ms FROM sync_invites WHERE invite_id = ?1", [invite_id], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .optional()?; let Some((secret_hash, expires_at_ms, used_at_ms)) = expected else { return Ok(false); }; if expires_at_ms < now_ms() || secret_hash != hash_secret(secret) { return Ok(false); } if used_at_ms.is_none() { return Ok(true); } let accepted: Option = conn .query_row( "SELECT 1 FROM sync_pending_pairing WHERE request_id = ?1 AND status = 'accepted'", [request_id], |row| row.get(0), ) .optional()?; Ok(accepted.is_some()) } fn is_active_trusted(sync: &DeviceSync, device_id: &str) -> Result { let own = sync.ensure_identity()?.device_id; if device_id == own { return Ok(true); } let conn = lock(&sync.conn); let found: Option = conn .query_row( "SELECT 1 FROM sync_devices WHERE device_id = ?1 AND trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL", [device_id], |row| row.get(0), ) .optional()?; Ok(found.is_some()) } fn init_schema(conn: &Connection) -> Result<()> { conn.execute_batch( r#" CREATE TABLE IF NOT EXISTS sync_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS sync_devices ( device_id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT '', client_version TEXT NOT NULL DEFAULT '', protocol_version INTEGER NOT NULL DEFAULT 1, endpoint_id TEXT NOT NULL DEFAULT '', endpoint_ticket TEXT NOT NULL DEFAULT '', trusted_at_ms INTEGER, last_seen_ms INTEGER, revoked_at_ms INTEGER, revoked_by TEXT, revoke_cutoff_seq INTEGER ); CREATE TABLE IF NOT EXISTS sync_invites ( invite_id TEXT PRIMARY KEY, secret_hash TEXT NOT NULL, expires_at_ms INTEGER NOT NULL, created_at_ms INTEGER NOT NULL, used_at_ms INTEGER ); CREATE TABLE IF NOT EXISTS sync_pending_pairing ( request_id TEXT PRIMARY KEY, device_id TEXT NOT NULL, name TEXT NOT NULL, client_version TEXT NOT NULL, endpoint_id TEXT NOT NULL, endpoint_ticket TEXT NOT NULL, invite_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL, answered_at_ms INTEGER, status TEXT NOT NULL, requester_group_id TEXT, requester_group_active_devices INTEGER NOT NULL DEFAULT 1, requester_group_devices_json TEXT NOT NULL DEFAULT '[]', use_requester_group INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS sync_ops ( op_id TEXT PRIMARY KEY, origin_device_id TEXT NOT NULL, seq INTEGER NOT NULL, kind TEXT NOT NULL, payload_json TEXT NOT NULL, hlc_ms INTEGER NOT NULL, received_at_ms INTEGER NOT NULL, tombstone INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_sync_ops_origin_seq ON sync_ops(origin_device_id, seq); CREATE INDEX IF NOT EXISTS idx_sync_ops_tombstone ON sync_ops(tombstone); CREATE TABLE IF NOT EXISTS sync_vectors ( device_id TEXT PRIMARY KEY, max_seq INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS sync_peer_acks ( peer_device_id TEXT NOT NULL, origin_device_id TEXT NOT NULL, max_seq INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (peer_device_id, origin_device_id) ); CREATE TABLE IF NOT EXISTS sync_compacted ( origin_device_id TEXT PRIMARY KEY, max_seq INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS sync_state_likes ( content_id TEXT PRIMARY KEY, liked INTEGER NOT NULL, hlc_ms INTEGER NOT NULL, op_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS sync_state_playlists ( playlist_id TEXT PRIMARY KEY, title TEXT NOT NULL, deleted INTEGER NOT NULL DEFAULT 0, hlc_ms INTEGER NOT NULL, op_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS sync_state_playlist_items ( playlist_id TEXT NOT NULL, content_id TEXT NOT NULL, present INTEGER NOT NULL DEFAULT 1, position INTEGER NOT NULL DEFAULT 0, hlc_ms INTEGER NOT NULL, op_id TEXT NOT NULL, PRIMARY KEY (playlist_id, content_id) ); CREATE TABLE IF NOT EXISTS sync_playback_applied ( op_id TEXT PRIMARY KEY, applied_at_ms INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS sync_podcast_pending ( op_id TEXT PRIMARY KEY ); CREATE INDEX IF NOT EXISTS idx_sync_ops_kind ON sync_ops(kind); "#, )?; ensure_column( conn, "sync_pending_pairing", "requester_group_id", "ALTER TABLE sync_pending_pairing ADD COLUMN requester_group_id TEXT", )?; ensure_column( conn, "sync_pending_pairing", "requester_group_active_devices", "ALTER TABLE sync_pending_pairing ADD COLUMN requester_group_active_devices INTEGER NOT NULL DEFAULT 1", )?; ensure_column( conn, "sync_pending_pairing", "requester_group_devices_json", "ALTER TABLE sync_pending_pairing ADD COLUMN requester_group_devices_json TEXT NOT NULL DEFAULT '[]'", )?; ensure_column( conn, "sync_pending_pairing", "use_requester_group", "ALTER TABLE sync_pending_pairing ADD COLUMN use_requester_group INTEGER NOT NULL DEFAULT 0", )?; // Protocol version the requester reported in its pairing request. ensure_column( conn, "sync_pending_pairing", "protocol_version", "ALTER TABLE sync_pending_pairing ADD COLUMN protocol_version INTEGER", )?; Ok(()) } fn ensure_column(conn: &Connection, table: &str, column: &str, ddl: &str) -> Result<()> { if table_has_column(conn, table, column)? { return Ok(()); } conn.execute(ddl, [])?; Ok(()) } fn table_has_column(conn: &Connection, table: &str, column: &str) -> Result { let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { let name: String = row.get(1)?; if name == column { return Ok(true); } } Ok(false) } fn get_meta(conn: &Connection, key: &str) -> Result> { Ok(conn .query_row("SELECT value FROM sync_meta WHERE key = ?1", [key], |row| { row.get(0) }) .optional()?) } fn set_meta(conn: &Connection, key: &str, value: &str) -> Result<()> { conn.execute( "INSERT INTO sync_meta (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value", params![key, value], )?; Ok(()) } fn delete_meta(conn: &Connection, key: &str) -> Result<()> { conn.execute("DELETE FROM sync_meta WHERE key = ?1", [key])?; Ok(()) } /// Keeps only the newest `podcast_progress_set` of an origin per episode in /// the log (the periodic checkpoints of a playing episode would grow it /// without bound). /// /// Safe because the operations of one origin are ordered: a later one has /// a later `hlc_ms` and never a lower progress generation, so it wins every /// register comparison against the earlier ones. Peers that have not /// received the removed operations get the later one, and their vector /// skips the gap as usual. fn coalesce_progress_ops(conn: &Connection, op: &SyncOpWire) -> Result<()> { let SyncOpPayload::PodcastProgressSet { episode_id, .. } = &op.payload else { return Ok(()); }; conn.execute( "DELETE FROM sync_ops WHERE origin_device_id = ?1 AND kind = 'podcast_progress_set' AND seq < ?2 AND json_extract(payload_json, '$.episode_id') = ?3", params![op.origin_device_id, op.seq, episode_id.as_str()], )?; Ok(()) } /// Podcast operations and playback commands with podcast items carry feed /// and audio URLs, which may hold private tokens: never log their payload. fn payload_may_hold_private_urls(payload: &SyncOpPayload) -> bool { payload.is_podcast() || matches!( payload, SyncOpPayload::PlaybackCommand { command, .. } if music_dht::device_sync::playback_command_has_podcasts(command) ) } fn payload_kind(payload: &SyncOpPayload) -> &'static str { match payload { SyncOpPayload::TrackLikeSet { .. } => "track_like_set", SyncOpPayload::PlaylistCreated { .. } => "playlist_created", SyncOpPayload::PlaylistRenamed { .. } => "playlist_renamed", SyncOpPayload::PlaylistDeleted { .. } => "playlist_deleted", SyncOpPayload::PlaylistTrackAdded { .. } => "playlist_track_added", SyncOpPayload::PlaylistTrackRemoved { .. } => "playlist_track_removed", SyncOpPayload::DeviceProfileSet { .. } => "device_profile_set", SyncOpPayload::DeviceTrusted { .. } => "device_trusted", SyncOpPayload::DeviceRevoked { .. } => "device_revoked", SyncOpPayload::PlaybackCommand { .. } => "playback_command", SyncOpPayload::ListenRecorded { .. } => "listen_recorded", SyncOpPayload::PodcastSubscriptionSet { .. } => "podcast_subscription_set", SyncOpPayload::PodcastShareSet { .. } => "podcast_share_set", SyncOpPayload::PodcastFeedAliasesAdded { .. } => "podcast_feed_aliases_added", SyncOpPayload::PodcastProgressSet { .. } => "podcast_progress_set", SyncOpPayload::PodcastListenRecorded { .. } => "podcast_listen_recorded", } } fn compactable_tombstone_ids(conn: &Connection) -> Result> { let own_device_id = get_meta(conn, "device_id")?.unwrap_or_default(); let active_devices = conn .prepare( "SELECT device_id FROM sync_devices WHERE trusted_at_ms IS NOT NULL AND revoked_at_ms IS NULL", )? .query_map([], |row| row.get::<_, String>(0))? .collect::>>()?; if active_devices.len() <= 1 { let mut stmt = conn.prepare( "SELECT op_id, origin_device_id, seq FROM sync_ops WHERE tombstone = 1", )?; return Ok(stmt .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))? .collect::>>()?); } let mut stmt = conn.prepare( "SELECT op_id, origin_device_id, seq FROM sync_ops WHERE tombstone = 1 ORDER BY received_at_ms", )?; let rows = stmt .query_map([], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, i64>(2)?, )) })? .collect::>>()?; let mut out = Vec::new(); for (op_id, origin, seq) in rows { let local_vector: i64 = conn .query_row( "SELECT max_seq FROM sync_vectors WHERE device_id = ?1", [&origin], |row| row.get(0), ) .optional()? .unwrap_or(0); let mut all_acked = true; for device in &active_devices { let seen = if device == &own_device_id { local_vector } else if device == &origin { seq } else { conn.query_row( "SELECT max_seq FROM sync_peer_acks WHERE peer_device_id = ?1 AND origin_device_id = ?2", params![device, origin], |row| row.get(0), ) .optional()? .unwrap_or(0) }; if seen < seq { all_acked = false; break; } } if all_acked { out.push((op_id, origin, seq)); } } Ok(out) } fn tombstone_revoke_target(conn: &Connection, op_id: &str) -> Result> { let payload_json: Option = conn .query_row( "SELECT payload_json FROM sync_ops WHERE op_id = ?1 AND kind = 'device_revoked'", [op_id], |row| row.get(0), ) .optional()?; let Some(payload_json) = payload_json else { return Ok(None); }; let payload: SyncOpPayload = serde_json::from_str(&payload_json)?; Ok(match payload { SyncOpPayload::DeviceRevoked { target_device_id, .. } => Some(target_device_id), _ => None, }) } fn has_revoke_op_for_target(conn: &Connection, device_id: &str) -> Result { let mut stmt = conn.prepare("SELECT payload_json FROM sync_ops WHERE kind = 'device_revoked'")?; let payloads = stmt .query_map([], |row| row.get::<_, String>(0))? .collect::>>()?; for payload_json in payloads { let Ok(SyncOpPayload::DeviceRevoked { target_device_id, .. }) = serde_json::from_str::(&payload_json) else { continue; }; if target_device_id == device_id { return Ok(true); } } Ok(false) } fn delete_revoked_device_if_fully_compacted(conn: &Connection, device_id: &str) -> Result<()> { let own_device_id = get_meta(conn, "device_id")?.unwrap_or_default(); if device_id == own_device_id { return Ok(()); } let origin_ops: i64 = conn.query_row( "SELECT COUNT(*) FROM sync_ops WHERE origin_device_id = ?1", [device_id], |row| row.get(0), )?; if origin_ops > 0 || has_revoke_op_for_target(conn, device_id)? { return Ok(()); } conn.execute( "DELETE FROM sync_devices WHERE device_id = ?1 AND revoked_at_ms IS NOT NULL", [device_id], )?; conn.execute( "DELETE FROM sync_peer_acks WHERE peer_device_id = ?1", [device_id], )?; Ok(()) } fn peer_ack_floor_label(conn: &Connection) -> Result { let count: i64 = conn.query_row("SELECT COUNT(*) FROM sync_peer_acks", [], |row| row.get(0))?; if count == 0 { return Ok("none".to_string()); } let min: i64 = conn.query_row( "SELECT COALESCE(MIN(max_seq), 0) FROM sync_peer_acks", [], |row| row.get(0), )?; let max: i64 = conn.query_row( "SELECT COALESCE(MAX(max_seq), 0) FROM sync_peer_acks", [], |row| row.get(0), )?; Ok(format!("{min}..{max} ({count} acks)")) } async fn write_msg(stream: &mut ByteStream, message: &WireMessage) -> Result<()> { let mut message = message.clone(); fit_podcast_snapshot(&mut message); music_dht::device_sync::write_msg(stream, &message).await?; Ok(()) } fn message_snapshot(message: &mut WireMessage) -> &mut SyncSnapshot { match message { WireMessage::PairRequest { snapshot, .. } | WireMessage::PairResponse { snapshot, .. } | WireMessage::Hello { snapshot, .. } | WireMessage::SyncResponse { snapshot, .. } => snapshot, } } /// Bounds the podcast snapshot sections so the whole line stays under the /// protocol limit (`podcast_snapshot_budget`): the operations, music /// snapshot and playback queue of the same message come first. fn fit_podcast_snapshot(message: &mut WireMessage) { if !message_snapshot(message).has_podcast_state() { return; } let podcast = message_snapshot(message).clone(); message_snapshot(message).strip_podcast_state(); let other_bytes = serde_json::to_vec(message).map_or(usize::MAX, |bytes| bytes.len()); let snapshot = message_snapshot(message); *snapshot = podcast; snapshot.bound_podcast_state_to(music_dht::device_sync::podcast_snapshot_budget(other_bytes)); } async fn finish_response(stream: &mut ByteStream) -> Result<()> { music_dht::device_sync::finish_response(stream, RESPONSE_DRAIN_TIMEOUT).await?; Ok(()) } fn parse_invite(value: &str) -> Result { anyhow::ensure!( value.trim().starts_with("frid://i/"), "usage: :connect frid://i/" ); Ok(music_dht::device_sync::parse_invite(value)?) } pub fn invite_network_id(value: &str) -> Result { let invite = parse_invite(value)?; let ticket: PeerTicket = invite .ticket .parse() .map_err(|err| anyhow::anyhow!("malformed invite ticket: {err}"))?; Ok(ticket.network_id) } fn hex_encode(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } fn default_device_name(device_id: &str) -> String { let host = std::env::var("HOSTNAME") .or_else(|_| std::env::var("COMPUTERNAME")) .ok() .filter(|value| !value.trim().is_empty()) .unwrap_or_else(|| "furumi".to_string()); format!("{host}-{}", short_id(device_id)) } fn short_id(value: &str) -> String { value.chars().take(10).collect() } #[cfg(test)] #[path = "devices/tests.rs"] mod tests; #[cfg(test)] mod interop_tests; #[cfg(test)] mod wire_compat_tests; #[cfg(test)] mod podcast_sync_tests; fn playback_clock() -> u64 { static START: std::sync::OnceLock = std::sync::OnceLock::new(); START .get_or_init(std::time::Instant::now) .elapsed() .as_millis() as u64 }