From 085bff6191b7eae96eacfa2d39fa775451c02711 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:10:33 +0200 Subject: [PATCH] Harden podcast sync registers, versions and snapshots - Order subscription and share registers by a per-podcast Lamport generation before hlc_ms (next_podcast_generation), so clock skew cannot undo an observed unsubscribe or share decision. - share=false is register state, not a compactable tombstone. - Add podcast_feed_aliases_added so refreshes can sync aliases without touching the registers; SubscriptionRecord::feed_urls keeps the current feed URL resolvable beyond the alias bound. - Carry progress_generation on queue items for handoff. - Distinguish self-reported and relayed peer versions, and gate PairRequest by an optional inviter version in InviteWire. - Give each podcast snapshot section its own byte budget, drop channel descriptions from snapshots and allow a caller-provided budget. - Reject one-slash http(s) typos in FeedUrl::from_user_input. - Pin that malformed ids and feed URLs reject the whole message. Co-Authored-By: Claude Opus 5.5 --- crates/music-dht/src/device_sync.rs | 469 +++++++++++++++++--- crates/music-dht/src/jam.rs | 1 + crates/music-dht/src/podcast/feed_url.rs | 29 +- crates/music-dht/src/podcast/meta.rs | 17 +- crates/music-dht/src/podcast/mod.rs | 15 +- crates/music-dht/src/podcast/state.rs | 133 ++++-- crates/music-dht/src/podcast/state_tests.rs | 133 +++++- 7 files changed, 699 insertions(+), 98 deletions(-) diff --git a/crates/music-dht/src/device_sync.rs b/crates/music-dht/src/device_sync.rs index 679abff..12769d4 100644 --- a/crates/music-dht/src/device_sync.rs +++ b/crates/music-dht/src/device_sync.rs @@ -11,13 +11,16 @@ //! | 1 | likes, playlists, membership (`furumi/sync/1`) | //! | 2 | [`SyncOpPayload::ListenRecorded`] (`furumi/sync/2`) | //! | 3 | fenced personal playback coordination | -//! | 4 | podcasts: four `podcast_*` operations, podcast snapshot sections, [`PlaybackSource`] on queue items, reserved `playback_rate` | +//! | 4 | podcasts: five `podcast_*` operations, podcast snapshot sections, [`PlaybackSource`] on queue items, reserved `playback_rate` | //! -//! The ALPN stays `furumi/sync/2` for v3 and v4. [`SyncOpPayload`] and -//! [`PlaybackSource`] are closed serde enums: a peer that does not know a -//! `kind` fails to deserialize the **whole** message that contains it, not -//! just one operation. Senders must therefore filter by the peer's version -//! with [`op_allowed_for_peer`] and [`playback_allowed_for_peer`]. See the +//! The ALPN stays `furumi/sync/2` for v3 and v4. [`SyncOpPayload`] is a +//! closed serde enum: a peer that does not know an operation `kind` fails to +//! deserialize the **whole** message that contains it, not just one +//! operation. Peers before v4 ignore [`PlaybackTrack::source`] and would treat +//! episodes as broken music items. Senders must therefore filter by the +//! peer's version ([`profile_protocol_version`], [`relayed_protocol_version`], +//! [`InviteWire::inviter_protocol_version`]) with [`op_allowed_for_peer`], +//! [`playback_allowed_for_peer`] and [`SyncSnapshot::for_peer`]. See the //! workspace `PODCASTS.md` for the podcast contract. use std::collections::BTreeMap; @@ -29,10 +32,10 @@ use tokio::io::{AsyncRead, AsyncReadExt}; use crate::error::{MusicDhtError, Result}; use crate::podcast::{ - EpisodeId, FeedUrl, ListenSession, MAX_PODCAST_SNAPSHOT_BYTES, MAX_SNAPSHOT_LISTEN_SESSIONS, - MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, MAX_SNAPSHOT_SUBSCRIPTIONS, PodcastEpisodeRef, - PodcastError, PodcastId, PodcastMeta, PodcastResult, ProgressRecord, ProgressStatus, - ShareRecord, SubscriptionRecord, + EpisodeId, FeedUrl, ListenSession, MAX_FEED_ALIASES, MAX_PODCAST_SNAPSHOT_BYTES, + MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, + MAX_SNAPSHOT_SUBSCRIPTIONS, PodcastEpisodeRef, PodcastError, PodcastId, PodcastMeta, + PodcastResult, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, }; /// Legacy ALPN used by personal-device sync v1. @@ -83,11 +86,20 @@ pub const fn supports_podcasts(protocol_version: u16) -> bool { protocol_version >= PODCAST_PROTOCOL_VERSION } -/// Protocol version to store for a device after reading one of its profiles. +/// Protocol version to store for a device after reading **its own** report. /// -/// `reported` is the version the device itself put into its -/// [`DeviceProfileWire`] (Hello/pairing) or [`SyncOpPayload::DeviceProfileSet`]; -/// `previously_known` is the stored value. Receivers and relays must never +/// Self-reports are: the `profile` of a `Hello`, `PairRequest` or +/// `PairResponse` sent by that device; the entry of a `devices` list whose +/// `device_id` is the authenticated sender of the message (a `SyncResponse` +/// has no `profile` field, so this is how an initiator learns the responder's +/// version); and a [`SyncOpPayload::DeviceProfileSet`] whose +/// `origin_device_id` is that device. `reported` is the version in the report +/// (`None` when an older client left it out) and `previously_known` the +/// stored value. A self-report replaces the stored value, so a downgrade is +/// noticed. +/// +/// Profiles of **other** devices relayed by a peer are not self-reports: use +/// [`relayed_protocol_version`] for them. Receivers and relays must never /// substitute their own [`DEVICE_SYNC_PROTOCOL_VERSION`] for another device: /// doing so would make version gating send podcast state to old clients. pub fn profile_protocol_version(reported: Option, previously_known: Option) -> u16 { @@ -96,6 +108,18 @@ pub fn profile_protocol_version(reported: Option, previously_known: Option< .unwrap_or(UNREPORTED_PROTOCOL_VERSION) } +/// Protocol version to store for a device after reading a profile of it +/// relayed by **another** device (an entry of a `devices` list). +/// +/// Deployed v1–v3 clients relay every device with their own protocol +/// constant, so a relayed value can understate a device's version. A relayed +/// value therefore only raises the stored version (`max(stored, relayed)`) +/// and is taken as is only when nothing is stored. Lowering requires a +/// self-report ([`profile_protocol_version`]). +pub fn relayed_protocol_version(relayed: u16, previously_known: Option) -> u16 { + previously_known.map_or(relayed, |stored| stored.max(relayed)) +} + /// Default invite lifetime used by applications unless they need a custom TTL. pub const DEFAULT_INVITE_TTL_MS: i64 = 10 * 60 * 1000; @@ -122,6 +146,25 @@ pub struct InviteWire { /// Expiration timestamp in unix milliseconds. #[serde(rename = "e")] pub expires_at_ms: i64, + /// Device-sync protocol version of the inviter (v4+). + /// + /// `None` in invites created by older clients. The invite consumer sends + /// its `PairRequest` before it hears from the inviter, so it gates the + /// operations, snapshot sections and playback state of that request by + /// [`InviteWire::inviter_protocol_version`]. + #[serde(rename = "p", default, skip_serializing_if = "Option::is_none")] + pub protocol_version: Option, +} + +impl InviteWire { + /// Protocol version to assume for the inviter while building a + /// `PairRequest`: the version in the invite, or + /// [`UNREPORTED_PROTOCOL_VERSION`] for invites from older clients, which + /// strips all podcast state from the request. Podcast state then catches + /// up through the snapshot of the next regular sync. + pub fn inviter_protocol_version(&self) -> u16 { + self.protocol_version.unwrap_or(UNREPORTED_PROTOCOL_VERSION) + } } /// Public device profile exchanged inside a trusted sync group. @@ -303,6 +346,13 @@ impl ListenEvent { /// Absent on music items from older clients, which are music. Podcast /// episodes are identified only by this field: never by a negative id, a /// placeholder release or a URL in `file_path`. +/// +/// Clients before v4 do not know [`PlaybackTrack::source`] at all and ignore +/// it: they would take an episode for a broken music item (id 0, no content +/// id) and forward it that way, losing the episode for every device. That is +/// why podcast-bearing playback state is gated by version. This enum is +/// closed for v4 clients: a new variant needs a protocol bump, because a v4 +/// client fails to parse the whole message on an unknown `kind`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlaybackSource { @@ -621,9 +671,10 @@ pub enum SyncOpPayload { }, /// Subscribe to or unsubscribe from a podcast (protocol v4). /// - /// Last-writer-wins by `(hlc_ms, op_id)` of the operation. Unsubscribing - /// is state, not a tombstone. Only explicit user actions emit it; RSS - /// refreshes never do. Materialize with [`SyncOpWire::podcast_subscription`]. + /// Last-writer-wins by `(generation, hlc_ms, op_id)`; see + /// [`SubscriptionRecord`]. Unsubscribing is state, not a tombstone. Only + /// explicit user actions emit it; RSS refreshes never do. Materialize with + /// [`SyncOpWire::podcast_subscription`]. PodcastSubscriptionSet { /// Podcast. podcast_id: PodcastId, @@ -634,16 +685,37 @@ pub enum SyncOpPayload { /// Known feed URLs (grow-only, bounded). #[serde(default)] feed_aliases: Vec, + /// Per-podcast Lamport generation + /// ([`crate::podcast::next_podcast_generation`]). + #[serde(default)] + generation: u64, }, /// Share or stop sharing a podcast card with the federation (protocol v4). /// - /// Last-writer-wins; `shared = false` is a tombstone. Unsubscribing also - /// emits `shared = false`. + /// Last-writer-wins by `(generation, hlc_ms, op_id)`; see [`ShareRecord`]. + /// `shared = false` is privacy-protecting state and **not** a tombstone: + /// it is never compacted. Unsubscribing also emits `shared = false`. PodcastShareSet { /// Podcast. podcast_id: PodcastId, /// Desired share state. shared: bool, + /// Per-podcast Lamport generation, shared with the subscription. + #[serde(default)] + generation: u64, + }, + /// Record feed URLs of a podcast learned by any device (protocol v4). + /// + /// Grow-only: merged into the bounded alias set with + /// [`crate::podcast::merge_feed_aliases`]; it never touches the + /// subscription or share register, so RSS refreshes may emit it (after a + /// permanent redirect or an `itunes:new-feed-url`), but only when merging + /// the new URLs changes the stored set. + PodcastFeedAliasesAdded { + /// Podcast. + podcast_id: PodcastId, + /// Canonical feed URLs to add (at most `MAX_FEED_ALIASES + 1`). + feed_aliases: Vec, }, /// Write the durable resume register of an episode (protocol v4). /// @@ -684,9 +756,12 @@ pub enum SyncOpPayload { impl SyncOpPayload { /// Returns true for operations that remove state and may eventually compact. /// - /// `podcast_share_set { shared: false }` is a tombstone. Unsubscribing - /// (`podcast_subscription_set { subscribed: false }`) is **not**: it is - /// last-writer-wins state that must keep beating stale subscriptions. + /// No podcast operation is a tombstone. Unsubscribing + /// (`podcast_subscription_set { subscribed: false }`) and revoking a share + /// (`podcast_share_set { shared: false }`) are last-writer-wins state that + /// must keep beating stale `true` writes delivered again later; compacting + /// them (and deleting the state row, as like tombstones do) would let an + /// older operation still in some log resubscribe or republish the podcast. pub fn is_tombstone(&self) -> bool { matches!( self, @@ -694,7 +769,6 @@ impl SyncOpPayload { | SyncOpPayload::PlaylistDeleted { .. } | SyncOpPayload::PlaylistTrackRemoved { .. } | SyncOpPayload::DeviceRevoked { .. } - | SyncOpPayload::PodcastShareSet { shared: false, .. } ) } @@ -704,6 +778,7 @@ impl SyncOpPayload { self, SyncOpPayload::PodcastSubscriptionSet { .. } | SyncOpPayload::PodcastShareSet { .. } + | SyncOpPayload::PodcastFeedAliasesAdded { .. } | SyncOpPayload::PodcastProgressSet { .. } | SyncOpPayload::PodcastListenRecorded { .. } ) @@ -713,13 +788,15 @@ impl SyncOpPayload { impl SyncOpWire { /// Subscription record written by a `podcast_subscription_set` operation. /// - /// The register stamp is the operation's `(hlc_ms, op_id)`. + /// The register stamp is the payload `generation` and the operation's + /// `(hlc_ms, op_id)`. pub fn podcast_subscription(&self) -> Option { let SyncOpPayload::PodcastSubscriptionSet { podcast_id, subscribed, podcast, feed_aliases, + generation, } = &self.payload else { return None; @@ -729,6 +806,7 @@ impl SyncOpWire { subscribed: *subscribed, podcast: podcast.clone(), feed_aliases: feed_aliases.clone(), + generation: *generation, hlc_ms: self.hlc_ms, op_id: self.op_id.clone(), }; @@ -738,17 +816,37 @@ impl SyncOpWire { /// Share record written by a `podcast_share_set` operation. pub fn podcast_share(&self) -> Option { - let SyncOpPayload::PodcastShareSet { podcast_id, shared } = &self.payload else { + let SyncOpPayload::PodcastShareSet { + podcast_id, + shared, + generation, + } = &self.payload + else { return None; }; Some(ShareRecord { podcast_id: podcast_id.clone(), shared: *shared, + generation: *generation, hlc_ms: self.hlc_ms, op_id: self.op_id.clone(), }) } + /// Podcast and feed URLs of a `podcast_feed_aliases_added` operation. + /// + /// Merge them with [`crate::podcast::merge_feed_aliases`] (or + /// [`SubscriptionRecord::add_feed_aliases`]); the registers are untouched. + pub fn podcast_feed_aliases(&self) -> Option<(&PodcastId, &[FeedUrl])> { + match &self.payload { + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id, + feed_aliases, + } => Some((podcast_id, feed_aliases)), + _ => None, + } + } + /// Progress register written by a `podcast_progress_set` operation. /// /// `device_id` is the operation origin. @@ -789,7 +887,16 @@ impl SyncOpWire { /// Validates a podcast operation against the contract. /// /// Non-podcast operations are accepted unchanged. Receivers must skip - /// (but still acknowledge) invalid podcast operations. + /// (but still acknowledge) podcast operations that fail this check + /// (bounds, stamps, mismatched references). + /// + /// Ids ([`PodcastId`], [`EpisodeId`]) and feed URLs ([`FeedUrl`]) are + /// checked earlier, while the message is deserialized: a non-canonical id + /// or URL is a schema violation like a wrong JSON type, and the whole + /// message fails to parse. Conforming clients cannot produce one, because + /// these types only serialize canonical values and their rules are frozen + /// with the `v1` derivation domains; changing them requires a new protocol + /// version. pub fn validate_podcast(&self) -> PodcastResult<()> { let check_ref = |episode: &Option, episode_id: &EpisodeId, @@ -812,6 +919,15 @@ impl SyncOpWire { SyncOpPayload::PodcastShareSet { .. } => { self.podcast_share().expect("share payload").validate() } + SyncOpPayload::PodcastFeedAliasesAdded { feed_aliases, .. } => { + if feed_aliases.is_empty() || feed_aliases.len() > MAX_FEED_ALIASES + 1 { + return Err(PodcastError::InvalidField { + field: "feed_aliases", + reason: "must hold 1 to MAX_FEED_ALIASES + 1 URLs", + }); + } + Ok(()) + } SyncOpPayload::PodcastProgressSet { episode_id, podcast_id, @@ -960,7 +1076,8 @@ pub fn restore_jam_private_items( /// Podcast state (protocol v4) **is** included, bounded by /// [`SyncSnapshot::bound_podcast_state`]. Operations hidden from a peer while /// it ran an older version are never re-requested after it upgrades, because -/// sync vectors only grow, so snapshots are the recovery path. Send podcast +/// sync vectors only grow, so snapshots are the only recovery path for such a +/// peer, and only for the records that fit the snapshot bounds. Send podcast /// sections only to v4 peers ([`SyncSnapshot::for_peer`]). #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct SyncSnapshot { @@ -1019,31 +1136,75 @@ impl SyncSnapshot { snapshot } - /// Sorts and truncates the podcast sections to the snapshot limits. - /// - /// Each section is ordered newest first (subscriptions, shares and - /// progress by `hlc_ms`, sessions by `ended_at_ms`, ties by id) and capped - /// by its `MAX_SNAPSHOT_*` count. Sections then share - /// [`MAX_PODCAST_SNAPSHOT_BYTES`] of serialized JSON in priority order: - /// subscriptions, shares, progress, sessions. Older registers that do not - /// fit still propagate through operations. + /// Bounds the podcast sections with the default budget + /// [`MAX_PODCAST_SNAPSHOT_BYTES`]; see + /// [`SyncSnapshot::bound_podcast_state_to`]. pub fn bound_podcast_state(&mut self) { - let mut budget = MAX_PODCAST_SNAPSHOT_BYTES; + self.bound_podcast_state_to(MAX_PODCAST_SNAPSHOT_BYTES); + } + + /// Sorts and truncates the podcast sections to the snapshot limits and to + /// `budget_bytes` of serialized JSON. + /// + /// * Subscription records lose `podcast.description`: it is the largest + /// field and every device refreshes channel metadata from the feed. The + /// register (`subscribed`, stamps) and feed aliases are kept whole. + /// * Each section is ordered newest first (subscriptions and shares by + /// `(generation, hlc_ms)`, progress by `(generation, hlc_ms)`, sessions + /// by `ended_at_ms`; ties by id) and capped by its `MAX_SNAPSHOT_*` + /// count. + /// * Each section has its own share of the budget (subscriptions 25 %, + /// shares 5 %, progress 45 %, sessions 25 %), so large subscription + /// records cannot crowd out progress. Unused budget passes on to the + /// following sections in that order. + /// + /// Callers pass [`MAX_PODCAST_SNAPSHOT_BYTES`], or less when the rest of + /// the message (music snapshot, operations, playback) leaves less room + /// under [`MAX_SYNC_LINE`]; see [`podcast_snapshot_budget`]. + /// + /// Records cut here reach a peer only through operations it has not + /// acknowledged yet. A peer that skipped podcast operations while it ran + /// an older version never receives the cut records: it recovers the newest + /// records only, and its play counts can differ from other devices. + pub fn bound_podcast_state_to(&mut self, budget_bytes: usize) { + let share = |percent: usize| budget_bytes / 100 * percent; + let mut budget = 0usize; + for record in &mut self.podcast_subscriptions { + record.podcast.description = None; + } self.podcast_subscriptions.sort_by(|a, b| { - (b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&(a.hlc_ms, &a.op_id, &a.podcast_id)) + (b.generation, b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&( + a.generation, + a.hlc_ms, + &a.op_id, + &a.podcast_id, + )) }); + budget += share(25); bound_section( &mut self.podcast_subscriptions, MAX_SNAPSHOT_SUBSCRIPTIONS, &mut budget, ); self.podcast_shares.sort_by(|a, b| { - (b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&(a.hlc_ms, &a.op_id, &a.podcast_id)) + (b.generation, b.hlc_ms, &b.op_id, &b.podcast_id).cmp(&( + a.generation, + a.hlc_ms, + &a.op_id, + &a.podcast_id, + )) }); + budget += share(5); bound_section(&mut self.podcast_shares, MAX_SNAPSHOT_SHARES, &mut budget); self.podcast_progress.sort_by(|a, b| { - (b.hlc_ms, &b.op_id, &b.episode_id).cmp(&(a.hlc_ms, &a.op_id, &a.episode_id)) + (b.generation, b.hlc_ms, &b.op_id, &b.episode_id).cmp(&( + a.generation, + a.hlc_ms, + &a.op_id, + &a.episode_id, + )) }); + budget += share(45); bound_section( &mut self.podcast_progress, MAX_SNAPSHOT_PROGRESS, @@ -1051,6 +1212,7 @@ impl SyncSnapshot { ); self.podcast_listen_sessions .sort_by(|a, b| (b.ended_at_ms, &b.session_id).cmp(&(a.ended_at_ms, &a.session_id))); + budget += budget_bytes - share(25) - share(5) - share(45); bound_section( &mut self.podcast_listen_sessions, MAX_SNAPSHOT_LISTEN_SESSIONS, @@ -1070,6 +1232,17 @@ impl SyncSnapshot { } } +/// Byte budget for the podcast snapshot sections of one message. +/// +/// `other_bytes` is the serialized size of the message without podcast +/// sections. The result keeps the whole line under [`MAX_SYNC_LINE`] with a +/// 64 KiB margin and never exceeds [`MAX_PODCAST_SNAPSHOT_BYTES`]. +pub fn podcast_snapshot_budget(other_bytes: usize) -> usize { + MAX_SYNC_LINE + .saturating_sub(other_bytes.saturating_add(64 * 1024)) + .min(MAX_PODCAST_SNAPSHOT_BYTES) +} + fn bound_section(items: &mut Vec, max_items: usize, budget: &mut usize) { let mut keep = 0; for item in items.iter().take(max_items) { @@ -1559,6 +1732,7 @@ mod tests { season: Some(1), episode_number: Some(3), image_url: None, + progress_generation: Some(4), } } @@ -1649,10 +1823,16 @@ mod tests { image_url: None, }, feed_aliases: vec![feed()], + generation: 1, }, SyncOpPayload::PodcastShareSet { podcast_id: podcast_id(), shared: false, + generation: 2, + }, + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id: podcast_id(), + feed_aliases: vec![FeedUrl::parse("https://moved.example.test/feed").unwrap()], }, SyncOpPayload::PodcastProgressSet { episode_id: episode_id(), @@ -1675,6 +1855,7 @@ mod tests { let kinds = [ "podcast_subscription_set", "podcast_share_set", + "podcast_feed_aliases_added", "podcast_progress_set", "podcast_listen_recorded", ]; @@ -1698,14 +1879,25 @@ mod tests { (1_001, "dev-a:1") ); assert!(subscription.subscribed); + assert_eq!(subscription.generation, 1); let share = ops[1].podcast_share().unwrap(); assert!(!share.shared); - let progress = ops[2].podcast_progress().unwrap(); + assert_eq!(share.generation, 2); + let (alias_podcast, aliases) = ops[2].podcast_feed_aliases().unwrap(); + assert_eq!(alias_podcast, &podcast_id()); + let mut with_alias = subscription.clone(); + with_alias.add_feed_aliases(aliases); + assert!(with_alias.feed_aliases.contains(&aliases[0])); + // The alias operation never touches the register. + assert_eq!(with_alias.order_key(), subscription.order_key()); + assert_eq!(with_alias.subscribed, subscription.subscribed); + let progress = ops[3].podcast_progress().unwrap(); assert_eq!(progress.device_id, "dev-a"); - assert_eq!(progress.op_id, "dev-a:3"); + assert_eq!(progress.op_id, "dev-a:4"); assert_eq!(progress.generation, 3); - assert_eq!(ops[3].podcast_listen_session(), Some(&session())); + assert_eq!(ops[4].podcast_listen_session(), Some(&session())); assert!(ops[0].podcast_share().is_none()); + assert!(ops[0].podcast_feed_aliases().is_none()); } #[test] @@ -1719,16 +1911,21 @@ mod tests { _ => unreachable!(), }, feed_aliases: Vec::new(), + generation: 3, }; assert!(!unsubscribe.is_tombstone(), "unsubscribe is LWW state"); - assert!(ops[1].is_tombstone(), "share=false is a tombstone"); + // share=false protects privacy: compacting it would let a stale + // share=true delivered again republish the podcast. + assert!(!ops[1].is_tombstone(), "share=false is LWW state"); let share = SyncOpPayload::PodcastShareSet { podcast_id: podcast_id(), shared: true, + generation: 4, }; assert!(!share.is_tombstone()); - assert!(!ops[2].is_tombstone()); - assert!(!ops[3].is_tombstone()); + for payload in &ops { + assert!(!payload.is_tombstone(), "{payload:?}"); + } } #[test] @@ -1781,6 +1978,49 @@ mod tests { }, ); like.validate_podcast().unwrap(); + let empty_aliases = op( + 5, + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id: podcast_id(), + feed_aliases: Vec::new(), + }, + ); + assert!(empty_aliases.validate_podcast().is_err()); + } + + #[test] + fn malformed_ids_and_feed_urls_reject_the_whole_message() { + // Pins the documented behaviour: ids and feed URLs are validated + // while deserializing, so a non-canonical value is a schema + // violation of the whole message, not a skippable operation. + let like = op( + 1, + SyncOpPayload::TrackLikeSet { + content_id: "b3:x".to_string(), + liked: true, + fed: None, + }, + ); + let share = op( + 2, + SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id(), + shared: true, + generation: 1, + }, + ); + let batch = serde_json::to_string(&vec![like, share]).unwrap(); + serde_json::from_str::>(&batch).unwrap(); + let upper = batch.replace(podcast_id().as_str(), &podcast_id().as_str().to_uppercase()); + assert!(serde_json::from_str::>(&upper).is_err()); + let subscription = op(3, podcast_ops().remove(0)); + let json = serde_json::to_string(&subscription).unwrap(); + let non_canonical = json.replace( + "https://private.example.test/feed", + "HTTPS://private.example.test:443/feed", + ); + assert_ne!(json, non_canonical); + assert!(serde_json::from_str::(&non_canonical).is_err()); } #[test] @@ -2014,6 +2254,67 @@ mod tests { UNREPORTED_PROTOCOL_VERSION ); assert!(!supports_podcasts(profile_protocol_version(None, None))); + // A self-report may lower the version (downgrade). + assert_eq!(profile_protocol_version(Some(3), Some(4)), 3); + } + + #[test] + fn relayed_versions_only_raise_the_stored_version() { + // A deployed v3 relay forwards every device with its own constant. + assert_eq!(relayed_protocol_version(3, Some(4)), 4); + assert!(supports_podcasts(relayed_protocol_version(3, Some(4)))); + assert_eq!(relayed_protocol_version(4, Some(3)), 4); + assert_eq!(relayed_protocol_version(3, None), 3); + assert_eq!(relayed_protocol_version(1, Some(2)), 2); + } + + #[test] + fn pair_requests_gate_by_the_invite_version() { + let invite = InviteWire { + v: 1, + ticket: "t".to_string(), + device_id: "inviter".to_string(), + invite_id: "i".to_string(), + secret: "s".to_string(), + expires_at_ms: 1, + protocol_version: None, + }; + // Invites of older clients carry no version: assume the oldest. + let legacy = r#"{"v":1,"t":"t","d":"inviter","i":"i","s":"s","e":1}"#; + let parsed: InviteWire = serde_json::from_str(legacy).unwrap(); + assert_eq!(parsed, invite); + assert_eq!(serde_json::to_string(&invite).unwrap(), legacy); + assert_eq!( + parsed.inviter_protocol_version(), + UNREPORTED_PROTOCOL_VERSION + ); + let snapshot = SyncSnapshot { + podcast_listen_sessions: vec![session()], + ..SyncSnapshot::default() + }; + assert!( + !snapshot + .for_peer(parsed.inviter_protocol_version()) + .has_podcast_state() + ); + for payload in podcast_ops() { + assert!(!op_allowed_for_peer( + &payload, + parsed.inviter_protocol_version() + )); + } + assert!(!playback_allowed_for_peer( + &mixed(), + parsed.inviter_protocol_version() + )); + let current = InviteWire { + protocol_version: Some(DEVICE_SYNC_PROTOCOL_VERSION), + ..invite + }; + let link = encode_invite(¤t).unwrap(); + let back = parse_invite(&link).unwrap(); + assert_eq!(back, current); + assert!(supports_podcasts(back.inviter_protocol_version())); } #[test] @@ -2091,7 +2392,7 @@ mod tests { let snapshot = SyncSnapshot { podcast_subscriptions: vec![ops[0].podcast_subscription().unwrap()], podcast_shares: vec![ops[1].podcast_share().unwrap()], - podcast_progress: vec![ops[2].podcast_progress().unwrap()], + podcast_progress: vec![ops[3].podcast_progress().unwrap()], podcast_listen_sessions: vec![session()], ..SyncSnapshot::default() }; @@ -2130,6 +2431,7 @@ mod tests { &FeedUrl::parse(&format!("https://a.test/{i}")).unwrap(), ), shared: i % 2 == 0, + generation: 1, hlc_ms: i as i64, op_id: format!("dev:{i}"), }) @@ -2156,7 +2458,10 @@ mod tests { #[test] fn snapshot_byte_budget_is_enforced() { - let big = "x".repeat(crate::podcast::MAX_DESCRIPTION_CHARS); + // Worst-case subscription records: long Cyrillic title, author and + // description, plus a few progress registers. + let big: String = "я".repeat(crate::podcast::MAX_DESCRIPTION_CHARS); + let title: String = big.chars().take(crate::podcast::MAX_TITLE_CHARS).collect(); let subscriptions: Vec = (0..MAX_SNAPSHOT_SUBSCRIPTIONS) .map(|i| { let url = FeedUrl::parse(&format!("https://a.test/{i}")).unwrap(); @@ -2165,38 +2470,82 @@ mod tests { subscribed: true, podcast: PodcastMeta { feed_url: url, - title: big.chars().take(1_000).collect(), - author: None, + title: title.clone(), + author: Some(title.clone()), description: Some(big.clone()), link: None, image_url: None, }, feed_aliases: Vec::new(), + generation: 1, hlc_ms: i as i64, op_id: format!("dev:{i}"), } }) .collect(); - let mut snapshot = SyncSnapshot { + let progress: Vec = (0..10) + .map(|i| { + op( + i + 1, + SyncOpPayload::PodcastProgressSet { + episode_id: EpisodeId::from_guid(&podcast_id(), &format!("e{i}")) + .unwrap(), + podcast_id: podcast_id(), + position_ms: 1_000, + status: ProgressStatus::InProgress, + generation: 1, + session_id: "s".to_string(), + episode: None, + }, + ) + .podcast_progress() + .unwrap() + }) + .collect(); + let original = SyncSnapshot { podcast_subscriptions: subscriptions, + podcast_progress: progress, podcast_listen_sessions: vec![session()], ..SyncSnapshot::default() }; + let section_bytes = |snapshot: &SyncSnapshot| -> usize { + fn sum(items: &[T]) -> usize { + items + .iter() + .map(|item| serde_json::to_vec(item).unwrap().len() + 1) + .sum() + } + sum(&snapshot.podcast_subscriptions) + + sum(&snapshot.podcast_shares) + + sum(&snapshot.podcast_progress) + + sum(&snapshot.podcast_listen_sessions) + }; + let mut snapshot = original.clone(); snapshot.bound_podcast_state(); - let bytes: usize = snapshot - .podcast_subscriptions - .iter() - .map(|record| serde_json::to_vec(record).unwrap().len() + 1) - .sum::() - + snapshot - .podcast_listen_sessions + assert!(section_bytes(&snapshot) <= MAX_PODCAST_SNAPSHOT_BYTES); + assert!( + snapshot + .podcast_subscriptions .iter() - .map(|record| serde_json::to_vec(record).unwrap().len() + 1) - .sum::(); - assert!(bytes <= MAX_PODCAST_SNAPSHOT_BYTES); + .all(|record| record.podcast.description.is_none()), + "descriptions are refreshed from RSS, not snapshotted" + ); assert!(snapshot.podcast_subscriptions.len() < MAX_SNAPSHOT_SUBSCRIPTIONS); assert!(!snapshot.podcast_subscriptions.is_empty()); + // Subscriptions cannot crowd out progress and sessions. + assert_eq!(snapshot.podcast_progress.len(), 10); + assert_eq!(snapshot.podcast_listen_sessions.len(), 1); assert!(serde_json::to_vec(&snapshot).unwrap().len() < MAX_SYNC_LINE); + + // A smaller budget, for a message whose other parts are large. + let budget = podcast_snapshot_budget(6 * 1024 * 1024); + assert!(budget < MAX_PODCAST_SNAPSHOT_BYTES); + let mut small = original; + small.bound_podcast_state_to(budget); + assert!(section_bytes(&small) <= budget); + assert_eq!(small.podcast_progress.len(), 10); + assert_eq!(podcast_snapshot_budget(0), MAX_PODCAST_SNAPSHOT_BYTES); + assert_eq!(podcast_snapshot_budget(MAX_SYNC_LINE), 0); } } } diff --git a/crates/music-dht/src/jam.rs b/crates/music-dht/src/jam.rs index b208b42..f530948 100644 --- a/crates/music-dht/src/jam.rs +++ b/crates/music-dht/src/jam.rs @@ -314,6 +314,7 @@ mod tests { season: None, episode_number: None, image_url: None, + progress_generation: Some(7), }; let snapshot = PlaybackSnapshot { device_id: "host".to_string(), diff --git a/crates/music-dht/src/podcast/feed_url.rs b/crates/music-dht/src/podcast/feed_url.rs index ea13774..d553577 100644 --- a/crates/music-dht/src/podcast/feed_url.rs +++ b/crates/music-dht/src/podcast/feed_url.rs @@ -43,18 +43,25 @@ impl FeedUrl { /// /// Surrounding whitespace is trimmed, `feed:`, `podcast:`, `pcast:` and /// `itpc:` prefixes map to `https://` (or to the embedded `http(s)` URL), - /// and input without a scheme defaults to `https://`. + /// and input without a scheme defaults to `https://`. `http:` or `https:` + /// not followed by `//` (for example `https:/host/feed`) is rejected. pub fn from_user_input(input: &str) -> PodcastResult { let trimmed = input.trim(); if trimmed.is_empty() { return Err(invalid("feed URL is empty")); } let lower = trimmed.to_ascii_lowercase(); + if is_http_scheme_typo(&lower) { + return Err(invalid("feed URL must start with http:// or https://")); + } let mut candidate = trimmed.to_string(); for scheme in ["feed:", "podcast:", "pcast:", "itpc:"] { if lower.starts_with(scheme) { let rest = &trimmed[scheme.len()..]; let rest_lower = rest.to_ascii_lowercase(); + if is_http_scheme_typo(&rest_lower) { + return Err(invalid("feed URL must start with http:// or https://")); + } candidate = if rest_lower.starts_with("http://") || rest_lower.starts_with("https://") { rest.to_string() @@ -110,6 +117,13 @@ fn invalid(reason: &str) -> PodcastError { PodcastError::InvalidFeedUrl(reason.to_string()) } +/// `http:` or `https:` not followed by `//`, such as `https:/host/feed`. +fn is_http_scheme_typo(lower: &str) -> bool { + ["http:", "https:"] + .iter() + .any(|scheme| lower.starts_with(scheme) && !lower[scheme.len()..].starts_with("//")) +} + fn has_scheme(text: &str) -> bool { let Some((scheme, rest)) = text.split_once(':') else { return false; @@ -252,7 +266,8 @@ fn push_ascii(out: &mut String, value: &str) { /// /// `known` yields `(podcast_id, url)` pairs of every URL already associated /// with a podcast: the URL of its first subscription, its current fetch URL -/// and its feed aliases. When `url` matches, the existing id is returned (the +/// and its feed aliases ([`super::SubscriptionRecord::feed_urls`]; the bounded +/// alias set alone can miss the current fetch URL). When `url` matches, the existing id is returned (the /// smallest id if, after a concurrent offline subscription, several podcasts /// share the URL). Otherwise a new id is derived from `url`. /// @@ -277,7 +292,9 @@ pub fn resolve_podcast_id<'a>( /// derived is always kept; of the other URLs the [`MAX_FEED_ALIASES`] /// lexicographically smallest are kept. Taking the smallest `k` elements of a /// union is commutative, associative and idempotent, so devices converge -/// regardless of delivery order. +/// regardless of delivery order. Other URLs, including a podcast's current +/// fetch URL, can fall outside the bound; the materialized subscription keeps +/// its `podcast.feed_url` separately. pub fn merge_feed_aliases(podcast: &PodcastId, a: &[FeedUrl], b: &[FeedUrl]) -> Vec { let mut origin = None; let mut rest = BTreeSet::new(); @@ -437,6 +454,12 @@ mod tests { "https://", "file:///etc/passwd", "javascript:alert(1)", + // One-slash typos must not become a host named "https". + "https:/a.test/feed", + "http:/a.test/feed", + "HTTPS:a.test/feed", + "https:///a.test/feed", + "feed:https:/a.test/feed", ] { assert!(FeedUrl::from_user_input(bad).is_err(), "{bad:?}"); } diff --git a/crates/music-dht/src/podcast/meta.rs b/crates/music-dht/src/podcast/meta.rs index 1f36ff7..aa036bf 100644 --- a/crates/music-dht/src/podcast/meta.rs +++ b/crates/music-dht/src/podcast/meta.rs @@ -227,6 +227,7 @@ impl EpisodeMeta { episode_number: self.episode_number, image_url: valid_url(self.image_url.as_deref()) .or_else(|| valid_url(podcast.image_url.as_deref())), + progress_generation: None, } } } @@ -236,8 +237,9 @@ impl EpisodeMeta { /// Carried by playback queues (`PlaybackSource::PodcastEpisode`) and by /// progress operations, so a receiving trusted device can display, resume and /// play the episode without being subscribed and without its own copy of the -/// feed. It contains no personal state. It is private: Jam snapshots replace it -/// with an opaque placeholder. +/// feed. Apart from the optional handoff hint +/// [`progress_generation`](Self::progress_generation) it contains no personal +/// state. It is private: Jam snapshots replace it with an opaque placeholder. /// /// If the audio URL went stale, refresh the feed at `feed_url` and look the /// episode up by `episode_id`. @@ -273,6 +275,17 @@ pub struct PodcastEpisodeRef { /// Episode or podcast artwork URL. #[serde(default)] pub image_url: Option, + /// Greatest progress generation the sender has seen for this episode. + /// + /// Set on queue items of a playback state (in particular on handoff), + /// because the playback state can arrive before the sender's progress + /// operations. A device that starts playing the item writes its explicit + /// action with + /// [`ProgressRecord::explicit_generation`](super::ProgressRecord::explicit_generation)`(max(local, progress_generation))`. + /// Ignored in progress and listening operations, whose own generation is + /// authoritative. Not part of the episode identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub progress_generation: Option, } impl PodcastEpisodeRef { diff --git a/crates/music-dht/src/podcast/mod.rs b/crates/music-dht/src/podcast/mod.rs index d4b2bb6..4daec10 100644 --- a/crates/music-dht/src/podcast/mod.rs +++ b/crates/music-dht/src/podcast/mod.rs @@ -28,11 +28,13 @@ //! //! # Personal state //! -//! * [`SubscriptionRecord`]: last-writer-wins by `(hlc_ms, op_id)`; -//! unsubscribing is state, not a tombstone. Feed aliases merge as a bounded -//! grow-only set. -//! * [`ShareRecord`]: last-writer-wins; `shared = false` is a tombstone. See -//! [`effective_share`] for the rule that a (re)subscription is private. +//! * [`SubscriptionRecord`]: last-writer-wins by `(generation, hlc_ms, op_id)` +//! where `generation` is a per-podcast Lamport counter +//! ([`next_podcast_generation`]); unsubscribing is state, not a tombstone. +//! Feed aliases merge as a bounded grow-only set. +//! * [`ShareRecord`]: last-writer-wins with the same ordering; `shared = false` +//! is register state that is never compacted. See [`effective_share`] for +//! the rule that a (re)subscription is private. //! * [`ProgressRecord`]: register ordered by //! `(generation, hlc_ms, device_id, op_id)`. Explicit user actions bump the //! generation to `max_seen + 1`; periodic checkpoints keep the generation of @@ -85,7 +87,8 @@ pub use meta::{EpisodeMeta, EpisodeType, PodcastEpisodeRef, PodcastMeta}; pub use state::{ COMPLETION_TAIL_MAX_MS, ListenSession, MIN_RECORDED_EPISODE_LISTEN_MS, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, completed_listen_count, effective_share, - merge_listen_session, merge_progress, merge_share, merge_subscription, reaches_completion, + merge_listen_session, merge_progress, merge_share, merge_subscription, next_podcast_generation, + reaches_completion, }; /// Maximum characters of a title (podcast, episode, author). diff --git a/crates/music-dht/src/podcast/state.rs b/crates/music-dht/src/podcast/state.rs index c258eb7..f238995 100644 --- a/crates/music-dht/src/podcast/state.rs +++ b/crates/music-dht/src/podcast/state.rs @@ -57,15 +57,29 @@ pub enum ProgressStatus { /// seeking backwards and "start over" are real changes. /// /// * Explicit user actions (start of a new listening session, seek, "start -/// over", "mark played", "mark unplayed", handoff to another device) use -/// [`ProgressRecord::explicit_generation`]: one more than the greatest -/// generation the device has seen for the episode. This is a Lamport clock, -/// so wall-clock skew cannot undo an explicit action that was observed. +/// over", "mark played", "mark unplayed", taking over playback through a +/// handoff) use [`ProgressRecord::explicit_generation`]: one more than the +/// greatest generation the device has seen for the episode. This is a +/// Lamport clock, so wall-clock skew cannot undo an explicit action that was +/// observed. An explicit action during a session becomes the generation of +/// that session. /// * Periodic checkpoints keep the generation of the latest explicit action of /// their session. A late checkpoint of an older session therefore loses /// against any newer explicit action, whatever its `hlc_ms`. /// * Concurrent explicit actions on offline devices receive the same /// generation and are ordered by `hlc_ms`, then `device_id`, then `op_id`. +/// * Handoff: only the receiving device writes the explicit action, with +/// `explicit_generation(max(local, carried))` where `carried` is +/// [`PodcastEpisodeRef::progress_generation`](super::PodcastEpisodeRef::progress_generation) +/// of the queue item. The sender writes a final checkpoint at its session +/// generation. +/// * A device that observes a greater generation for the episode it is +/// playing stops writing at its old generation: either playback moved away, +/// or the user keeps listening locally and the device writes its next +/// record as an explicit action. +/// * Outgoing-log coalescing may drop a pending progress operation only when +/// another pending operation of the same episode [supersedes](Self::supersedes) +/// it; dropping by sequence order could discard an explicit action. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ProgressRecord { /// Episode the register belongs to. @@ -162,18 +176,25 @@ pub fn merge_progress( /// Personal subscription state of one podcast. /// /// `subscribed` and `podcast` form one last-writer-wins register ordered by -/// `(hlc_ms, op_id)`. Unsubscribing writes `subscribed = false`: it is state, -/// not a tombstone, and never deletes progress or history, so subscribing -/// again continues where the user stopped. +/// `(generation, hlc_ms, op_id)`. `generation` is a Lamport counter shared by +/// the subscription and share registers of the podcast (see +/// [`next_podcast_generation`]), so an action taken after observing another +/// device's write wins whatever the device clocks say. Unsubscribing writes +/// `subscribed = false`: it is state, not a tombstone, is never compacted, and +/// never deletes progress or history, so subscribing again continues where +/// the user stopped. /// /// Only explicit user actions (subscribe, unsubscribe) write this register. /// Background RSS refreshes keep their metadata locally and must not emit a /// subscription operation; otherwise a stale device could resurrect an -/// unsubscribed podcast. +/// unsubscribed podcast. Feed aliases learned by refreshes travel in their +/// own grow-only operation. /// /// `feed_aliases` is a bounded grow-only set merged independently of the -/// register (see [`merge_feed_aliases`]); it always contains every -/// `podcast.feed_url` seen. +/// register (see [`merge_feed_aliases`]): the URL the id was derived from +/// plus the [`MAX_FEED_ALIASES`] smallest other URLs. The current +/// `podcast.feed_url` may fall outside that bound, so look podcasts up by +/// [`SubscriptionRecord::feed_urls`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SubscriptionRecord { /// Podcast. @@ -185,6 +206,9 @@ pub struct SubscriptionRecord { /// Known feed URLs of the podcast. #[serde(default)] pub feed_aliases: Vec, + /// Lamport generation shared with the share register (see type docs). + #[serde(default)] + pub generation: u64, /// Logical write time. pub hlc_ms: i64, /// Writing operation id. @@ -192,9 +216,14 @@ pub struct SubscriptionRecord { } impl SubscriptionRecord { + /// Register ordering key: `(generation, hlc_ms, op_id)`. + pub fn order_key(&self) -> (u64, i64, &str) { + (self.generation, self.hlc_ms, self.op_id.as_str()) + } + fn total_cmp(&self, other: &Self) -> Ordering { - (self.hlc_ms, self.op_id.as_str()) - .cmp(&(other.hlc_ms, other.op_id.as_str())) + self.order_key() + .cmp(&other.order_key()) .then_with(|| self.subscribed.cmp(&other.subscribed)) .then_with(|| self.podcast.cmp(&other.podcast)) } @@ -204,7 +233,10 @@ impl SubscriptionRecord { self.total_cmp(other) == Ordering::Greater } - /// Ensures `feed_aliases` is sorted, bounded and contains `podcast.feed_url`. + /// Merges `podcast.feed_url` into `feed_aliases` and bounds the set. + /// + /// The result is sorted and bounded by [`merge_feed_aliases`]; it may not + /// contain `podcast.feed_url` when many smaller aliases are known. pub fn normalize(&mut self) { self.feed_aliases = merge_feed_aliases( &self.podcast_id, @@ -213,6 +245,23 @@ impl SubscriptionRecord { ); } + /// Adds aliases learned through a `podcast_feed_aliases_added` operation. + /// + /// Only the grow-only alias set changes; the register is untouched. + pub fn add_feed_aliases(&mut self, aliases: &[FeedUrl]) { + self.feed_aliases = merge_feed_aliases(&self.podcast_id, &self.feed_aliases, aliases); + } + + /// Every URL that identifies this podcast: the aliases and the current + /// `podcast.feed_url`, sorted and without duplicates. + /// + /// Stores pass these pairs to [`super::resolve_podcast_id`]. + pub fn feed_urls(&self) -> Vec { + let mut urls: BTreeSet = self.feed_aliases.iter().cloned().collect(); + urls.insert(self.podcast.feed_url.clone()); + urls.into_iter().collect() + } + /// Validates contract bounds. pub fn validate(&self) -> PodcastResult<()> { self.podcast.validate()?; @@ -251,8 +300,12 @@ pub fn merge_subscription( /// Decision to publish a podcast card to the federation. /// -/// Last-writer-wins by `(hlc_ms, op_id)`; `shared = false` is the tombstone -/// that revokes the publication. Unsubscribing must also write +/// Last-writer-wins by `(generation, hlc_ms, op_id)`, with `generation` the +/// per-podcast Lamport counter shared with [`SubscriptionRecord`] (see +/// [`next_podcast_generation`]). `shared = false` revokes the publication. +/// It is privacy-protecting register state, **not** a tombstone: like an +/// unsubscribe it is never compacted, otherwise a stale `shared = true` +/// delivered again would republish the podcast. Unsubscribing must also write /// `shared = false`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ShareRecord { @@ -260,6 +313,9 @@ pub struct ShareRecord { pub podcast_id: PodcastId, /// Whether the user chose to share the podcast. pub shared: bool, + /// Lamport generation shared with the subscription register. + #[serde(default)] + pub generation: u64, /// Logical write time. pub hlc_ms: i64, /// Writing operation id. @@ -267,12 +323,15 @@ pub struct ShareRecord { } impl ShareRecord { + /// Register ordering key: `(generation, hlc_ms, op_id)`. + pub fn order_key(&self) -> (u64, i64, &str) { + (self.generation, self.hlc_ms, self.op_id.as_str()) + } + fn total_cmp(&self, other: &Self) -> Ordering { - (self.hlc_ms, self.op_id.as_str(), self.shared).cmp(&( - other.hlc_ms, - other.op_id.as_str(), - other.shared, - )) + self.order_key() + .cmp(&other.order_key()) + .then_with(|| self.shared.cmp(&other.shared)) } /// Returns whether this record wins over `other`. @@ -298,13 +357,36 @@ pub fn merge_share(current: &ShareRecord, incoming: &ShareRecord) -> PodcastResu }) } +/// Generation for a new write of the subscription or share register. +/// +/// One more than the greatest generation of either register of the podcast +/// (`None` when it has none), so a write made after observing another +/// device's subscription, unsubscription or share decision orders after it +/// even when that device's clock runs ahead. Concurrent writes on devices +/// that did not see each other receive the same generation and are ordered +/// by `hlc_ms`, then `op_id`. +pub fn next_podcast_generation( + subscription: Option<&SubscriptionRecord>, + share: Option<&ShareRecord>, +) -> u64 { + subscription + .map(|record| record.generation) + .into_iter() + .chain(share.map(|record| record.generation)) + .max() + .map_or(1, |generation| generation.saturating_add(1)) +} + /// Whether a podcast must currently be published. /// /// True only when the user is subscribed, chose to share, and the share -/// decision is newer than the latest subscription write. A new subscription -/// (including re-subscribing, or subscribing from someone else's public card) -/// is therefore always private until the user shares it again, and a stale -/// device cannot republish a podcast through an old share decision. +/// decision orders after the latest subscription write (by +/// `(generation, hlc_ms, op_id)`). A new subscription (including +/// re-subscribing, or subscribing from someone else's public card) is +/// therefore always private until the user shares it again, and a stale +/// device cannot republish a podcast through an old share decision. A share +/// decision taken after observing the subscription always applies, because it +/// carries a greater generation. pub fn effective_share( subscription: Option<&SubscriptionRecord>, share: Option<&ShareRecord>, @@ -314,8 +396,7 @@ pub fn effective_share( subscription.subscribed && share.shared && subscription.podcast_id == share.podcast_id - && (share.hlc_ms, share.op_id.as_str()) - > (subscription.hlc_ms, subscription.op_id.as_str()) + && share.order_key() > subscription.order_key() } _ => false, } diff --git a/crates/music-dht/src/podcast/state_tests.rs b/crates/music-dht/src/podcast/state_tests.rs index 2bbf60a..8110268 100644 --- a/crates/music-dht/src/podcast/state_tests.rs +++ b/crates/music-dht/src/podcast/state_tests.rs @@ -282,6 +282,7 @@ fn subscription( subscribed, podcast: podcast_meta(title, url), feed_aliases: Vec::new(), + generation: 1, hlc_ms, op_id: op.to_string(), }; @@ -289,6 +290,12 @@ fn subscription( record } +fn with_generation(record: &T, set: impl Fn(&mut T)) -> T { + let mut record = record.clone(); + set(&mut record); + record +} + #[test] fn subscriptions_converge_for_every_permutation() { let ops = [ @@ -330,13 +337,14 @@ fn share(shared: bool, hlc_ms: i64, op: &str) -> ShareRecord { ShareRecord { podcast_id: podcast_id(), shared, + generation: 1, hlc_ms, op_id: op.to_string(), } } #[test] -fn shares_converge_and_false_is_a_tombstone() { +fn shares_converge_and_false_is_never_beaten_by_a_stale_true() { let ops = [ share(true, 100, "dev-a:1"), share(false, 200, "dev-a:2"), @@ -364,6 +372,129 @@ fn effective_share_rules() { assert!(!effective_share(Some(&unsubscribed), Some(&stale))); } +#[test] +fn clock_skew_cannot_undo_an_observed_unsubscribe() { + // Device A's clock runs ten minutes ahead. B sees A's subscription, then + // the user unsubscribes on B. + let sub_a = subscription(true, 1_000_600_000, "dev-a:1", "Show", "feed"); + assert_eq!(next_podcast_generation(None, None), 1); + let generation = next_podcast_generation(Some(&sub_a), None); + assert_eq!(generation, 2); + let unsub_b = with_generation( + &subscription(false, 1_000_060_000, "dev-b:1", "Show", "feed"), + |r| r.generation = generation, + ); + let merged = assert_converges(&[sub_a.clone(), unsub_b.clone()], merge_subscription); + assert!(!merged.subscribed, "the later unsubscribe wins"); + // Concurrent writes (same generation) still fall back to the clock. + let concurrent = with_generation(&unsub_b, |r| r.generation = 1); + assert!(merge_subscription(&sub_a, &concurrent).unwrap().subscribed); +} + +#[test] +fn share_after_subscribe_applies_despite_clock_skew() { + let sub_a = subscription(true, 1_000_600_000, "dev-a:1", "Show", "feed"); + let share_b = with_generation(&share(true, 1_000_060_000, "dev-b:1"), |r| { + r.generation = next_podcast_generation(Some(&sub_a), None) + }); + assert!(effective_share(Some(&sub_a), Some(&share_b))); + // Unsubscribe written after the share (higher generation) revokes it, + // and so does a later re-subscription until the user shares again. + let unsub = with_generation(&subscription(false, 1, "dev-a:2", "Show", "feed"), |r| { + r.generation = next_podcast_generation(Some(&sub_a), Some(&share_b)) + }); + assert!(!effective_share(Some(&unsub), Some(&share_b))); + let resub = with_generation(&subscription(true, 2, "dev-a:3", "Show", "feed"), |r| { + r.generation = next_podcast_generation(Some(&unsub), Some(&share_b)) + }); + assert!(!effective_share(Some(&resub), Some(&share_b))); + let reshare = with_generation(&share(true, 3, "dev-a:4"), |r| { + r.generation = next_podcast_generation(Some(&resub), Some(&share_b)) + }); + assert!(effective_share(Some(&resub), Some(&reshare))); + // Revoking (shared = false) with a higher generation wins over every + // stale true, in any order. + let revoke = with_generation(&share(false, 0, "dev-b:2"), |r| { + r.generation = next_podcast_generation(Some(&resub), Some(&reshare)) + }); + let merged = assert_converges(&[share_b, reshare, revoke.clone()], merge_share); + assert_eq!(merged, revoke); +} + +#[test] +fn current_feed_url_stays_resolvable_beyond_the_alias_bound() { + let mut record = subscription(true, 1, "dev-a:1", "Show", "feed"); + record.add_feed_aliases( + &(0..=crate::podcast::MAX_FEED_ALIASES) + .map(|i| feed_url(&format!("a{i}"))) + .collect::>(), + ); + record.podcast.feed_url = FeedUrl::parse("https://z.test/current").unwrap(); + record.normalize(); + assert!(!record.feed_aliases.contains(&record.podcast.feed_url)); + let urls = record.feed_urls(); + assert!(urls.contains(&record.podcast.feed_url)); + assert!(urls.contains(&feed_url("feed")), "origin is kept"); + let known: Vec<(&PodcastId, &FeedUrl)> = + urls.iter().map(|url| (&record.podcast_id, url)).collect(); + assert_eq!( + crate::podcast::resolve_podcast_id(&record.podcast.feed_url, known), + podcast_id() + ); +} + +#[test] +fn handoff_receiver_continues_with_the_carried_generation() { + use ProgressStatus::*; + // Sender A plays session s1 at generation 5 with a fast clock; its + // progress operations have not reached B when the handoff arrives. + let a_checkpoint = progress(5, 9_000_000, "dev-a", "dev-a:9", 600_000, InProgress, "s1"); + let carried = Some(a_checkpoint.generation); + let b_local = Some(2); + let generation = ProgressRecord::explicit_generation(b_local.max(carried)); + assert_eq!(generation, 6); + let b_start = progress( + generation, 1_000, "dev-b", "dev-b:1", 600_000, InProgress, "s2", + ); + let b_pause = progress( + generation, 2_000, "dev-b", "dev-b:2", 660_000, InProgress, "s2", + ); + // Without the carried generation B would write generation 3 and lose + // every write to A's older checkpoint. + let uninformed = progress( + ProgressRecord::explicit_generation(b_local), + 2_000, + "dev-b", + "dev-b:2", + 660_000, + InProgress, + "s2", + ); + assert!(a_checkpoint.supersedes(&uninformed)); + let merged = assert_converges(&[a_checkpoint, b_start, b_pause.clone()], merge_progress); + assert_eq!(merged, b_pause); + assert_eq!(merged.resume_position_ms(), 660_000); +} + +#[test] +fn coalescing_keeps_the_superseding_record_not_the_last_queued() { + use ProgressStatus::*; + // "Mark played" (explicit, generation 5) during a session, followed in + // the outgoing log by a checkpoint of the old session (generation 3). + let mark_played = progress(5, 100, "dev-a", "dev-a:7", 0, Completed, "mark"); + let late_checkpoint = progress(3, 101, "dev-a", "dev-a:8", 30_000, InProgress, "s1"); + let pending = [mark_played.clone(), late_checkpoint.clone()]; + // Keep only records that no other pending record supersedes. + let kept: Vec<&ProgressRecord> = pending + .iter() + .filter(|record| !pending.iter().any(|other| other.supersedes(record))) + .collect(); + assert_eq!(kept, vec![&mark_played]); + // Dropping by sequence order would have published the wrong state. + assert_ne!(pending.last(), Some(&mark_played)); + assert_eq!(fold(&pending, merge_progress), mark_played); +} + /// Minimal client store used to show that whole operation sets converge and /// that publisher metadata never reaches personal progress. #[derive(Debug, Default, Clone, PartialEq)]