From e22c470ea766b2cd155eb2e0b4ae944c309b2a93 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 16:42:52 +0200 Subject: [PATCH] Add SQLite podcast store with prototype import New `furumi_library::podcasts::PodcastStore` in its own podcasts.sqlite3: - user_version migrations in immediate transactions; version 1 is the prototype schema of the original podcast PR, version 2 renames its tables to legacy_* and imports feeds, episodes and fetch state with contract ids; unimportable rows go to legacy_quarantine; newer or foreign files are refused before anything is written; - catalog (podcasts, feed aliases, seasons, episodes), subscription, share and progress registers, listening sessions and local-only fetch state; - local actions that apply and return podcast sync operations, remote operation and snapshot application through the music-dht materializers and merge functions, bounded snapshot export; - RSS refresh ingestion with stable episode ids and alias learning that never touches personal state, and queries for the UI. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/furumi-library/Cargo.toml | 5 +- crates/furumi-library/src/lib.rs | 4 + crates/furumi-library/src/podcasts/actions.rs | 544 +++++ crates/furumi-library/src/podcasts/apply.rs | 742 +++++++ crates/furumi-library/src/podcasts/mod.rs | 451 +++++ crates/furumi-library/src/podcasts/queries.rs | 605 ++++++ crates/furumi-library/src/podcasts/refresh.rs | 465 +++++ crates/furumi-library/src/podcasts/schema.rs | 729 +++++++ crates/furumi-library/src/podcasts/tests.rs | 1760 +++++++++++++++++ 10 files changed, 5305 insertions(+), 1 deletion(-) create mode 100644 crates/furumi-library/src/podcasts/actions.rs create mode 100644 crates/furumi-library/src/podcasts/apply.rs create mode 100644 crates/furumi-library/src/podcasts/mod.rs create mode 100644 crates/furumi-library/src/podcasts/queries.rs create mode 100644 crates/furumi-library/src/podcasts/refresh.rs create mode 100644 crates/furumi-library/src/podcasts/schema.rs create mode 100644 crates/furumi-library/src/podcasts/tests.rs diff --git a/Cargo.lock b/Cargo.lock index f22946e..7344f6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -942,6 +942,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "tempfile", "tracing", ] diff --git a/crates/furumi-library/Cargo.toml b/crates/furumi-library/Cargo.toml index 2c44a11..12c3e98 100644 --- a/crates/furumi-library/Cargo.toml +++ b/crates/furumi-library/Cargo.toml @@ -16,8 +16,11 @@ anyhow = { workspace = true } blake3 = { workspace = true } directories = { workspace = true } lofty = { workspace = true } -music-dht = { workspace = true } +music-dht = { workspace = true, features = ["podcast-feed"] } rusqlite = { workspace = true, features = ["functions"] } serde = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/furumi-library/src/lib.rs b/crates/furumi-library/src/lib.rs index 9522957..fecd2e4 100644 --- a/crates/furumi-library/src/lib.rs +++ b/crates/furumi-library/src/lib.rs @@ -6,9 +6,13 @@ //! //! All methods take `&self` and lock the connection internally, so an //! `Arc` can be shared across blocking tasks. +//! +//! Podcasts live in a separate database, [`podcasts::PodcastStore`] +//! (`podcasts.sqlite3`); the music library never stores episodes. pub mod import; pub mod models; +pub mod podcasts; use std::collections::{HashMap, HashSet}; use std::io::Write as _; diff --git a/crates/furumi-library/src/podcasts/actions.rs b/crates/furumi-library/src/podcasts/actions.rs new file mode 100644 index 0000000..1e766a4 --- /dev/null +++ b/crates/furumi-library/src/podcasts/actions.rs @@ -0,0 +1,544 @@ +//! Local user actions: they build the device-sync operation, apply it through +//! the shared code path and return it for the client's sync log. + +use anyhow::Result; +use music_dht::device_sync::SyncOpPayload; +use music_dht::podcast::{ + EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord, + ProgressStatus, effective_share, merge_feed_aliases, merge_listen_session, + next_podcast_generation, +}; +use rusqlite::{Connection, TransactionBehavior}; + +use super::apply::{self, ApplyOutcome}; +use super::{OpStamp, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource}; + +/// Result of [`PodcastStore::subscribe`]. +#[derive(Debug, Clone, PartialEq)] +pub struct SubscribeOutcome { + /// Podcast the URL resolved to (an existing one when the URL is a known + /// feed URL or alias). + pub podcast_id: PodcastId, + /// The podcast was already subscribed; no operation was emitted. + pub already_subscribed: bool, + /// Operations to append to the sync log. + pub ops: Vec, +} + +/// A listening session of this device as seen by the progress register. +/// +/// Returned by explicit actions; periodic checkpoints of the session use its +/// `generation` ([`PodcastStore::checkpoint`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProgressSession { + /// Episode being played. + pub episode: PodcastEpisodeRef, + /// Session id (also the `ListenSession::session_id`). + pub session_id: String, + /// Generation of the session's latest explicit action. + pub generation: u64, +} + +/// A progress register written by a local action. +#[derive(Debug, Clone, PartialEq)] +pub struct ProgressWrite { + /// The register as written. + pub record: ProgressRecord, + /// The session after the action (its generation for later checkpoints). + pub session: ProgressSession, + /// Operation to append to the sync log. + pub op: PodcastLocalOp, +} + +/// Result of [`PodcastStore::checkpoint`]. +#[derive(Debug, Clone, PartialEq)] +pub enum CheckpointOutcome { + /// The checkpoint was written. + Written(Box), + /// The register holds a greater generation (another device or an + /// explicit action took over). Nothing was written: stop writing at the + /// old generation, or continue locally with an explicit action + /// ([`PodcastStore::start_session`] / [`PodcastStore::seek`]). + Superseded { + /// Current register. + current: ProgressRecord, + }, + /// This session already completed the episode at its generation; a + /// checkpoint would turn it back into "in progress". Nothing was written. + AlreadyCompleted { + /// Current register. + current: ProgressRecord, + }, +} + +#[derive(Debug, Clone, Copy)] +enum ExplicitAction { + Start(i64), + Seek(i64), + Restart, + Complete(i64), + MarkPlayed, + MarkUnplayed, +} + +fn invalid(reason: impl ToString) -> anyhow::Error { + PodcastStoreError::Invalid(reason.to_string()).into() +} + +/// Validates the payload (with a provisional stamp), draws the real stamp, +/// and applies the operation through [`apply::apply_op`]. +pub(crate) fn emit( + conn: &Connection, + stamps: &mut dyn StampSource, + make: impl Fn(&OpStamp) -> SyncOpPayload, +) -> Result { + let device_id = stamps.device_id().to_string(); + let provisional = OpStamp::new("provisional", device_id.clone(), 0); + provisional + .wire(make(&provisional)) + .validate_podcast() + .map_err(invalid)?; + let stamp = stamps.next_stamp()?; + if stamp.origin_device_id != device_id { + return Err(invalid("stamp origin differs from the local device id")); + } + let payload = make(&stamp); + let op = stamp.wire(payload.clone()); + match apply::apply_op(conn, &op)? { + ApplyOutcome::Rejected(reason) => Err(invalid(reason)), + _ => Ok(PodcastLocalOp { stamp, payload }), + } +} + +/// Resolves a feed URL against every known URL of every podcast. +pub(crate) fn resolve_podcast(conn: &Connection, url: &FeedUrl) -> Result { + let mut statement = conn.prepare_cached( + "SELECT podcast_id FROM podcast_feed_aliases WHERE feed_url = ?1 + UNION SELECT podcast_id FROM podcasts WHERE feed_url = ?1 + UNION SELECT podcast_id FROM podcast_subscriptions + WHERE json_extract(podcast_json, '$.feed_url') = ?1", + )?; + let ids = statement + .query_map([url.as_str()], |row| super::podcast_id_col(row.get(0)?))? + .collect::>>()?; + let pairs: Vec<(PodcastId, FeedUrl)> = ids.into_iter().map(|id| (id, url.clone())).collect(); + Ok(music_dht::podcast::resolve_podcast_id( + url, + pairs.iter().map(|(id, url)| (id, url)), + )) +} + +fn catalog_meta(conn: &Connection, podcast_id: &PodcastId) -> Result> { + use rusqlite::OptionalExtension as _; + Ok(conn + .query_row( + "SELECT feed_url, title, author, description, link, image_url + FROM podcasts WHERE podcast_id = ?1", + [podcast_id.as_str()], + |row| { + Ok(PodcastMeta { + feed_url: super::feed_url_col(row.get(0)?)?, + title: row.get(1)?, + author: row.get(2)?, + description: row.get(3)?, + link: row.get(4)?, + image_url: row.get(5)?, + }) + }, + ) + .optional()?) +} + +fn share_payload(conn: &Connection, podcast_id: &PodcastId, shared: bool) -> Result { + let subscription = apply::load_subscription_register(conn, podcast_id)?; + let share = apply::load_share(conn, podcast_id)?; + Ok(SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id.clone(), + shared, + generation: next_podcast_generation(subscription.as_ref(), share.as_ref()), + }) +} + +/// The reference carried by progress operations: identity and display only. +fn op_ref(episode: &PodcastEpisodeRef) -> PodcastEpisodeRef { + let mut episode = episode.clone(); + episode.progress_generation = None; + episode +} + +impl PodcastStore { + /// Subscribes to the podcast of `podcast.feed_url` (explicit user action). + /// + /// The URL is resolved against known feed URLs and aliases first, so + /// subscribing again, through a redirect target or from someone else's + /// card reuses the existing podcast with its progress and history. When + /// the podcast is already subscribed nothing is emitted. Otherwise one + /// `podcast_subscription_set` with generation + /// `next_podcast_generation(..)` and the known aliases is applied and + /// returned. The new subscription is private until + /// [`PodcastStore::set_shared`]. + /// + /// Pass the channel metadata of the fetched feed + /// (`ParsedFeed::to_podcast_meta(url)`), then ingest the episodes with + /// [`PodcastStore::apply_feed`]. + pub fn subscribe( + &self, + stamps: &mut dyn StampSource, + podcast: &PodcastMeta, + ) -> Result { + let meta = podcast.bounded(); + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let podcast_id = resolve_podcast(&tx, &meta.feed_url)?; + let current = apply::load_subscription_register(&tx, &podcast_id)?; + if current.as_ref().is_some_and(|record| record.subscribed) { + return Ok(SubscribeOutcome { + podcast_id, + already_subscribed: true, + ops: Vec::new(), + }); + } + let share = apply::load_share(&tx, &podcast_id)?; + let generation = next_podcast_generation(current.as_ref(), share.as_ref()); + let aliases = merge_feed_aliases( + &podcast_id, + &apply::load_aliases(&tx, &podcast_id)?, + std::slice::from_ref(&meta.feed_url), + ); + let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id.clone(), + subscribed: true, + podcast: meta.clone(), + feed_aliases: aliases.clone(), + generation, + })?; + tx.commit()?; + Ok(SubscribeOutcome { + podcast_id, + already_subscribed: false, + ops: vec![op], + }) + } + + /// Unsubscribes (explicit user action). + /// + /// Emits `podcast_subscription_set { subscribed: false }` followed by + /// `podcast_share_set { shared: false }`, each with the next podcast + /// generation. Progress, sessions and the catalog are kept, so + /// subscribing again continues where the user stopped. Does nothing when + /// the podcast is not subscribed. + pub fn unsubscribe( + &self, + stamps: &mut dyn StampSource, + podcast_id: &PodcastId, + ) -> Result> { + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let Some(current) = apply::load_subscription_register(&tx, podcast_id)? else { + return Ok(Vec::new()); + }; + if !current.subscribed { + return Ok(Vec::new()); + } + let share = apply::load_share(&tx, podcast_id)?; + let generation = next_podcast_generation(Some(¤t), share.as_ref()); + let mut meta = catalog_meta(&tx, podcast_id)? + .map(|meta| meta.bounded()) + .unwrap_or_else(|| current.podcast.clone()); + meta.feed_url = current.podcast.feed_url.clone(); + let aliases = current.feed_aliases.clone(); + let unsubscribe = emit(&tx, stamps, |_| SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id.clone(), + subscribed: false, + podcast: meta.clone(), + feed_aliases: aliases.clone(), + generation, + })?; + let revoke_payload = share_payload(&tx, podcast_id, false)?; + let revoke = emit(&tx, stamps, |_| revoke_payload.clone())?; + tx.commit()?; + Ok(vec![unsubscribe, revoke]) + } + + /// Sets the decision to publish the podcast to the federation. + /// + /// Sharing needs an active subscription and is a no-op when the podcast + /// is already effectively shared (`effective_share`). Revoking is a no-op + /// when no share decision says `true`. The write uses the next podcast + /// generation, so it orders after the subscription it observed. + pub fn set_shared( + &self, + stamps: &mut dyn StampSource, + podcast_id: &PodcastId, + shared: bool, + ) -> Result> { + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let subscription = apply::load_subscription_register(&tx, podcast_id)?; + let share = apply::load_share(&tx, podcast_id)?; + if shared { + if !subscription + .as_ref() + .is_some_and(|record| record.subscribed) + { + return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into()); + } + if effective_share(subscription.as_ref(), share.as_ref()) { + return Ok(Vec::new()); + } + } else if !share.as_ref().is_some_and(|record| record.shared) { + return Ok(Vec::new()); + } + let payload = share_payload(&tx, podcast_id, shared)?; + let op = emit(&tx, stamps, |_| payload.clone())?; + tx.commit()?; + Ok(vec![op]) + } + + fn explicit_progress( + &self, + stamps: &mut dyn StampSource, + episode: &PodcastEpisodeRef, + session_id: Option<&str>, + carried_generation: Option, + action: ExplicitAction, + ) -> Result { + episode.validate().map_err(invalid)?; + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = apply::load_progress(&tx, &episode.episode_id)?; + let max_seen = current + .as_ref() + .map(|record| record.generation) + .into_iter() + .chain(carried_generation) + .max(); + let generation = ProgressRecord::explicit_generation(max_seen); + let current_position = current.as_ref().map_or(0, |record| record.position_ms); + let (position_ms, status) = match action { + ExplicitAction::Start(position) | ExplicitAction::Seek(position) => { + (position, ProgressStatus::InProgress) + } + ExplicitAction::Restart => (0, ProgressStatus::InProgress), + ExplicitAction::Complete(position) => (position, ProgressStatus::Completed), + ExplicitAction::MarkPlayed => (current_position, ProgressStatus::Completed), + ExplicitAction::MarkUnplayed => (0, ProgressStatus::NotStarted), + }; + let reference = op_ref(episode); + let op = emit(&tx, stamps, |stamp| SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms, + status, + generation, + session_id: session_id.unwrap_or(&stamp.op_id).to_string(), + episode: Some(reference.clone()), + })?; + tx.commit()?; + let wire = op.stamp.wire(op.payload.clone()); + let record = wire.podcast_progress().expect("progress payload"); + Ok(ProgressWrite { + session: ProgressSession { + episode: episode.clone(), + session_id: record.session_id.clone(), + generation, + }, + record, + op, + }) + } + + /// Starts a listening session at `position_ms` (explicit action). + /// + /// The generation is `explicit_generation(max(local, carried_generation))`; + /// pass `PodcastEpisodeRef::progress_generation` of a queue item received + /// through a handoff as `carried_generation`. Ordinary Play resumes from + /// [`PodcastStore::resume_position_ms`]. + pub fn start_session( + &self, + stamps: &mut dyn StampSource, + episode: &PodcastEpisodeRef, + session_id: &str, + position_ms: i64, + carried_generation: Option, + ) -> Result { + self.explicit_progress( + stamps, + episode, + Some(session_id), + carried_generation, + ExplicitAction::Start(position_ms), + ) + } + + /// Periodic checkpoint (or pause/stop save) of a running session. + /// + /// Keeps the session generation. When the register already holds a + /// greater generation nothing is written ([`CheckpointOutcome::Superseded`]). + pub fn checkpoint( + &self, + stamps: &mut dyn StampSource, + session: &ProgressSession, + position_ms: i64, + ) -> Result { + let episode = &session.episode; + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + if let Some(current) = apply::load_progress(&tx, &episode.episode_id)? { + if current.generation > session.generation { + return Ok(CheckpointOutcome::Superseded { current }); + } + if current.generation == session.generation + && current.session_id == session.session_id + && current.status == ProgressStatus::Completed + { + return Ok(CheckpointOutcome::AlreadyCompleted { current }); + } + } + let reference = op_ref(episode); + let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms, + status: ProgressStatus::InProgress, + generation: session.generation, + session_id: session.session_id.clone(), + episode: Some(reference.clone()), + })?; + tx.commit()?; + let record = op + .stamp + .wire(op.payload.clone()) + .podcast_progress() + .expect("progress payload"); + Ok(CheckpointOutcome::Written(Box::new(ProgressWrite { + record, + session: session.clone(), + op, + }))) + } + + /// User seek during a session (explicit action; the returned session + /// carries the new generation for later checkpoints). + pub fn seek( + &self, + stamps: &mut dyn StampSource, + session: &ProgressSession, + position_ms: i64, + ) -> Result { + self.explicit_progress( + stamps, + &session.episode, + Some(&session.session_id), + Some(session.generation), + ExplicitAction::Seek(position_ms), + ) + } + + /// The session reached the end (`reaches_completion` or end of stream): + /// writes `completed` with an explicit generation. Record the session + /// itself with [`PodcastStore::record_listen_session`]. + pub fn complete_session( + &self, + stamps: &mut dyn StampSource, + session: &ProgressSession, + position_ms: i64, + ) -> Result { + self.explicit_progress( + stamps, + &session.episode, + Some(&session.session_id), + Some(session.generation), + ExplicitAction::Complete(position_ms), + ) + } + + /// "Start over": position 0, in progress, explicit generation. + pub fn restart( + &self, + stamps: &mut dyn StampSource, + episode: &PodcastEpisodeRef, + session_id: &str, + ) -> Result { + self.explicit_progress( + stamps, + episode, + Some(session_id), + None, + ExplicitAction::Restart, + ) + } + + /// "Mark played": status completed, explicit generation. Creates no + /// listening session and invents no listening time. + pub fn mark_played( + &self, + stamps: &mut dyn StampSource, + episode: &PodcastEpisodeRef, + ) -> Result { + self.explicit_progress(stamps, episode, None, None, ExplicitAction::MarkPlayed) + } + + /// "Mark unplayed": status not started at position 0, explicit generation. + pub fn mark_unplayed( + &self, + stamps: &mut dyn StampSource, + episode: &PodcastEpisodeRef, + ) -> Result { + self.explicit_progress(stamps, episode, None, None, ExplicitAction::MarkUnplayed) + } + + /// Records (or grows) a listening session of this device. + /// + /// Sessions below the history threshold (`ListenSession::should_record`) + /// and versions that do not grow the stored session emit nothing. + /// `session.device_id` must be the local device id. The episode reference + /// defaults to the catalog's. + pub fn record_listen_session( + &self, + stamps: &mut dyn StampSource, + session: &ListenSession, + episode: Option<&PodcastEpisodeRef>, + ) -> Result> { + if session.device_id != stamps.device_id() { + return Err(invalid( + "session device_id differs from the local device id", + )); + } + session.validate().map_err(invalid)?; + if !session.should_record() { + return Ok(None); + } + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + if let Some(current) = apply::load_session(&tx, &session.session_id)? { + let merged = merge_listen_session(¤t, session).map_err(invalid)?; + if merged == current { + return Ok(None); + } + } + let reference = match episode { + Some(episode) => Some(op_ref(episode)), + None => super::queries::episode_ref_in(&tx, &session.episode_id)?.map(|r| op_ref(&r)), + }; + let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastListenRecorded { + session: session.clone(), + episode: reference.clone(), + })?; + tx.commit()?; + Ok(Some(op)) + } + + /// Resolves a feed URL to a podcast id: the existing podcast when the + /// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise + /// the id a first subscription through this URL would derive. + pub fn resolve_podcast_id(&self, url: &FeedUrl) -> Result { + resolve_podcast(&self.lock(), url) + } + + /// The episode reference for queueing or progress actions, or `None` + /// when the episode is unknown. + pub fn episode_ref(&self, episode_id: &EpisodeId) -> Result> { + super::queries::episode_ref_in(&self.lock(), episode_id) + } +} diff --git a/crates/furumi-library/src/podcasts/apply.rs b/crates/furumi-library/src/podcasts/apply.rs new file mode 100644 index 0000000..c5bfb4d --- /dev/null +++ b/crates/furumi-library/src/podcasts/apply.rs @@ -0,0 +1,742 @@ +//! The single code path that applies podcast records to the store. +//! +//! Remote operations, local actions and snapshot sections all end here. +//! Records come from the `SyncOpWire` materializers of +//! `music_dht::device_sync` and are merged with the functions of +//! `music_dht::podcast`, so applying is idempotent and order-independent. + +use anyhow::Result; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot}; +use music_dht::podcast::{ + EpisodeIdentityHints, EpisodeMeta, FeedUrl, KnownEpisode, ListenSession, PodcastEpisodeRef, + PodcastError, PodcastId, ProgressRecord, ShareRecord, SubscriptionRecord, merge_feed_aliases, + merge_listen_session, +}; +use rusqlite::{Connection, OptionalExtension as _, TransactionBehavior, params}; + +use super::{ + PROGRESS_COLUMNS, PodcastStore, PodcastStoreError, SESSION_COLUMNS, episode_id_col, + feed_url_col, generation_to_sql, now_ms, podcast_meta_col, progress_from_row, session_from_row, + status_to_sql, +}; + +/// Result of applying one operation. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum ApplyOutcome { + /// Stored state changed. + Changed, + /// The operation was already reflected (duplicate, or older than the + /// stored register). + Unchanged, + /// Not a podcast operation; nothing to do here. + Ignored, + /// The operation violates the contract and was skipped. Acknowledge it + /// anyway so the log keeps moving (`PODCASTS.md`). + Rejected(String), +} + +/// Counters returned by [`PodcastStore::apply_snapshot`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SnapshotApplyStats { + /// Records that changed stored state. + pub changed: usize, + /// Valid records already reflected locally. + pub unchanged: usize, + /// Records that failed validation and were skipped. + pub rejected: usize, + /// Whether the podcast sections were ignored for exceeding the counts. + pub ignored_over_limits: bool, +} + +/// Where an episode row came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EpisodeSource { + /// Parsed from the podcast's feed on this device. + Feed, + /// Learned from a `PodcastEpisodeRef` (sync operation or queue item). + Sync, +} + +impl EpisodeSource { + fn as_sql(self) -> &'static str { + match self { + Self::Feed => "feed", + Self::Sync => "sync", + } + } +} + +fn invalid(error: impl ToString) -> anyhow::Error { + PodcastStoreError::Invalid(error.to_string()).into() +} + +pub(crate) fn load_subscription_register( + conn: &Connection, + podcast_id: &PodcastId, +) -> Result> { + let row = conn + .query_row( + "SELECT subscribed, podcast_json, generation, hlc_ms, op_id + FROM podcast_subscriptions WHERE podcast_id = ?1", + [podcast_id.as_str()], + |row| { + let generation: i64 = row.get(2)?; + Ok(SubscriptionRecord { + podcast_id: podcast_id.clone(), + subscribed: row.get(0)?, + podcast: podcast_meta_col(&row.get::<_, String>(1)?)?, + feed_aliases: Vec::new(), + generation: u64::try_from(generation).unwrap_or(0), + hlc_ms: row.get(3)?, + op_id: row.get(4)?, + }) + }, + ) + .optional()?; + let Some(mut record) = row else { + return Ok(None); + }; + record.feed_aliases = load_aliases(conn, podcast_id)?; + Ok(Some(record)) +} + +pub(crate) fn load_aliases(conn: &Connection, podcast_id: &PodcastId) -> Result> { + let mut statement = conn.prepare_cached( + "SELECT feed_url FROM podcast_feed_aliases WHERE podcast_id = ?1 ORDER BY feed_url", + )?; + let urls = statement + .query_map([podcast_id.as_str()], |row| feed_url_col(row.get(0)?))? + .collect::>>()?; + Ok(urls) +} + +pub(crate) fn load_share(conn: &Connection, podcast_id: &PodcastId) -> Result> { + Ok(conn + .query_row( + "SELECT shared, generation, hlc_ms, op_id FROM podcast_shares WHERE podcast_id = ?1", + [podcast_id.as_str()], + |row| { + let generation: i64 = row.get(1)?; + Ok(ShareRecord { + podcast_id: podcast_id.clone(), + shared: row.get(0)?, + generation: u64::try_from(generation).unwrap_or(0), + hlc_ms: row.get(2)?, + op_id: row.get(3)?, + }) + }, + ) + .optional()?) +} + +pub(crate) fn load_progress( + conn: &Connection, + episode_id: &music_dht::podcast::EpisodeId, +) -> Result> { + Ok(conn + .query_row( + &format!("SELECT {PROGRESS_COLUMNS} FROM episode_progress p WHERE p.episode_id = ?1"), + [episode_id.as_str()], + |row| progress_from_row(row, 0), + ) + .optional()?) +} + +pub(crate) fn load_session(conn: &Connection, session_id: &str) -> Result> { + Ok(conn + .query_row( + &format!( + "SELECT {SESSION_COLUMNS} FROM podcast_listen_sessions s WHERE s.session_id = ?1" + ), + [session_id], + |row| session_from_row(row, 0), + ) + .optional()?) +} + +/// Replaces the alias set of a podcast with `merged` when it differs. +fn store_aliases(conn: &Connection, podcast_id: &PodcastId, merged: &[FeedUrl]) -> Result { + let current = load_aliases(conn, podcast_id)?; + if current == merged { + return Ok(false); + } + conn.execute( + "DELETE FROM podcast_feed_aliases WHERE podcast_id = ?1", + [podcast_id.as_str()], + )?; + let mut insert = conn.prepare_cached( + "INSERT INTO podcast_feed_aliases (podcast_id, feed_url) VALUES (?1, ?2)", + )?; + for url in merged { + insert.execute(params![podcast_id.as_str(), url.as_str()])?; + } + Ok(true) +} + +/// Merges feed URLs into the bounded grow-only alias set of a podcast. +/// +/// Returns whether the stored set changed. +pub(crate) fn merge_aliases( + conn: &Connection, + podcast_id: &PodcastId, + urls: &[FeedUrl], +) -> Result { + let current = load_aliases(conn, podcast_id)?; + let merged = merge_feed_aliases(podcast_id, ¤t, urls); + store_aliases(conn, podcast_id, &merged) +} + +/// Inserts the catalog row of a podcast if it does not exist yet. +fn ensure_podcast_row( + conn: &Connection, + podcast_id: &PodcastId, + feed_url: &FeedUrl, + title: &str, + now: i64, +) -> Result<()> { + conn.execute( + "INSERT OR IGNORE INTO podcasts + (podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms) + VALUES (?1, ?2, ?3, 0, ?4, ?4)", + params![podcast_id.as_str(), feed_url.as_str(), title, now], + )?; + Ok(()) +} + +/// Applies a subscription register record (operation or snapshot). +/// +/// The register is last-writer-wins ([`SubscriptionRecord::supersedes`]); +/// the alias set is a bounded grow-only set merged independently, so the +/// stored aliases are the bounded union of every URL seen (equal to what +/// `merge_subscription` computes). The catalog row is created from the +/// register metadata and follows the winning register until this device has +/// refreshed the feed itself. +pub(crate) fn apply_subscription_record( + conn: &Connection, + record: &SubscriptionRecord, + now: i64, +) -> Result { + generation_to_sql(record.generation)?; + let podcast_id = &record.podcast_id; + let current = load_subscription_register(conn, podcast_id)?; + let wins = current + .as_ref() + .is_none_or(|current| record.supersedes(current)); + let mut changed = false; + if wins { + conn.execute( + "INSERT INTO podcast_subscriptions + (podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(podcast_id) DO UPDATE SET + subscribed = excluded.subscribed, podcast_json = excluded.podcast_json, + generation = excluded.generation, hlc_ms = excluded.hlc_ms, + op_id = excluded.op_id", + params![ + podcast_id.as_str(), + record.subscribed, + serde_json::to_string(&record.podcast)?, + generation_to_sql(record.generation)?, + record.hlc_ms, + record.op_id, + ], + )?; + changed = true; + } + let mut urls = record.feed_aliases.clone(); + urls.push(record.podcast.feed_url.clone()); + changed |= merge_aliases(conn, podcast_id, &urls)?; + + ensure_podcast_row( + conn, + podcast_id, + &record.podcast.feed_url, + &record.podcast.title, + now, + )?; + if wins { + let meta = &record.podcast; + conn.execute( + "UPDATE podcasts + SET feed_url = ?2, title = ?3, author = ?4, + description = COALESCE(?5, description), link = ?6, image_url = ?7, + updated_at_ms = ?8 + WHERE podcast_id = ?1 AND meta_from_feed = 0", + params![ + podcast_id.as_str(), + meta.feed_url.as_str(), + meta.title, + meta.author, + meta.description, + meta.link, + meta.image_url, + now + ], + )?; + } + Ok(changed) +} + +/// Applies a share register record. +pub(crate) fn apply_share_record(conn: &Connection, record: &ShareRecord) -> Result { + let generation = generation_to_sql(record.generation)?; + let current = load_share(conn, &record.podcast_id)?; + if current + .as_ref() + .is_some_and(|current| !record.supersedes(current)) + { + return Ok(false); + } + conn.execute( + "INSERT INTO podcast_shares (podcast_id, shared, generation, hlc_ms, op_id) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(podcast_id) DO UPDATE SET + shared = excluded.shared, generation = excluded.generation, + hlc_ms = excluded.hlc_ms, op_id = excluded.op_id", + params![ + record.podcast_id.as_str(), + record.shared, + generation, + record.hlc_ms, + record.op_id + ], + )?; + Ok(true) +} + +/// Applies a progress register record. Never touches the catalog. +pub(crate) fn apply_progress_record(conn: &Connection, record: &ProgressRecord) -> Result { + let generation = generation_to_sql(record.generation)?; + let current = load_progress(conn, &record.episode_id)?; + if current + .as_ref() + .is_some_and(|current| !record.supersedes(current)) + { + return Ok(false); + } + conn.execute( + "INSERT INTO episode_progress + (episode_id, podcast_id, position_ms, status, generation, session_id, hlc_ms, + device_id, op_id) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) + ON CONFLICT(episode_id) DO UPDATE SET + podcast_id = excluded.podcast_id, position_ms = excluded.position_ms, + status = excluded.status, generation = excluded.generation, + session_id = excluded.session_id, hlc_ms = excluded.hlc_ms, + device_id = excluded.device_id, op_id = excluded.op_id", + params![ + record.episode_id.as_str(), + record.podcast_id.as_str(), + record.position_ms, + status_to_sql(record.status), + generation, + record.session_id, + record.hlc_ms, + record.device_id, + record.op_id, + ], + )?; + Ok(true) +} + +/// Adds or grows a listening session (grow-only set keyed by `session_id`). +pub(crate) fn apply_listen_session(conn: &Connection, session: &ListenSession) -> Result { + let current = load_session(conn, &session.session_id)?; + let merged = match ¤t { + Some(current) => merge_listen_session(current, session).map_err(invalid)?, + None => session.clone(), + }; + if current.as_ref() == Some(&merged) { + return Ok(false); + } + conn.execute( + "INSERT INTO podcast_listen_sessions + (session_id, episode_id, podcast_id, device_id, started_at_ms, ended_at_ms, + start_position_ms, end_position_ms, listened_ms, completed) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT(session_id) DO UPDATE SET + started_at_ms = excluded.started_at_ms, ended_at_ms = excluded.ended_at_ms, + start_position_ms = excluded.start_position_ms, + end_position_ms = excluded.end_position_ms, + listened_ms = excluded.listened_ms, completed = excluded.completed", + params![ + merged.session_id, + merged.episode_id.as_str(), + merged.podcast_id.as_str(), + merged.device_id, + merged.started_at_ms, + merged.ended_at_ms, + merged.start_position_ms, + merged.end_position_ms, + merged.listened_ms, + merged.completed, + ], + )?; + Ok(true) +} + +/// Inserts an episode row. +pub(crate) fn insert_episode( + conn: &Connection, + meta: &EpisodeMeta, + source: EpisodeSource, + in_feed: bool, + legacy_guid: Option<&str>, + first_seen_at_ms: i64, + now: i64, +) -> Result<()> { + conn.execute( + "INSERT INTO episodes + (episode_id, podcast_id, guid, title, description, season, episode_number, + episode_type, published_at_ms, duration_ms, audio_url, mime, byte_len, + image_url, link, source, in_feed, legacy_guid, first_seen_at_ms, updated_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, + ?18, ?19, ?20)", + params![ + meta.episode_id.as_str(), + meta.podcast_id.as_str(), + meta.guid, + meta.title, + meta.description, + meta.season, + meta.episode_number, + episode_type_to_sql(meta.episode_type), + meta.published_at_ms, + meta.duration_ms, + meta.audio_url, + meta.mime, + meta.byte_len.and_then(|len| i64::try_from(len).ok()), + meta.image_url, + meta.link, + source.as_sql(), + in_feed, + legacy_guid, + first_seen_at_ms, + now, + ], + )?; + Ok(()) +} + +pub(crate) fn episode_type_to_sql(kind: music_dht::podcast::EpisodeType) -> &'static str { + match kind { + music_dht::podcast::EpisodeType::Full => "full", + music_dht::podcast::EpisodeType::Trailer => "trailer", + music_dht::podcast::EpisodeType::Bonus => "bonus", + } +} + +/// Records an episode learned through sync when the catalog does not know it. +/// +/// Feed metadata always wins: an existing row is left untouched. The +/// podcast catalog row is created from the reference when missing (no +/// register is written). +pub(crate) fn ensure_episode_from_ref( + conn: &Connection, + episode: &PodcastEpisodeRef, +) -> Result<()> { + let exists = conn + .query_row( + "SELECT 1 FROM episodes WHERE episode_id = ?1", + [episode.episode_id.as_str()], + |_| Ok(()), + ) + .optional()? + .is_some(); + if exists { + return Ok(()); + } + let now = now_ms(); + ensure_podcast_row( + conn, + &episode.podcast_id, + &episode.feed_url, + &episode.podcast_title, + now, + )?; + let meta = EpisodeMeta { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + guid: None, + title: episode.episode_title.clone(), + description: None, + season: episode.season, + episode_number: episode.episode_number, + episode_type: Default::default(), + published_at_ms: episode.published_at_ms, + duration_ms: episode.duration_ms, + audio_url: episode.audio_url.clone(), + mime: episode.mime.clone(), + byte_len: None, + image_url: episode.image_url.clone(), + link: None, + }; + insert_episode(conn, &meta, EpisodeSource::Sync, false, None, now, now)?; + refresh_seasons(conn, &episode.podcast_id)?; + Ok(()) +} + +/// Recomputes the derived season rows of a podcast from its episodes. +pub(crate) fn refresh_seasons(conn: &Connection, podcast_id: &PodcastId) -> Result<()> { + conn.execute( + "DELETE FROM podcast_seasons WHERE podcast_id = ?1", + [podcast_id.as_str()], + )?; + conn.execute( + "INSERT INTO podcast_seasons + (podcast_id, season, episode_count, first_published_at_ms, last_published_at_ms) + SELECT podcast_id, season, COUNT(*), MIN(published_at_ms), MAX(published_at_ms) + FROM episodes WHERE podcast_id = ?1 GROUP BY season", + [podcast_id.as_str()], + )?; + Ok(()) +} + +/// Known episodes of a podcast with the identity hints they were last seen with. +pub(crate) fn known_episodes( + conn: &Connection, + podcast_id: &PodcastId, +) -> Result> { + let mut statement = conn.prepare_cached( + "SELECT episode_id, guid, audio_url, link, published_at_ms, title + FROM episodes WHERE podcast_id = ?1 ORDER BY episode_id", + )?; + let known = statement + .query_map([podcast_id.as_str()], |row| { + Ok(KnownEpisode { + episode_id: episode_id_col(row.get(0)?)?, + hints: EpisodeIdentityHints { + guid: row.get(1)?, + enclosure_url: Some(row.get(2)?), + link: row.get(3)?, + published_at_ms: row.get(4)?, + title: Some(row.get(5)?), + }, + }) + })? + .collect::>>()?; + Ok(known) +} + +/// Applies one podcast operation inside the caller's transaction. +pub(crate) fn apply_op(conn: &Connection, op: &SyncOpWire) -> Result { + if !op.payload.is_podcast() { + return Ok(ApplyOutcome::Ignored); + } + if let Err(error) = op.validate_podcast() { + return Ok(ApplyOutcome::Rejected(error.to_string())); + } + let result = match &op.payload { + SyncOpPayload::PodcastSubscriptionSet { .. } => { + let record = op.podcast_subscription().expect("subscription payload"); + apply_subscription_record(conn, &record, now_ms()) + } + SyncOpPayload::PodcastShareSet { .. } => { + apply_share_record(conn, &op.podcast_share().expect("share payload")) + } + SyncOpPayload::PodcastFeedAliasesAdded { .. } => { + let (podcast_id, urls) = op.podcast_feed_aliases().expect("alias payload"); + merge_aliases(conn, podcast_id, urls) + } + SyncOpPayload::PodcastProgressSet { episode, .. } => { + let record = op.podcast_progress().expect("progress payload"); + if let Err(error) = generation_to_sql(record.generation) { + return Ok(ApplyOutcome::Rejected(error.to_string())); + } + if let Some(episode) = episode { + ensure_episode_from_ref(conn, episode)?; + } + apply_progress_record(conn, &record) + } + SyncOpPayload::PodcastListenRecorded { session, episode } => { + if let Some(current) = load_session(conn, &session.session_id)? + && let Err(error) = merge_listen_session(¤t, session) + { + return Ok(ApplyOutcome::Rejected(error.to_string())); + } + if let Some(episode) = episode { + ensure_episode_from_ref(conn, episode)?; + } + apply_listen_session(conn, session) + } + _ => return Ok(ApplyOutcome::Ignored), + }; + match result { + Ok(true) => Ok(ApplyOutcome::Changed), + Ok(false) => Ok(ApplyOutcome::Unchanged), + Err(error) => match error.downcast_ref::() { + Some(PodcastStoreError::Invalid(reason)) => Ok(ApplyOutcome::Rejected(reason.clone())), + _ => Err(error), + }, + } +} + +fn record_outcome(stats: &mut SnapshotApplyStats, result: Result) -> Result<()> { + match result { + Ok(true) => stats.changed += 1, + Ok(false) => stats.unchanged += 1, + Err(error) => match error.downcast_ref::() { + Some(PodcastStoreError::Invalid(_)) => stats.rejected += 1, + _ => return Err(error), + }, + } + Ok(()) +} + +fn check(result: Result<(), PodcastError>) -> Result<()> { + result.map_err(invalid) +} + +impl PodcastStore { + /// Applies a device-sync operation received from a trusted device. + /// + /// Non-podcast operations return [`ApplyOutcome::Ignored`]. Podcast + /// operations are validated (`SyncOpWire::validate_podcast`) and merged + /// with the contract functions in one transaction; applying the same + /// operations in any order, with duplicates, gives the same state. + pub fn apply_remote_op(&self, op: &SyncOpWire) -> Result { + if !op.payload.is_podcast() { + return Ok(ApplyOutcome::Ignored); + } + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let outcome = apply_op(&tx, op)?; + tx.commit()?; + Ok(outcome) + } + + /// Merges the podcast sections of a sync snapshot. + /// + /// Sections exceeding the contract counts are ignored + /// (`SyncSnapshot::podcast_state_within_limits`); every record is + /// validated and merged with the same functions as operations, so a + /// snapshot never replaces newer local state. One transaction. + pub fn apply_snapshot(&self, snapshot: &SyncSnapshot) -> Result { + let mut stats = SnapshotApplyStats::default(); + if !snapshot.podcast_state_within_limits() { + stats.ignored_over_limits = true; + return Ok(stats); + } + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let now = now_ms(); + for record in &snapshot.podcast_subscriptions { + let mut record = record.clone(); + record.normalize(); + let result = check(record.validate()) + .and_then(|()| apply_subscription_record(&tx, &record, now)); + record_outcome(&mut stats, result)?; + } + for record in &snapshot.podcast_shares { + let result = check(record.validate()).and_then(|()| apply_share_record(&tx, record)); + record_outcome(&mut stats, result)?; + } + for record in &snapshot.podcast_progress { + let result = check(record.validate()).and_then(|()| apply_progress_record(&tx, record)); + record_outcome(&mut stats, result)?; + } + for session in &snapshot.podcast_listen_sessions { + let result = + check(session.validate()).and_then(|()| apply_listen_session(&tx, session)); + record_outcome(&mut stats, result)?; + } + tx.commit()?; + Ok(stats) + } + + /// Fills the podcast sections of `snapshot` with this store's registers + /// and sessions, bounded by `SyncSnapshot::bound_podcast_state_to`. + /// + /// `budget_bytes` is usually `MAX_PODCAST_SNAPSHOT_BYTES` or + /// `device_sync::podcast_snapshot_budget(other_bytes)`. Records are read + /// newest first and capped by the per-section counts before the byte + /// budget is applied. Local fetch state and the catalog are never part + /// of the snapshot. Send it only to v4 peers (`SyncSnapshot::for_peer`). + pub fn fill_snapshot(&self, snapshot: &mut SyncSnapshot, budget_bytes: usize) -> Result<()> { + use music_dht::podcast::{ + MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, + MAX_SNAPSHOT_SUBSCRIPTIONS, + }; + let conn = self.lock(); + let ids: Vec = { + let mut statement = conn.prepare( + "SELECT podcast_id FROM podcast_subscriptions + ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC + LIMIT ?1", + )?; + statement + .query_map([MAX_SNAPSHOT_SUBSCRIPTIONS as i64], |row| { + super::podcast_id_col(row.get(0)?) + })? + .collect::>>()? + }; + let mut subscriptions = Vec::with_capacity(ids.len()); + for id in &ids { + if let Some(record) = load_subscription_register(&conn, id)? { + subscriptions.push(record); + } + } + let shares = { + let mut statement = conn.prepare( + "SELECT podcast_id, shared, generation, hlc_ms, op_id FROM podcast_shares + ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC + LIMIT ?1", + )?; + statement + .query_map([MAX_SNAPSHOT_SHARES as i64], |row| { + let generation: i64 = row.get(2)?; + Ok(ShareRecord { + podcast_id: super::podcast_id_col(row.get(0)?)?, + shared: row.get(1)?, + generation: u64::try_from(generation).unwrap_or(0), + hlc_ms: row.get(3)?, + op_id: row.get(4)?, + }) + })? + .collect::>>()? + }; + let progress = { + let mut statement = conn.prepare(&format!( + "SELECT {PROGRESS_COLUMNS} FROM episode_progress p + ORDER BY p.generation DESC, p.hlc_ms DESC, p.op_id DESC, p.episode_id DESC + LIMIT ?1" + ))?; + statement + .query_map([MAX_SNAPSHOT_PROGRESS as i64], |row| { + progress_from_row(row, 0) + })? + .collect::>>()? + }; + let sessions = { + let mut statement = conn.prepare(&format!( + "SELECT {SESSION_COLUMNS} FROM podcast_listen_sessions s + ORDER BY s.ended_at_ms DESC, s.session_id DESC + LIMIT ?1" + ))?; + statement + .query_map([MAX_SNAPSHOT_LISTEN_SESSIONS as i64], |row| { + session_from_row(row, 0) + })? + .collect::>>()? + }; + drop(conn); + snapshot.podcast_subscriptions = subscriptions; + snapshot.podcast_shares = shares; + snapshot.podcast_progress = progress; + snapshot.podcast_listen_sessions = sessions; + snapshot.bound_podcast_state_to(budget_bytes); + Ok(()) + } + + /// A snapshot holding only this store's podcast sections, bounded with + /// the default budget (`MAX_PODCAST_SNAPSHOT_BYTES`). + pub fn export_snapshot(&self) -> Result { + let mut snapshot = SyncSnapshot::default(); + self.fill_snapshot( + &mut snapshot, + music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES, + )?; + Ok(snapshot) + } +} diff --git a/crates/furumi-library/src/podcasts/mod.rs b/crates/furumi-library/src/podcasts/mod.rs new file mode 100644 index 0000000..98116ee --- /dev/null +++ b/crates/furumi-library/src/podcasts/mod.rs @@ -0,0 +1,451 @@ +//! SQLite reference store for podcasts (`podcasts.sqlite3`). +//! +//! [`PodcastStore`] persists the podcast contract of `music_dht::podcast` +//! for SQLite clients: the local episode catalog refreshed from RSS, the +//! synchronized personal registers (subscription, share, progress), the +//! grow-only listening sessions and the local-only fetch state. It owns no +//! networking, HTTP, scheduling or audio: clients fetch feeds, parse them +//! with `music_dht::podcast::feed`, append sync operations to their own log +//! and exchange snapshots. The normative rules live in the workspace +//! `PODCASTS.md` (section "SQLite reference store" describes this schema). +//! +//! # Files and migrations +//! +//! The store lives in its own file, conventionally [`PODCAST_DB_FILE`] next +//! to the library database; the caller passes the path. Connections use +//! WAL, foreign keys and a busy timeout. `PRAGMA user_version` drives the +//! migrations, each applied in its own `BEGIN IMMEDIATE` transaction that +//! re-checks the version, so concurrent openers and interrupted upgrades are +//! safe: +//! +//! | `user_version` | Schema | +//! | --- | --- | +//! | 0 | empty file | +//! | 1 | prototype schema of the original podcast PR (`podcasts`, `podcast_episodes`) | +//! | 2 | contract model; the prototype tables are renamed `legacy_*` and imported | +//! +//! A database written by a newer Furumi ([`PODCAST_SCHEMA_VERSION`] lower +//! than its `user_version`) or a file that is not a podcast database is +//! refused before anything is written ([`PodcastStoreError`]). +//! +//! # Personal state and sync +//! +//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...) +//! draw operation stamps from a [`StampSource`], build the device-sync +//! operation, apply it through the same code path as +//! [`PodcastStore::apply_remote_op`] (the `SyncOpWire` materializers of +//! `music_dht::device_sync` and the merge functions of `music_dht::podcast`) +//! and return the [`PodcastLocalOp`]s. The caller appends them to its sync +//! log **with exactly these stamps**; applying them again is a no-op. +//! Applying operations and snapshot sections is idempotent and independent +//! of delivery order. +//! +//! RSS refreshes ([`PodcastStore::apply_feed`]) only touch the catalog and +//! feed aliases: they never write a register, progress or a session. +//! +//! ``` +//! use furumi_library::podcasts::{ApplyOutcome, OpStamp, PodcastStore, StampSource}; +//! use music_dht::podcast::{FeedUrl, PodcastMeta}; +//! +//! /// Stand-in for the client's sync log, which reserves seq and hlc. +//! struct Log { +//! device: String, +//! seq: i64, +//! } +//! +//! impl StampSource for Log { +//! fn device_id(&self) -> &str { +//! &self.device +//! } +//! fn next_stamp(&mut self) -> anyhow::Result { +//! self.seq += 1; +//! let op_id = format!("{}:{}", self.device, self.seq); +//! Ok(OpStamp::new(op_id, self.device.clone(), 1_700_000_000_000 + self.seq)) +//! } +//! } +//! +//! let store = PodcastStore::open_in_memory()?; +//! let mut log = Log { device: "device-a".into(), seq: 0 }; +//! let meta = PodcastMeta { +//! feed_url: FeedUrl::parse("https://example.test/feed.xml")?, +//! title: "Show".into(), +//! author: None, +//! description: None, +//! link: None, +//! image_url: None, +//! }; +//! let outcome = store.subscribe(&mut log, &meta)?; +//! assert_eq!(outcome.ops.len(), 1); +//! for op in outcome.ops { +//! // Append to the sync log with this stamp; re-applying is a no-op. +//! let op = op.into_wire(log.seq); +//! assert_eq!(store.apply_remote_op(&op)?, ApplyOutcome::Unchanged); +//! } +//! # Ok::<(), anyhow::Error>(()) +//! ``` +//! +//! # Thread safety +//! +//! All methods take `&self` and serialize on an internal mutex around one +//! synchronous `rusqlite` connection, so an `Arc` can be shared +//! between threads. Calls block on SQLite I/O: run them on a blocking pool +//! (for example `tokio::task::spawn_blocking`), never on an async reactor +//! thread. Several processes may open the same file; SQLite serializes the +//! writers. + +mod actions; +mod apply; +mod queries; +mod refresh; +mod schema; + +#[cfg(test)] +mod tests; + +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire}; +use music_dht::podcast::{ + EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord, + ProgressStatus, +}; +use rusqlite::Connection; + +pub use actions::{CheckpointOutcome, ProgressSession, ProgressWrite, SubscribeOutcome}; +pub use apply::{ApplyOutcome, SnapshotApplyStats}; +pub use queries::{ + EpisodeListItem, EpisodeQuery, EpisodeSort, LegacyQuarantineEntry, PlayedFilter, + PodcastListFilter, PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary, +}; +pub use refresh::{FeedRefresh, FetchState, RefreshOutcome}; +pub use schema::{LEGACY_PROTOTYPE_SCHEMA_VERSION, PODCAST_SCHEMA_VERSION}; + +/// Conventional file name of the podcast store, next to `library.db`. +pub const PODCAST_DB_FILE: &str = "podcasts.sqlite3"; + +/// `op_id` prefix of registers imported from the prototype database. +/// +/// Imported subscriptions use generation 0, `hlc_ms` = the prototype +/// `added_at` and `op_id` = this prefix followed by the podcast id, so every +/// write made through the contract orders after them. +pub const LEGACY_IMPORT_OP_PREFIX: &str = "legacy-import:"; + +/// Typed errors of [`PodcastStore`]; other failures are SQLite or I/O errors. +/// +/// They are returned inside `anyhow::Error`; use `downcast_ref` to match. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum PodcastStoreError { + /// The database was written by a newer Furumi. It was not modified. + NewerSchema { + /// `user_version` found in the file. + found: i64, + /// Newest version this build understands. + supported: i64, + }, + /// The file is not a podcast database (or has an unexpected shape). It + /// was not modified. + UnrecognizedDatabase(String), + /// The podcast is unknown to this store. + UnknownPodcast(PodcastId), + /// The action needs an active subscription. + NotSubscribed(PodcastId), + /// Input violates the podcast contract. + Invalid(String), +} + +impl fmt::Display for PodcastStoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NewerSchema { found, supported } => write!( + f, + "podcast database schema v{found} is newer than this Furumi supports (v{supported}); \ + update Furumi, the database was left unchanged" + ), + Self::UnrecognizedDatabase(reason) => { + write!(f, "not a Furumi podcast database: {reason}") + } + Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"), + Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"), + Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"), + } + } +} + +impl std::error::Error for PodcastStoreError {} + +impl From for PodcastStoreError { + fn from(error: PodcastError) -> Self { + Self::Invalid(error.to_string()) + } +} + +/// Stamp of a device-sync operation: the fields the `SyncOpWire` +/// materializers turn into register stamps. +/// +/// `seq` is not part of it: the client's sync log assigns it when it appends +/// the operation ([`PodcastLocalOp::into_wire`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpStamp { + /// Unique operation id, usually `:`. + pub op_id: String, + /// Origin device id (the progress register `device_id`). + pub origin_device_id: String, + /// Operation timestamp (register `hlc_ms`). + pub hlc_ms: i64, +} + +impl OpStamp { + /// Builds a stamp. + pub fn new(op_id: impl Into, origin_device_id: impl Into, hlc_ms: i64) -> Self { + Self { + op_id: op_id.into(), + origin_device_id: origin_device_id.into(), + hlc_ms, + } + } + + /// The stamp of an existing operation. + pub fn of(op: &SyncOpWire) -> Self { + Self::new(op.op_id.clone(), op.origin_device_id.clone(), op.hlc_ms) + } + + pub(crate) fn wire(&self, payload: SyncOpPayload) -> SyncOpWire { + SyncOpWire { + op_id: self.op_id.clone(), + origin_device_id: self.origin_device_id.clone(), + seq: 0, + hlc_ms: self.hlc_ms, + payload, + } + } +} + +/// Source of operation stamps for local actions, usually the client's sync +/// log (which reserves the next sequence number and timestamp). +/// +/// A stamp is drawn only when an action actually emits an operation. +pub trait StampSource { + /// The local device id; every stamp must use it as origin. + fn device_id(&self) -> &str; + /// Reserves the stamp of the next local operation. + fn next_stamp(&mut self) -> Result; +} + +/// A local operation produced by a [`PodcastStore`] action. +/// +/// The store has already applied it. Append it to the sync log with exactly +/// this stamp: a different `op_id` or `hlc_ms` would make other devices +/// materialize a different register than this one. +#[derive(Debug, Clone, PartialEq)] +pub struct PodcastLocalOp { + /// Stamp drawn from the [`StampSource`]. + pub stamp: OpStamp, + /// Operation payload (one of the `podcast_*` kinds). + pub payload: SyncOpPayload, +} + +impl PodcastLocalOp { + /// The operation with the sequence number assigned by the sync log. + pub fn into_wire(self, seq: i64) -> SyncOpWire { + let mut op = self.stamp.wire(self.payload); + op.seq = seq; + op + } +} + +/// SQLite store of podcast catalog, personal state and fetch state. +/// +/// See the [module documentation](self) for the schema, the sync model and +/// thread safety. +pub struct PodcastStore { + conn: Mutex, + path: Option, +} + +impl fmt::Debug for PodcastStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PodcastStore") + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl PodcastStore { + /// Opens (creating and migrating if needed) the podcast store at `path`. + /// + /// A prototype database (`user_version` 1) is imported; see the module + /// documentation. Fails without modifying the file when it was written + /// by a newer Furumi or is not a podcast database. + pub fn open(path: &Path) -> Result { + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + let conn = Connection::open(path) + .with_context(|| format!("opening podcast database {}", path.display()))?; + Self::init(conn, Some(path.to_path_buf())) + } + + /// Opens a private in-memory store (tests and previews). + pub fn open_in_memory() -> Result { + Self::init(Connection::open_in_memory()?, None) + } + + fn init(mut conn: Connection, path: Option) -> Result { + conn.busy_timeout(Duration::from_secs(5))?; + // Refuse newer or foreign files before anything (WAL included) is written. + schema::check_openable(&conn)?; + if path.is_some() { + let _: String = conn.query_row("PRAGMA journal_mode=WAL", [], |row| row.get(0))?; + } + conn.pragma_update(None, "foreign_keys", "ON")?; + register_norm_function(&conn)?; + schema::migrate(&mut conn)?; + Ok(Self { + conn: Mutex::new(conn), + path, + }) + } + + /// Path of the database file (`None` for in-memory stores). + pub fn path(&self) -> Option<&Path> { + self.path.as_deref() + } + + /// Current `PRAGMA user_version` (always [`PODCAST_SCHEMA_VERSION`] + /// after a successful open). + pub fn schema_version(&self) -> Result { + Ok(self + .lock() + .query_row("PRAGMA user_version", [], |row| row.get(0))?) + } + + pub(crate) fn lock(&self) -> MutexGuard<'_, Connection> { + self.conn.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +fn register_norm_function(conn: &Connection) -> Result<()> { + use rusqlite::functions::FunctionFlags; + conn.create_scalar_function( + "norm", + 1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + |ctx| { + let value: Option = ctx.get(0)?; + Ok(music_dht::normalize_name( + value.as_deref().unwrap_or_default(), + )) + }, + )?; + Ok(()) +} + +pub(crate) fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +/// Converts a contract generation to the SQLite column value. +/// +/// Generations above `i64::MAX` cannot be stored; records carrying one are +/// rejected (no conforming client produces them). +pub(crate) fn generation_to_sql(generation: u64) -> Result { + i64::try_from(generation) + .map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string())) +} + +pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str { + match status { + ProgressStatus::NotStarted => "not_started", + ProgressStatus::InProgress => "in_progress", + ProgressStatus::Completed => "completed", + } +} + +pub(crate) fn status_from_sql(value: &str) -> ProgressStatus { + match value { + "in_progress" => ProgressStatus::InProgress, + "completed" => ProgressStatus::Completed, + _ => ProgressStatus::NotStarted, + } +} + +/// Parses a stored id; stored ids are validated on write. +pub(crate) fn podcast_id_col(value: String) -> rusqlite::Result { + PodcastId::try_from(value).map_err(|error| conversion_error(error.to_string())) +} + +pub(crate) fn episode_id_col(value: String) -> rusqlite::Result { + EpisodeId::try_from(value).map_err(|error| conversion_error(error.to_string())) +} + +pub(crate) fn feed_url_col(value: String) -> rusqlite::Result { + FeedUrl::try_from(value).map_err(|error| conversion_error(error.to_string())) +} + +pub(crate) fn podcast_meta_col(value: &str) -> rusqlite::Result { + serde_json::from_str(value).map_err(|error| conversion_error(error.to_string())) +} + +pub(crate) fn conversion_error(message: String) -> rusqlite::Error { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + Box::new(PodcastStoreError::Invalid(message)), + ) +} + +pub(crate) fn progress_from_row( + row: &rusqlite::Row<'_>, + at: usize, +) -> rusqlite::Result { + let generation: i64 = row.get(at + 4)?; + Ok(ProgressRecord { + episode_id: episode_id_col(row.get(at)?)?, + podcast_id: podcast_id_col(row.get(at + 1)?)?, + position_ms: row.get(at + 2)?, + status: status_from_sql(&row.get::<_, String>(at + 3)?), + generation: u64::try_from(generation).unwrap_or(0), + session_id: row.get(at + 5)?, + hlc_ms: row.get(at + 6)?, + device_id: row.get(at + 7)?, + op_id: row.get(at + 8)?, + }) +} + +/// Column list matching [`progress_from_row`], for table alias `p`. +pub(crate) const PROGRESS_COLUMNS: &str = "p.episode_id, p.podcast_id, p.position_ms, p.status, \ + p.generation, p.session_id, p.hlc_ms, p.device_id, p.op_id"; + +pub(crate) fn session_from_row( + row: &rusqlite::Row<'_>, + at: usize, +) -> rusqlite::Result { + Ok(ListenSession { + session_id: row.get(at)?, + episode_id: episode_id_col(row.get(at + 1)?)?, + podcast_id: podcast_id_col(row.get(at + 2)?)?, + device_id: row.get(at + 3)?, + started_at_ms: row.get(at + 4)?, + ended_at_ms: row.get(at + 5)?, + start_position_ms: row.get(at + 6)?, + end_position_ms: row.get(at + 7)?, + listened_ms: row.get(at + 8)?, + completed: row.get(at + 9)?, + }) +} + +/// Column list matching [`session_from_row`], for table alias `s`. +pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \ + s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \ + s.completed"; diff --git a/crates/furumi-library/src/podcasts/queries.rs b/crates/furumi-library/src/podcasts/queries.rs new file mode 100644 index 0000000..967fbbb --- /dev/null +++ b/crates/furumi-library/src/podcasts/queries.rs @@ -0,0 +1,605 @@ +//! Read-only queries for clients' podcast UI. + +use anyhow::Result; +use music_dht::podcast::{ + EpisodeId, EpisodeMeta, EpisodeType, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, + PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, effective_share, +}; +use rusqlite::types::Value; +use rusqlite::{Connection, OptionalExtension as _, params, params_from_iter}; + +use super::apply; +use super::{ + PROGRESS_COLUMNS, PodcastStore, SESSION_COLUMNS, episode_id_col, feed_url_col, podcast_id_col, + progress_from_row, session_from_row, +}; +use crate::TrackItem; + +/// Which podcasts [`PodcastStore::list_podcasts`] returns. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum PodcastListFilter { + /// Podcasts with an active subscription. + #[default] + Subscribed, + /// Every podcast in the catalog, including unsubscribed ones and + /// podcasts only known from synced progress or queue items. + All, +} + +/// A podcast with its personal state and episode counters. +#[derive(Debug, Clone, PartialEq)] +pub struct PodcastSummary { + /// Podcast id. + pub podcast_id: PodcastId, + /// Local channel metadata; `feed_url` is the URL this device fetches. + pub meta: PodcastMeta, + /// Whether the subscription register says subscribed. + pub subscribed: bool, + /// Whether the podcast must currently be published (`effective_share`). + pub shared: bool, + /// Episodes in the catalog. + pub episode_count: u64, + /// Episodes not completed (no progress, not started or in progress). + pub unplayed_count: u64, + /// Episodes in progress. + pub in_progress_count: u64, + /// Newest publication time. + pub latest_published_at_ms: Option, + /// Numbered seasons (episodes without a season are not counted). + pub season_count: u64, +} + +/// One season group of a podcast. `season == None` groups the episodes +/// without `itunes:season`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SeasonSummary { + /// Season number; `None` for episodes without a season. + pub season: Option, + /// Episodes in the group. + pub episode_count: u64, + /// Episodes not completed. + pub unplayed_count: u64, + /// Oldest publication time. + pub first_published_at_ms: Option, + /// Newest publication time. + pub last_published_at_ms: Option, +} + +/// Season filter of an [`EpisodeQuery`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum SeasonFilter { + /// All episodes. + #[default] + Any, + /// Episodes of one season. + Season(u32), + /// Episodes without a season. + NoSeason, +} + +/// Listening-state filter of an [`EpisodeQuery`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum PlayedFilter { + /// All episodes. + #[default] + Any, + /// Not completed (not started or in progress). + Unplayed, + /// Never started or marked unplayed. + NotStarted, + /// In progress. + InProgress, + /// Completed. + Played, +} + +/// Episode ordering of an [`EpisodeQuery`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum EpisodeSort { + /// Newest publication first; undated episodes last. + #[default] + NewestFirst, + /// Oldest publication first; undated episodes last. + OldestFirst, + /// By season, then episode number, ascending; unnumbered last. + EpisodeNumber, + /// By season, then episode number, descending; unnumbered last. + EpisodeNumberDesc, +} + +/// Parameters of [`PodcastStore::episodes`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EpisodeQuery { + /// Podcast. + pub podcast_id: PodcastId, + /// Season filter. + pub season: SeasonFilter, + /// Listening-state filter. + pub played: PlayedFilter, + /// Text searched in titles and descriptions (Unicode-normalized, + /// case-insensitive substring match). Blank means no filter. + pub search: Option, + /// Ordering. + pub sort: EpisodeSort, + /// Maximum rows (`None`: all). + pub limit: Option, + /// Rows to skip. + pub offset: usize, +} + +impl EpisodeQuery { + /// All episodes of a podcast, newest first. + pub fn new(podcast_id: PodcastId) -> Self { + Self { + podcast_id, + season: SeasonFilter::Any, + played: PlayedFilter::Any, + search: None, + sort: EpisodeSort::NewestFirst, + limit: None, + offset: 0, + } + } +} + +/// An episode with its personal state. +#[derive(Debug, Clone, PartialEq)] +pub struct EpisodeListItem { + /// Episode metadata (feed or synced reference). + pub episode: EpisodeMeta, + /// Progress register, if any. + pub progress: Option, + /// Distinct completed listening sessions (`completed_listen_count`). + pub completed_count: u64, + /// Whether the latest refresh of the feed listed the episode. + pub in_feed: bool, +} + +impl EpisodeListItem { + /// Listening status (`NotStarted` without a register). + pub fn status(&self) -> ProgressStatus { + self.progress + .as_ref() + .map_or(ProgressStatus::NotStarted, |progress| progress.status) + } + + /// Position ordinary Play resumes from. + pub fn resume_position_ms(&self) -> i64 { + self.progress + .as_ref() + .map_or(0, ProgressRecord::resume_position_ms) + } +} + +/// A listening session in history, with display titles when known. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PodcastListenEntry { + /// The session. + pub session: ListenSession, + /// Episode title from the catalog. + pub episode_title: Option, + /// Podcast title from the catalog. + pub podcast_title: Option, +} + +/// A prototype row that could not be imported (kept in its `legacy_*` table). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyQuarantineEntry { + /// `legacy_podcasts` or `legacy_podcast_episodes`. + pub legacy_table: String, + /// Row id in that table. + pub legacy_id: i64, + /// Short reason. + pub reason: String, + /// Details (the offending value or error). + pub detail: Option, +} + +const EPISODE_COLUMNS: &str = "e.episode_id, e.podcast_id, e.guid, e.title, e.description, \ + e.season, e.episode_number, e.episode_type, e.published_at_ms, e.duration_ms, \ + e.audio_url, e.mime, e.byte_len, e.image_url, e.link"; +const EPISODE_COLUMN_COUNT: usize = 15; + +fn episode_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(EpisodeMeta { + episode_id: episode_id_col(row.get(0)?)?, + podcast_id: podcast_id_col(row.get(1)?)?, + guid: row.get(2)?, + title: row.get(3)?, + description: row.get(4)?, + season: row.get(5)?, + episode_number: row.get(6)?, + episode_type: EpisodeType::parse(&row.get::<_, String>(7)?), + published_at_ms: row.get(8)?, + duration_ms: row.get(9)?, + audio_url: row.get(10)?, + mime: row.get(11)?, + byte_len: row + .get::<_, Option>(12)? + .and_then(|len| u64::try_from(len).ok()), + image_url: row.get(13)?, + link: row.get(14)?, + }) +} + +fn optional_progress( + row: &rusqlite::Row<'_>, + at: usize, +) -> rusqlite::Result> { + if row.get::<_, Option>(at)?.is_none() { + return Ok(None); + } + progress_from_row(row, at).map(Some) +} + +pub(crate) fn episode_meta_in( + conn: &Connection, + episode_id: &EpisodeId, +) -> Result> { + Ok(conn + .query_row( + &format!("SELECT {EPISODE_COLUMNS} FROM episodes e WHERE e.episode_id = ?1"), + [episode_id.as_str()], + episode_from_row, + ) + .optional()?) +} + +pub(crate) fn podcast_meta_in( + conn: &Connection, + podcast_id: &PodcastId, +) -> Result> { + Ok(conn + .query_row( + "SELECT feed_url, title, author, description, link, image_url + FROM podcasts WHERE podcast_id = ?1", + [podcast_id.as_str()], + |row| { + Ok(PodcastMeta { + feed_url: feed_url_col(row.get(0)?)?, + title: row.get(1)?, + author: row.get(2)?, + description: row.get(3)?, + link: row.get(4)?, + image_url: row.get(5)?, + }) + }, + ) + .optional()?) +} + +/// Queue reference of a catalog episode, with `progress_generation` set to +/// the generation of its progress register (handoff hint). +pub(crate) fn episode_ref_in( + conn: &Connection, + episode_id: &EpisodeId, +) -> Result> { + let Some(episode) = episode_meta_in(conn, episode_id)? else { + return Ok(None); + }; + let Some(podcast) = podcast_meta_in(conn, &episode.podcast_id)? else { + return Ok(None); + }; + let mut reference = episode.to_ref(&podcast); + reference.progress_generation = + apply::load_progress(conn, episode_id)?.map(|progress| progress.generation); + Ok(Some(reference)) +} + +const NOT_COMPLETED: &str = "IFNULL(g.status, 'not_started') <> 'completed'"; + +impl PodcastStore { + /// Podcasts with subscription state and counters, sorted by title. + pub fn list_podcasts(&self, filter: PodcastListFilter) -> Result> { + let conn = self.lock(); + let join = match filter { + PodcastListFilter::Subscribed => { + "JOIN podcast_subscriptions r ON r.podcast_id = p.podcast_id AND r.subscribed = 1" + } + PodcastListFilter::All => "", + }; + let ids: Vec = { + let mut statement = conn.prepare(&format!( + "SELECT p.podcast_id FROM podcasts p {join} + ORDER BY p.title COLLATE NOCASE, p.podcast_id" + ))?; + statement + .query_map([], |row| podcast_id_col(row.get(0)?))? + .collect::>>()? + }; + let mut out = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(summary) = summary_in(&conn, &id)? { + out.push(summary); + } + } + Ok(out) + } + + /// One podcast with subscription state and counters. + pub fn podcast(&self, podcast_id: &PodcastId) -> Result> { + summary_in(&self.lock(), podcast_id) + } + + /// The subscription register (with the stored alias set), if any. + pub fn subscription(&self, podcast_id: &PodcastId) -> Result> { + apply::load_subscription_register(&self.lock(), podcast_id) + } + + /// The share register, if any. + pub fn share(&self, podcast_id: &PodcastId) -> Result> { + apply::load_share(&self.lock(), podcast_id) + } + + /// The bounded, synced feed alias set of a podcast. + pub fn feed_aliases(&self, podcast_id: &PodcastId) -> Result> { + apply::load_aliases(&self.lock(), podcast_id) + } + + /// Season groups of a podcast: numbered seasons ascending, then the + /// group of episodes without a season. + pub fn seasons(&self, podcast_id: &PodcastId) -> Result> { + let conn = self.lock(); + let mut statement = conn.prepare(&format!( + "SELECT s.season, s.episode_count, s.first_published_at_ms, s.last_published_at_ms, + (SELECT COUNT(*) FROM episodes e + LEFT JOIN episode_progress g ON g.episode_id = e.episode_id + WHERE e.podcast_id = s.podcast_id AND e.season IS s.season + AND {NOT_COMPLETED}) + FROM podcast_seasons s WHERE s.podcast_id = ?1 + ORDER BY s.season IS NULL, s.season" + ))?; + let seasons = statement + .query_map([podcast_id.as_str()], |row| { + Ok(SeasonSummary { + season: row.get(0)?, + episode_count: count(row.get(1)?), + first_published_at_ms: row.get(2)?, + last_published_at_ms: row.get(3)?, + unplayed_count: count(row.get(4)?), + }) + })? + .collect::>>()?; + Ok(seasons) + } + + /// Episodes of a podcast or season with their progress. + pub fn episodes(&self, query: &EpisodeQuery) -> Result> { + let mut conditions = vec!["e.podcast_id = ?".to_string()]; + let mut values = vec![Value::Text(query.podcast_id.as_str().to_string())]; + match query.season { + SeasonFilter::Any => {} + SeasonFilter::Season(season) => { + conditions.push("e.season = ?".to_string()); + values.push(Value::Integer(i64::from(season))); + } + SeasonFilter::NoSeason => conditions.push("e.season IS NULL".to_string()), + } + match query.played { + PlayedFilter::Any => {} + PlayedFilter::Unplayed => conditions.push(NOT_COMPLETED.to_string()), + PlayedFilter::NotStarted => { + conditions.push("IFNULL(g.status, 'not_started') = 'not_started'".to_string()); + } + PlayedFilter::InProgress => conditions.push("g.status = 'in_progress'".to_string()), + PlayedFilter::Played => conditions.push("g.status = 'completed'".to_string()), + } + if let Some(search) = query.search.as_deref() { + let needle = music_dht::normalize_name(search); + if !needle.is_empty() { + conditions.push( + "(instr(norm(e.title), ?) > 0 OR instr(norm(e.description), ?) > 0)" + .to_string(), + ); + values.push(Value::Text(needle.clone())); + values.push(Value::Text(needle)); + } + } + let order = match query.sort { + EpisodeSort::NewestFirst => { + "e.published_at_ms IS NULL, e.published_at_ms DESC, e.episode_id" + } + EpisodeSort::OldestFirst => { + "e.published_at_ms IS NULL, e.published_at_ms, e.episode_id" + } + EpisodeSort::EpisodeNumber => { + "e.season IS NULL, e.season, e.episode_number IS NULL, e.episode_number, \ + e.published_at_ms IS NULL, e.published_at_ms, e.episode_id" + } + EpisodeSort::EpisodeNumberDesc => { + "e.season IS NULL, e.season DESC, e.episode_number IS NULL, \ + e.episode_number DESC, e.published_at_ms IS NULL, e.published_at_ms DESC, \ + e.episode_id" + } + }; + values.push(Value::Integer( + query + .limit + .map_or(-1, |limit| i64::try_from(limit).unwrap_or(i64::MAX)), + )); + values.push(Value::Integer( + i64::try_from(query.offset).unwrap_or(i64::MAX), + )); + let progress_columns = PROGRESS_COLUMNS.replace("p.", "g."); + let sql = format!( + "SELECT {EPISODE_COLUMNS}, e.in_feed, {progress_columns}, + (SELECT COUNT(*) FROM podcast_listen_sessions s + WHERE s.episode_id = e.episode_id AND s.completed = 1) + FROM episodes e + LEFT JOIN episode_progress g ON g.episode_id = e.episode_id + WHERE {} + ORDER BY {order} + LIMIT ? OFFSET ?", + conditions.join(" AND ") + ); + let conn = self.lock(); + let mut statement = conn.prepare(&sql)?; + let items = statement + .query_map(params_from_iter(values), list_item_from_row)? + .collect::>>()?; + Ok(items) + } + + /// One episode with its progress. + pub fn episode(&self, episode_id: &EpisodeId) -> Result> { + let progress_columns = PROGRESS_COLUMNS.replace("p.", "g."); + Ok(self + .lock() + .query_row( + &format!( + "SELECT {EPISODE_COLUMNS}, e.in_feed, {progress_columns}, + (SELECT COUNT(*) FROM podcast_listen_sessions s + WHERE s.episode_id = e.episode_id AND s.completed = 1) + FROM episodes e + LEFT JOIN episode_progress g ON g.episode_id = e.episode_id + WHERE e.episode_id = ?1" + ), + [episode_id.as_str()], + list_item_from_row, + ) + .optional()?) + } + + /// Progress register of an episode (also for episodes the catalog does + /// not know yet). + pub fn progress(&self, episode_id: &EpisodeId) -> Result> { + apply::load_progress(&self.lock(), episode_id) + } + + /// Position ordinary Play resumes from (`ProgressRecord::resume_position_ms`, 0 without progress). + pub fn resume_position_ms(&self, episode_id: &EpisodeId) -> Result { + Ok(self + .progress(episode_id)? + .map_or(0, |progress| progress.resume_position_ms())) + } + + /// A queue item for an episode (see [`TrackItem::podcast_episode`]). + pub fn episode_track(&self, episode_id: &EpisodeId) -> Result> { + Ok(self + .episode_ref(episode_id)? + .map(TrackItem::podcast_episode)) + } + + /// Listening history (sessions that `should_record`), newest first. + pub fn listen_history( + &self, + podcast_id: Option<&PodcastId>, + limit: usize, + ) -> Result> { + let conn = self.lock(); + let mut statement = conn.prepare(&format!( + "SELECT {SESSION_COLUMNS}, e.title, p.title + FROM podcast_listen_sessions s + LEFT JOIN episodes e ON e.episode_id = s.episode_id + LEFT JOIN podcasts p ON p.podcast_id = s.podcast_id + WHERE (s.completed = 1 OR s.listened_ms >= ?1) + AND (?2 IS NULL OR s.podcast_id = ?2) + ORDER BY s.ended_at_ms DESC, s.session_id DESC + LIMIT ?3" + ))?; + let entries = statement + .query_map( + params![ + music_dht::podcast::MIN_RECORDED_EPISODE_LISTEN_MS, + podcast_id.map(PodcastId::as_str), + i64::try_from(limit).unwrap_or(i64::MAX) + ], + |row| { + Ok(PodcastListenEntry { + session: session_from_row(row, 0)?, + episode_title: row.get(10)?, + podcast_title: row.get(11)?, + }) + }, + )? + .collect::>>()?; + Ok(entries) + } + + /// Number of distinct completed sessions of an episode. + pub fn completed_listen_count(&self, episode_id: &EpisodeId) -> Result { + let total: i64 = self.lock().query_row( + "SELECT COUNT(*) FROM podcast_listen_sessions WHERE episode_id = ?1 AND completed = 1", + [episode_id.as_str()], + |row| row.get(0), + )?; + Ok(count(total)) + } + + /// Prototype rows that could not be imported. + pub fn legacy_quarantine(&self) -> Result> { + let conn = self.lock(); + let mut statement = conn.prepare( + "SELECT legacy_table, legacy_id, reason, detail FROM legacy_quarantine + ORDER BY legacy_table, legacy_id", + )?; + let entries = statement + .query_map([], |row| { + Ok(LegacyQuarantineEntry { + legacy_table: row.get(0)?, + legacy_id: row.get(1)?, + reason: row.get(2)?, + detail: row.get(3)?, + }) + })? + .collect::>>()?; + Ok(entries) + } +} + +fn count(value: i64) -> u64 { + u64::try_from(value).unwrap_or(0) +} + +fn list_item_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(EpisodeListItem { + episode: episode_from_row(row)?, + in_feed: row.get(EPISODE_COLUMN_COUNT)?, + progress: optional_progress(row, EPISODE_COLUMN_COUNT + 1)?, + completed_count: count(row.get(EPISODE_COLUMN_COUNT + 10)?), + }) +} + +fn summary_in(conn: &Connection, podcast_id: &PodcastId) -> Result> { + let Some(meta) = podcast_meta_in(conn, podcast_id)? else { + return Ok(None); + }; + let (episode_count, unplayed_count, in_progress_count, latest, season_count) = conn.query_row( + &format!( + "SELECT COUNT(*), + COUNT(*) FILTER (WHERE {NOT_COMPLETED}), + COUNT(*) FILTER (WHERE g.status = 'in_progress'), + MAX(e.published_at_ms), + (SELECT COUNT(*) FROM podcast_seasons s + WHERE s.podcast_id = ?1 AND s.season IS NOT NULL) + FROM episodes e + LEFT JOIN episode_progress g ON g.episode_id = e.episode_id + WHERE e.podcast_id = ?1" + ), + [podcast_id.as_str()], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, i64>(4)?, + )) + }, + )?; + let subscription = apply::load_subscription_register(conn, podcast_id)?; + let share = apply::load_share(conn, podcast_id)?; + Ok(Some(PodcastSummary { + podcast_id: podcast_id.clone(), + meta, + subscribed: subscription + .as_ref() + .is_some_and(|record| record.subscribed), + shared: effective_share(subscription.as_ref(), share.as_ref()), + episode_count: count(episode_count), + unplayed_count: count(unplayed_count), + in_progress_count: count(in_progress_count), + latest_published_at_ms: latest, + season_count: count(season_count), + })) +} diff --git a/crates/furumi-library/src/podcasts/refresh.rs b/crates/furumi-library/src/podcasts/refresh.rs new file mode 100644 index 0000000..2ee0d00 --- /dev/null +++ b/crates/furumi-library/src/podcasts/refresh.rs @@ -0,0 +1,465 @@ +//! Feed refresh ingestion and the local-only fetch state. + +use std::collections::BTreeSet; + +use anyhow::Result; +use music_dht::device_sync::SyncOpPayload; +use music_dht::podcast::feed::ParsedFeed; +use music_dht::podcast::{EpisodeId, EpisodeMeta, FeedUrl, PodcastId, merge_feed_aliases}; +use rusqlite::{Connection, OptionalExtension as _, TransactionBehavior, params}; + +use super::apply::{self, EpisodeSource}; +use super::{PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource, now_ms}; + +/// A fetched and parsed feed document for [`PodcastStore::apply_feed`]. +#[derive(Debug, Clone, Copy)] +pub struct FeedRefresh<'a> { + /// Podcast the feed belongs to. + pub podcast_id: &'a PodcastId, + /// The parsed document (`music_dht::podcast::feed::parse_feed_with_base` + /// with `fetched_from` as base). + pub feed: &'a ParsedFeed, + /// URL the document was finally served from, after redirects. + pub fetched_from: &'a FeedUrl, + /// Whether every redirect leading to `fetched_from` was permanent + /// (301/308). Temporary redirects do not move the feed. + pub moved_permanently: bool, +} + +/// Result of [`PodcastStore::apply_feed`]. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct RefreshOutcome { + /// Episodes seen for the first time. + pub new_episodes: usize, + /// Known episodes whose metadata changed (title, audio URL, ...). + pub updated_episodes: usize, + /// Known episodes without metadata changes. + pub unchanged_episodes: usize, + /// Items without identity or with invalid metadata. + pub skipped_items: usize, + /// Feed URLs learned from a permanent redirect or `itunes:new-feed-url`. + pub learned_aliases: Vec, + /// URL the podcast is fetched from after this refresh. + pub feed_url: Option, + /// `podcast_feed_aliases_added`, only when the learned URLs changed the + /// stored alias set. Append it to the sync log. + pub ops: Vec, +} + +/// Local-only fetch state of a podcast. Never part of sync output. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct FetchState { + /// HTTP `ETag` validator of the last successful fetch. + pub etag: Option, + /// HTTP `Last-Modified` validator of the last successful fetch. + pub last_modified: Option, + /// Last fetch attempt, unix milliseconds. + pub checked_at_ms: Option, + /// Last successful fetch (including "not modified"). + pub last_success_at_ms: Option, + /// When the scheduler should check again (`None`: due now). + pub next_check_at_ms: Option, + /// Error of the last attempt; `None` after a success. + pub last_error: Option, + /// Failed attempts since the last success. + pub consecutive_failures: u32, + /// Image URL the cached cover was downloaded from. + pub cover_url: Option, + /// Local path of the cached cover. + pub cover_path: Option, +} + +fn podcast_fetch_url(conn: &Connection, podcast_id: &PodcastId) -> Result> { + Ok(conn + .query_row( + "SELECT feed_url FROM podcasts WHERE podcast_id = ?1", + [podcast_id.as_str()], + |row| super::feed_url_col(row.get(0)?), + ) + .optional()?) +} + +fn has_personal_state(conn: &Connection, episode_id: &EpisodeId) -> Result { + Ok(conn.query_row( + "SELECT EXISTS (SELECT 1 FROM episode_progress WHERE episode_id = ?1) + OR EXISTS (SELECT 1 FROM podcast_listen_sessions WHERE episode_id = ?1)", + [episode_id.as_str()], + |row| row.get(0), + )?) +} + +/// Re-keys episodes imported from the prototype whose stored `guid` was the +/// enclosure URL (`legacy_guid`) when the feed shows it is a real GUID. +/// +/// Only episodes without progress or sessions move, so no personal state is +/// touched; otherwise the GUID item becomes a separate episode. +fn rekey_legacy_guids(conn: &Connection, podcast_id: &PodcastId, feed: &ParsedFeed) -> Result<()> { + for item in &feed.episodes { + let Some(guid) = item.guid.as_deref() else { + continue; + }; + let Some(guid_id) = EpisodeId::from_guid(podcast_id, guid) else { + continue; + }; + let taken = conn + .query_row( + "SELECT 1 FROM episodes WHERE episode_id = ?1", + [guid_id.as_str()], + |_| Ok(()), + ) + .optional()? + .is_some(); + if taken { + continue; + } + let legacy: Option = conn + .query_row( + "SELECT episode_id FROM episodes + WHERE podcast_id = ?1 AND legacy_guid = ?2 ORDER BY episode_id LIMIT 1", + params![podcast_id.as_str(), guid], + |row| row.get(0), + ) + .optional()?; + let Some(legacy) = legacy else { + continue; + }; + let legacy = super::episode_id_col(legacy)?; + if has_personal_state(conn, &legacy)? { + continue; + } + conn.execute( + "UPDATE episodes SET episode_id = ?2, guid = ?3, legacy_guid = NULL + WHERE episode_id = ?1", + params![legacy.as_str(), guid_id.as_str(), guid], + )?; + } + Ok(()) +} + +fn upsert_feed_episode(conn: &Connection, meta: &EpisodeMeta, now: i64) -> Result> { + let Some(stored) = super::queries::episode_meta_in(conn, &meta.episode_id)? else { + apply::insert_episode(conn, meta, EpisodeSource::Feed, true, None, now, now)?; + return Ok(None); + }; + let changed = stored != *meta; + conn.execute( + "UPDATE episodes + SET podcast_id = ?2, guid = ?3, title = ?4, description = ?5, season = ?6, + episode_number = ?7, episode_type = ?8, published_at_ms = ?9, + duration_ms = ?10, audio_url = ?11, mime = ?12, byte_len = ?13, + image_url = ?14, link = ?15, source = 'feed', in_feed = 1, + legacy_guid = NULL, + updated_at_ms = CASE WHEN ?16 THEN ?17 ELSE updated_at_ms END + WHERE episode_id = ?1", + params![ + meta.episode_id.as_str(), + meta.podcast_id.as_str(), + meta.guid, + meta.title, + meta.description, + meta.season, + meta.episode_number, + apply::episode_type_to_sql(meta.episode_type), + meta.published_at_ms, + meta.duration_ms, + meta.audio_url, + meta.mime, + meta.byte_len.and_then(|len| i64::try_from(len).ok()), + meta.image_url, + meta.link, + changed, + now, + ], + )?; + Ok(Some(changed)) +} + +impl PodcastStore { + /// Ingests a fetched feed document for a podcast. + /// + /// * Channel metadata of the catalog row is replaced by the feed's. + /// * Episode ids are resolved with the contract identity rule + /// (`ParsedFeed::resolve_episode_ids`) against every stored episode of + /// the podcast, including episodes learned through sync, so a changed + /// title, audio URL or a GUID-less item keeps its id. Known episodes + /// get the new metadata (including the audio URL); new ones are added; + /// episodes missing from the document are kept (`in_feed = false`). + /// * A permanent redirect target and `itunes:new-feed-url` become feed + /// aliases and the new fetch URL. When they change the stored alias + /// set, one `podcast_feed_aliases_added` operation is emitted. + /// * Subscription, share and progress registers and listening sessions + /// are never touched, so a refresh cannot resurrect an unsubscribed + /// podcast or move anyone's position. + /// + /// Record the HTTP outcome separately with + /// [`PodcastStore::record_fetch_success`] and friends. + pub fn apply_feed( + &self, + stamps: &mut dyn StampSource, + refresh: &FeedRefresh<'_>, + ) -> Result { + let podcast_id = refresh.podcast_id; + let feed = refresh.feed; + let now = now_ms(); + let mut conn = self.lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current_url = match podcast_fetch_url(&tx, podcast_id)? { + Some(url) => url, + None => { + let Some(register) = apply::load_subscription_register(&tx, podcast_id)? else { + return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into()); + }; + apply::apply_subscription_record(&tx, ®ister, now)?; + register.podcast.feed_url + } + }; + + rekey_legacy_guids(&tx, podcast_id, feed)?; + let known = apply::known_episodes(&tx, podcast_id)?; + let ids = feed.resolve_episode_ids(podcast_id, &known); + tx.execute( + "UPDATE episodes SET in_feed = 0 WHERE podcast_id = ?1", + [podcast_id.as_str()], + )?; + let mut outcome = RefreshOutcome::default(); + for (item, id) in feed.episodes.iter().zip(ids) { + let Some(id) = id else { + outcome.skipped_items += 1; + continue; + }; + let meta = item.to_meta(podcast_id, id); + if meta.validate().is_err() { + outcome.skipped_items += 1; + continue; + } + match upsert_feed_episode(&tx, &meta, now)? { + None => outcome.new_episodes += 1, + Some(true) => outcome.updated_episodes += 1, + Some(false) => outcome.unchanged_episodes += 1, + } + } + + let mut fetch_url = current_url.clone(); + let mut learned = BTreeSet::new(); + if refresh.moved_permanently && *refresh.fetched_from != fetch_url { + learned.insert(fetch_url.clone()); + learned.insert(refresh.fetched_from.clone()); + fetch_url = refresh.fetched_from.clone(); + } + if let Some(new_url) = feed + .new_feed_url + .as_deref() + .and_then(|url| FeedUrl::parse(url).ok()) + && new_url != fetch_url + { + learned.insert(fetch_url.clone()); + learned.insert(new_url.clone()); + fetch_url = new_url; + } + let meta = feed.to_podcast_meta(fetch_url.clone()); + tx.execute( + "UPDATE podcasts + SET feed_url = ?2, title = ?3, author = ?4, description = ?5, link = ?6, + image_url = ?7, meta_from_feed = 1, updated_at_ms = ?8 + WHERE podcast_id = ?1", + params![ + podcast_id.as_str(), + meta.feed_url.as_str(), + meta.title, + meta.author, + meta.description, + meta.link, + meta.image_url, + now + ], + )?; + if !learned.is_empty() { + let learned: Vec = learned.into_iter().collect(); + let current = apply::load_aliases(&tx, podcast_id)?; + if merge_feed_aliases(podcast_id, ¤t, &learned) != current { + let op = super::actions::emit(&tx, stamps, |_| { + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id: podcast_id.clone(), + feed_aliases: learned.clone(), + } + })?; + outcome.ops.push(op); + } + outcome.learned_aliases = learned; + } + apply::refresh_seasons(&tx, podcast_id)?; + tx.commit()?; + outcome.feed_url = Some(fetch_url); + Ok(outcome) + } + + /// Local fetch state of a podcast (defaults when never fetched). + pub fn fetch_state(&self, podcast_id: &PodcastId) -> Result { + Ok(self + .lock() + .query_row( + "SELECT etag, last_modified, checked_at_ms, last_success_at_ms, + next_check_at_ms, last_error, consecutive_failures, cover_url, + cover_path + FROM podcast_fetch_state WHERE podcast_id = ?1", + [podcast_id.as_str()], + |row| { + Ok(FetchState { + etag: row.get(0)?, + last_modified: row.get(1)?, + checked_at_ms: row.get(2)?, + last_success_at_ms: row.get(3)?, + next_check_at_ms: row.get(4)?, + last_error: row.get(5)?, + consecutive_failures: u32::try_from(row.get::<_, i64>(6)?) + .unwrap_or(u32::MAX), + cover_url: row.get(7)?, + cover_path: row.get(8)?, + }) + }, + ) + .optional()? + .unwrap_or_default()) + } + + fn update_fetch_state( + &self, + podcast_id: &PodcastId, + update: impl FnOnce(&mut FetchState), + ) -> Result { + let mut state = self.fetch_state(podcast_id)?; + update(&mut state); + let conn = self.lock(); + if podcast_fetch_url(&conn, podcast_id)?.is_none() { + return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into()); + } + conn.execute( + "INSERT INTO podcast_fetch_state + (podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms, + next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT(podcast_id) DO UPDATE SET + etag = excluded.etag, last_modified = excluded.last_modified, + checked_at_ms = excluded.checked_at_ms, + last_success_at_ms = excluded.last_success_at_ms, + next_check_at_ms = excluded.next_check_at_ms, + last_error = excluded.last_error, + consecutive_failures = excluded.consecutive_failures, + cover_url = excluded.cover_url, cover_path = excluded.cover_path", + params![ + podcast_id.as_str(), + state.etag, + state.last_modified, + state.checked_at_ms, + state.last_success_at_ms, + state.next_check_at_ms, + state.last_error, + i64::from(state.consecutive_failures), + state.cover_url, + state.cover_path, + ], + )?; + Ok(state) + } + + /// Records a successful fetch (HTTP 200) with its new validators. + pub fn record_fetch_success( + &self, + podcast_id: &PodcastId, + now_ms: i64, + etag: Option<&str>, + last_modified: Option<&str>, + next_check_at_ms: Option, + ) -> Result<()> { + self.update_fetch_state(podcast_id, |state| { + state.etag = etag.map(str::to_string); + state.last_modified = last_modified.map(str::to_string); + state.checked_at_ms = Some(now_ms); + state.last_success_at_ms = Some(now_ms); + state.next_check_at_ms = next_check_at_ms; + state.last_error = None; + state.consecutive_failures = 0; + }) + .map(|_| ()) + } + + /// Records a conditional fetch answered with "304 Not Modified". + pub fn record_fetch_not_modified( + &self, + podcast_id: &PodcastId, + now_ms: i64, + next_check_at_ms: Option, + ) -> Result<()> { + self.update_fetch_state(podcast_id, |state| { + state.checked_at_ms = Some(now_ms); + state.last_success_at_ms = Some(now_ms); + state.next_check_at_ms = next_check_at_ms; + state.last_error = None; + state.consecutive_failures = 0; + }) + .map(|_| ()) + } + + /// Records a failed fetch; returns the consecutive failure count. The + /// stored catalog is kept. + pub fn record_fetch_failure( + &self, + podcast_id: &PodcastId, + now_ms: i64, + error: &str, + next_check_at_ms: Option, + ) -> Result { + self.update_fetch_state(podcast_id, |state| { + state.checked_at_ms = Some(now_ms); + state.next_check_at_ms = next_check_at_ms; + state.last_error = Some(error.to_string()); + state.consecutive_failures = state.consecutive_failures.saturating_add(1); + }) + .map(|state| state.consecutive_failures) + } + + /// Records the locally cached cover image of a podcast. + pub fn set_cover_cache( + &self, + podcast_id: &PodcastId, + cover_url: Option<&str>, + cover_path: Option<&str>, + ) -> Result<()> { + self.update_fetch_state(podcast_id, |state| { + state.cover_url = cover_url.map(str::to_string); + state.cover_path = cover_path.map(str::to_string); + }) + .map(|_| ()) + } + + /// Subscribed podcasts due for a refresh at `now_ms` (never checked or + /// `next_check_at_ms <= now_ms`), most overdue first, with their fetch URL. + pub fn podcasts_due_for_refresh( + &self, + now_ms: i64, + limit: usize, + ) -> Result> { + let conn = self.lock(); + let mut statement = conn.prepare( + "SELECT p.podcast_id, p.feed_url + FROM podcasts p + JOIN podcast_subscriptions s ON s.podcast_id = p.podcast_id AND s.subscribed = 1 + LEFT JOIN podcast_fetch_state f ON f.podcast_id = p.podcast_id + WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= ?1 + ORDER BY IFNULL(f.next_check_at_ms, 0), p.podcast_id + LIMIT ?2", + )?; + let due = statement + .query_map( + params![now_ms, i64::try_from(limit).unwrap_or(i64::MAX)], + |row| { + Ok(( + super::podcast_id_col(row.get(0)?)?, + super::feed_url_col(row.get(1)?)?, + )) + }, + )? + .collect::>>()?; + Ok(due) + } +} diff --git a/crates/furumi-library/src/podcasts/schema.rs b/crates/furumi-library/src/podcasts/schema.rs new file mode 100644 index 0000000..8c75f38 --- /dev/null +++ b/crates/furumi-library/src/podcasts/schema.rs @@ -0,0 +1,729 @@ +//! Schema, migrations and the import of the prototype database. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Context as _, Result}; +use music_dht::podcast::{ + EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode, + MAX_DESCRIPTION_CHARS, MAX_TITLE_CHARS, MAX_URL_BYTES, PodcastId, PodcastMeta, + SubscriptionRecord, bounded_text, is_http_url, resolve_episode_ids, resolve_podcast_id, +}; +use rusqlite::{Connection, OptionalExtension as _, Transaction, TransactionBehavior, params}; + +use super::apply::{self, EpisodeSource}; +use super::{LEGACY_IMPORT_OP_PREFIX, PodcastStoreError, now_ms}; + +/// Newest schema version (`PRAGMA user_version`) this build writes. +pub const PODCAST_SCHEMA_VERSION: i64 = 2; + +/// `user_version` of the prototype database of the original podcast PR. +pub const LEGACY_PROTOTYPE_SCHEMA_VERSION: i64 = 1; + +/// Migration 1: the prototype schema of the original podcast PR, verbatim +/// (`furumi-tui` `src/podcasts/store.rs`, `MIGRATIONS[0]`). It versioned +/// itself with `PRAGMA user_version = 1` in the same transaction. +const PROTOTYPE_SCHEMA_V1: &str = " + CREATE TABLE IF NOT EXISTS podcasts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_url TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + author TEXT, + description TEXT, + link TEXT, + image_url TEXT, + cover_url TEXT, + cover_path TEXT, + etag TEXT, + last_modified TEXT, + added_at INTEGER NOT NULL, + last_checked_at INTEGER, + last_error TEXT + ); + CREATE TABLE IF NOT EXISTS podcast_episodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE, + guid TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT, + published_at INTEGER, + duration_seconds REAL, + audio_url TEXT NOT NULL, + mime_type TEXT, + byte_len INTEGER, + link TEXT, + image_url TEXT, + first_seen_at INTEGER NOT NULL, + UNIQUE (podcast_id, guid) + ); + CREATE INDEX IF NOT EXISTS podcast_episodes_by_date + ON podcast_episodes (podcast_id, published_at DESC, id DESC); +"; + +#[cfg(test)] +pub(super) const PROTOTYPE_SCHEMA_V1_FOR_TESTS: &str = PROTOTYPE_SCHEMA_V1; + +/// Columns every prototype `podcasts` table has. +const PROTOTYPE_PODCAST_COLUMNS: &[&str] = &[ + "id", + "feed_url", + "title", + "author", + "description", + "link", + "image_url", + "cover_url", + "cover_path", + "etag", + "last_modified", + "added_at", + "last_checked_at", + "last_error", +]; + +/// Columns every prototype `podcast_episodes` table has. +const PROTOTYPE_EPISODE_COLUMNS: &[&str] = &[ + "id", + "podcast_id", + "guid", + "title", + "description", + "published_at", + "duration_seconds", + "audio_url", + "mime_type", + "byte_len", + "link", + "image_url", + "first_seen_at", +]; + +/// Migration 2: the contract model. See `PODCASTS.md`, "SQLite reference store". +const SCHEMA_V2: &str = " + CREATE TABLE IF NOT EXISTS podcasts ( + podcast_id TEXT PRIMARY KEY, + feed_url TEXT NOT NULL, + title TEXT NOT NULL, + author TEXT, + description TEXT, + link TEXT, + image_url TEXT, + meta_from_feed INTEGER NOT NULL DEFAULT 0, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS podcast_feed_aliases ( + podcast_id TEXT NOT NULL, + feed_url TEXT NOT NULL, + PRIMARY KEY (podcast_id, feed_url) + ); + CREATE INDEX IF NOT EXISTS podcast_feed_aliases_by_url + ON podcast_feed_aliases (feed_url); + CREATE TABLE IF NOT EXISTS episodes ( + episode_id TEXT PRIMARY KEY, + podcast_id TEXT NOT NULL REFERENCES podcasts(podcast_id), + guid TEXT, + title TEXT NOT NULL, + description TEXT, + season INTEGER, + episode_number INTEGER, + episode_type TEXT NOT NULL DEFAULT 'full', + published_at_ms INTEGER, + duration_ms INTEGER, + audio_url TEXT NOT NULL, + mime TEXT, + byte_len INTEGER, + image_url TEXT, + link TEXT, + source TEXT NOT NULL CHECK (source IN ('feed', 'sync')), + in_feed INTEGER NOT NULL DEFAULT 0, + legacy_guid TEXT, + first_seen_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS episodes_by_podcast_date + ON episodes (podcast_id, published_at_ms DESC); + CREATE INDEX IF NOT EXISTS episodes_by_podcast_season + ON episodes (podcast_id, season, episode_number); + CREATE TABLE IF NOT EXISTS podcast_seasons ( + podcast_id TEXT NOT NULL REFERENCES podcasts(podcast_id), + season INTEGER, + episode_count INTEGER NOT NULL, + first_published_at_ms INTEGER, + last_published_at_ms INTEGER + ); + CREATE UNIQUE INDEX IF NOT EXISTS podcast_seasons_key + ON podcast_seasons (podcast_id, IFNULL(season, -1)); + CREATE TABLE IF NOT EXISTS podcast_subscriptions ( + podcast_id TEXT PRIMARY KEY, + subscribed INTEGER NOT NULL, + podcast_json TEXT NOT NULL, + generation INTEGER NOT NULL CHECK (generation >= 0), + hlc_ms INTEGER NOT NULL, + op_id TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS podcast_shares ( + podcast_id TEXT PRIMARY KEY, + shared INTEGER NOT NULL, + generation INTEGER NOT NULL CHECK (generation >= 0), + hlc_ms INTEGER NOT NULL, + op_id TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS episode_progress ( + episode_id TEXT PRIMARY KEY, + podcast_id TEXT NOT NULL, + position_ms INTEGER NOT NULL, + status TEXT NOT NULL + CHECK (status IN ('not_started', 'in_progress', 'completed')), + generation INTEGER NOT NULL CHECK (generation >= 0), + session_id TEXT NOT NULL, + hlc_ms INTEGER NOT NULL, + device_id TEXT NOT NULL, + op_id TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS episode_progress_by_podcast + ON episode_progress (podcast_id); + CREATE TABLE IF NOT EXISTS podcast_listen_sessions ( + session_id TEXT PRIMARY KEY, + episode_id TEXT NOT NULL, + podcast_id TEXT NOT NULL, + device_id TEXT NOT NULL, + started_at_ms INTEGER NOT NULL, + ended_at_ms INTEGER NOT NULL, + start_position_ms INTEGER NOT NULL, + end_position_ms INTEGER NOT NULL, + listened_ms INTEGER NOT NULL, + completed INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS podcast_listen_sessions_by_episode + ON podcast_listen_sessions (episode_id); + CREATE INDEX IF NOT EXISTS podcast_listen_sessions_by_end + ON podcast_listen_sessions (ended_at_ms DESC); + CREATE TABLE IF NOT EXISTS podcast_fetch_state ( + podcast_id TEXT PRIMARY KEY REFERENCES podcasts(podcast_id), + etag TEXT, + last_modified TEXT, + checked_at_ms INTEGER, + last_success_at_ms INTEGER, + next_check_at_ms INTEGER, + last_error TEXT, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + cover_url TEXT, + cover_path TEXT + ); + CREATE TABLE IF NOT EXISTS legacy_quarantine ( + legacy_table TEXT NOT NULL, + legacy_id INTEGER NOT NULL, + reason TEXT NOT NULL, + detail TEXT, + PRIMARY KEY (legacy_table, legacy_id) + ); +"; + +/// Tables a version-2 store owns (the `legacy_*` tables included). +const V2_TABLES: &[&str] = &[ + "podcasts", + "podcast_feed_aliases", + "episodes", + "podcast_seasons", + "podcast_subscriptions", + "podcast_shares", + "episode_progress", + "podcast_listen_sessions", + "podcast_fetch_state", + "legacy_quarantine", + "legacy_podcasts", + "legacy_podcast_episodes", +]; + +fn user_version(conn: &Connection) -> Result { + Ok(conn.query_row("PRAGMA user_version", [], |row| row.get(0))?) +} + +fn user_tables(conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'", + )?; + let names = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::>>()?; + Ok(names) +} + +pub(crate) fn table_columns(conn: &Connection, table: &str) -> Result> { + let mut statement = conn.prepare("SELECT name FROM pragma_table_info(?1)")?; + let names = statement + .query_map([table], |row| row.get::<_, String>(0))? + .collect::>>()?; + Ok(names) +} + +fn unrecognized(reason: impl Into) -> anyhow::Error { + PodcastStoreError::UnrecognizedDatabase(reason.into()).into() +} + +fn has_columns(conn: &Connection, table: &str, required: &[&str]) -> Result { + let columns = table_columns(conn, table)?; + Ok(required.iter().all(|column| columns.contains(*column))) +} + +/// Checks the prototype tables that exist have the prototype shape. +fn check_prototype_shape(conn: &Connection, tables: &BTreeSet) -> Result<()> { + if tables.contains("podcasts") && !has_columns(conn, "podcasts", PROTOTYPE_PODCAST_COLUMNS)? { + return Err(unrecognized( + "table `podcasts` does not have the prototype columns", + )); + } + if tables.contains("podcast_episodes") + && !has_columns(conn, "podcast_episodes", PROTOTYPE_EPISODE_COLUMNS)? + { + return Err(unrecognized( + "table `podcast_episodes` does not have the prototype columns", + )); + } + Ok(()) +} + +/// Read-only check run before anything is written to the file. +/// +/// * `user_version` above [`PODCAST_SCHEMA_VERSION`]: written by a newer +/// Furumi, refused. +/// * 0: must be empty, or hold only the prototype tables (the prototype +/// sets its version in the same transaction, so this is defensive). +/// * 1: the prototype database; its tables must have the prototype shape. +/// * 2: the current schema. +/// +/// Any other table means the path points at a different database (for +/// example `library.db`), which is refused instead of being modified. +pub(super) fn check_openable(conn: &Connection) -> Result<()> { + let version = user_version(conn)?; + if version > PODCAST_SCHEMA_VERSION { + return Err(PodcastStoreError::NewerSchema { + found: version, + supported: PODCAST_SCHEMA_VERSION, + } + .into()); + } + if version < 0 { + return Err(unrecognized(format!("negative user_version {version}"))); + } + let tables = user_tables(conn)?; + let allowed: &[&str] = if version < PODCAST_SCHEMA_VERSION { + &["podcasts", "podcast_episodes"] + } else { + V2_TABLES + }; + if let Some(table) = tables + .iter() + .find(|table| !allowed.contains(&table.as_str())) + { + return Err(unrecognized(format!( + "unexpected table `{table}` at schema version {version}" + ))); + } + if version < PODCAST_SCHEMA_VERSION { + check_prototype_shape(conn, &tables)?; + } else if tables.contains("podcasts") + && !has_columns(conn, "podcasts", &["podcast_id", "feed_url"])? + { + return Err(unrecognized("table `podcasts` is not the version 2 table")); + } + Ok(()) +} + +/// Applies every pending migration, each in its own immediate transaction +/// that re-reads the version (another process may have migrated meanwhile). +pub(super) fn migrate(conn: &mut Connection) -> Result<()> { + for target in 1..=PODCAST_SCHEMA_VERSION { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let version = user_version(&tx)?; + if version > PODCAST_SCHEMA_VERSION { + return Err(PodcastStoreError::NewerSchema { + found: version, + supported: PODCAST_SCHEMA_VERSION, + } + .into()); + } + if version >= target { + continue; + } + match target { + 1 => tx + .execute_batch(PROTOTYPE_SCHEMA_V1) + .context("creating the prototype podcast schema (v1)")?, + 2 => migrate_to_v2(&tx).context("migrating the podcast database to v2")?, + _ => unreachable!("no migration for version {target}"), + } + tx.pragma_update(None, "user_version", target)?; + tx.commit() + .with_context(|| format!("committing podcast schema v{target}"))?; + } + Ok(()) +} + +fn migrate_to_v2(tx: &Transaction<'_>) -> Result<()> { + let tables = user_tables(tx)?; + // Rename, never drop: the prototype data stays available as `legacy_*`. + // Renaming `podcasts` also rewrites the foreign key of the episode table. + if tables.contains("podcasts") && !tables.contains("legacy_podcasts") { + tx.execute_batch("ALTER TABLE podcasts RENAME TO legacy_podcasts")?; + } + if tables.contains("podcast_episodes") && !tables.contains("legacy_podcast_episodes") { + tx.execute_batch("ALTER TABLE podcast_episodes RENAME TO legacy_podcast_episodes")?; + } + tx.execute_batch(SCHEMA_V2)?; + let tables = user_tables(tx)?; + if tables.contains("legacy_podcasts") { + import_prototype(tx, tables.contains("legacy_podcast_episodes"))?; + } + Ok(()) +} + +struct LegacyPodcast { + id: i64, + feed_url: String, + title: String, + author: Option, + description: Option, + link: Option, + image_url: Option, + cover_url: Option, + cover_path: Option, + etag: Option, + last_modified: Option, + added_at: i64, + last_checked_at: Option, + last_error: Option, +} + +struct LegacyEpisode { + id: i64, + guid: String, + title: String, + description: Option, + published_at: Option, + duration_seconds: Option, + audio_url: String, + mime_type: Option, + byte_len: Option, + link: Option, + image_url: Option, + first_seen_at: i64, +} + +fn quarantine( + tx: &Transaction<'_>, + table: &str, + id: i64, + reason: &str, + detail: &str, +) -> Result<()> { + tx.execute( + "INSERT OR IGNORE INTO legacy_quarantine (legacy_table, legacy_id, reason, detail) + VALUES (?1, ?2, ?3, ?4)", + params![table, id, reason, detail], + )?; + Ok(()) +} + +fn secs_to_ms(secs: i64) -> i64 { + secs.saturating_mul(1_000) +} + +/// Imports the prototype tables into the contract model. +/// +/// * Every prototype row is an active subscription: it becomes a +/// subscription register with `subscribed = true`, generation 0, +/// `hlc_ms` = `added_at` and `op_id` = [`LEGACY_IMPORT_OP_PREFIX`] + id, +/// so any later contract write orders after it. Other devices learn it +/// through the snapshot sections. Nothing is shared (private by default). +/// * The podcast id is derived from the canonical feed URL +/// ([`FeedUrl::parse`], falling back to [`FeedUrl::from_user_input`]); rows +/// whose URL cannot be canonicalized are recorded in `legacy_quarantine` +/// and stay in `legacy_podcasts` with their episodes. Rows with the same +/// canonical URL merge into one podcast (the first row's subscription and +/// fetch state win; the episodes of all of them are imported). +/// * Episodes get ids from the contract identity rule over the whole stored +/// document. The prototype stored the enclosure URL as `guid` when the +/// item had none: such rows are imported as GUID-less items and keep the +/// stored value in `legacy_guid`, which a later refresh uses to re-key the +/// episode to its GUID id when the feed does have that GUID. +/// * ETag, Last-Modified, check time, last error and the cover cache move to +/// the local fetch state. The prototype has no progress or history, so no +/// progress register or session is created. +fn import_prototype(tx: &Transaction<'_>, with_episodes: bool) -> Result<()> { + let now = now_ms(); + let legacy: Vec = { + let mut statement = tx.prepare( + "SELECT id, feed_url, title, author, description, link, image_url, cover_url, + cover_path, etag, last_modified, added_at, last_checked_at, last_error + FROM legacy_podcasts ORDER BY id", + )?; + statement + .query_map([], |row| { + Ok(LegacyPodcast { + id: row.get(0)?, + feed_url: row.get(1)?, + title: row.get(2)?, + author: row.get(3)?, + description: row.get(4)?, + link: row.get(5)?, + image_url: row.get(6)?, + cover_url: row.get(7)?, + cover_path: row.get(8)?, + etag: row.get(9)?, + last_modified: row.get(10)?, + added_at: row.get(11)?, + last_checked_at: row.get(12)?, + last_error: row.get(13)?, + }) + })? + .collect::>>()? + }; + let mut known_urls: Vec<(PodcastId, FeedUrl)> = Vec::new(); + for podcast in legacy { + let url = match FeedUrl::parse(&podcast.feed_url) + .or_else(|_| FeedUrl::from_user_input(&podcast.feed_url)) + { + Ok(url) => url, + Err(error) => { + quarantine( + tx, + "legacy_podcasts", + podcast.id, + "invalid feed URL", + &error.to_string(), + )?; + continue; + } + }; + let podcast_id = resolve_podcast_id(&url, known_urls.iter().map(|(id, url)| (id, url))); + let duplicate = known_urls.iter().any(|(id, _)| *id == podcast_id); + known_urls.push((podcast_id.clone(), url.clone())); + if duplicate { + // Same canonical feed as an earlier row: its episodes join that + // podcast; the first row's subscription and fetch state are kept. + if with_episodes { + import_prototype_episodes(tx, &podcast, &podcast_id, now)?; + } + apply::refresh_seasons(tx, &podcast_id)?; + continue; + } + let meta = PodcastMeta { + feed_url: url.clone(), + title: podcast.title.clone(), + author: podcast.author.clone(), + description: podcast.description.clone(), + link: podcast.link.clone(), + image_url: podcast.image_url.clone(), + } + .bounded(); + let added_ms = secs_to_ms(podcast.added_at); + let record = SubscriptionRecord { + podcast_id: podcast_id.clone(), + subscribed: true, + podcast: meta.clone(), + feed_aliases: vec![url.clone()], + generation: 0, + hlc_ms: added_ms, + op_id: format!("{LEGACY_IMPORT_OP_PREFIX}{podcast_id}"), + }; + apply::apply_subscription_record(tx, &record, now)?; + // The prototype metadata came from the feed itself. + tx.execute( + "UPDATE podcasts + SET title = ?2, author = ?3, description = ?4, link = ?5, image_url = ?6, + meta_from_feed = 1, created_at_ms = MIN(created_at_ms, ?7) + WHERE podcast_id = ?1 AND meta_from_feed = 0", + params![ + podcast_id.as_str(), + meta.title, + meta.author, + meta.description, + meta.link, + meta.image_url, + added_ms + ], + )?; + tx.execute( + "INSERT OR IGNORE INTO podcast_fetch_state + (podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms, + next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path) + VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7, ?8, ?9)", + params![ + podcast_id.as_str(), + podcast.etag, + podcast.last_modified, + podcast.last_checked_at.map(secs_to_ms), + podcast + .last_error + .is_none() + .then(|| podcast.last_checked_at.map(secs_to_ms)) + .flatten(), + podcast.last_error, + i64::from(podcast.last_error.is_some()), + podcast.cover_url, + podcast.cover_path, + ], + )?; + if with_episodes { + import_prototype_episodes(tx, &podcast, &podcast_id, now)?; + } + apply::refresh_seasons(tx, &podcast_id)?; + } + Ok(()) +} + +fn import_prototype_episodes( + tx: &Transaction<'_>, + podcast: &LegacyPodcast, + podcast_id: &PodcastId, + now: i64, +) -> Result<()> { + let rows: Vec = { + let mut statement = tx.prepare( + "SELECT id, guid, title, description, published_at, duration_seconds, audio_url, + mime_type, byte_len, link, image_url, first_seen_at + FROM legacy_podcast_episodes WHERE podcast_id = ?1 ORDER BY id", + )?; + statement + .query_map([podcast.id], |row| { + Ok(LegacyEpisode { + id: row.get(0)?, + guid: row.get(1)?, + title: row.get(2)?, + description: row.get(3)?, + published_at: row.get(4)?, + duration_seconds: row.get(5)?, + audio_url: row.get(6)?, + mime_type: row.get(7)?, + byte_len: row.get(8)?, + link: row.get(9)?, + image_url: row.get(10)?, + first_seen_at: row.get(11)?, + }) + })? + .collect::>>()? + }; + let mut usable = Vec::new(); + for row in rows { + if !is_http_url(&row.audio_url) { + quarantine( + tx, + "legacy_podcast_episodes", + row.id, + "invalid audio URL", + &row.audio_url, + )?; + continue; + } + usable.push(row); + } + // The prototype used the enclosure URL as GUID when the item had none. + let real_guid = |row: &LegacyEpisode| { + let guid = row.guid.trim(); + (!guid.is_empty() && guid != row.audio_url.trim()).then(|| guid.to_string()) + }; + let hints: Vec = usable + .iter() + .map(|row| EpisodeIdentityHints { + guid: real_guid(row), + enclosure_url: Some(row.audio_url.clone()), + link: row.link.clone(), + published_at_ms: row.published_at.map(secs_to_ms), + title: Some(row.title.clone()), + }) + .collect(); + let known: Vec = apply::known_episodes(tx, podcast_id)?; + let ids = resolve_episode_ids(podcast_id, podcast.link.as_deref(), &hints, &known); + let mut seen: BTreeMap = BTreeMap::new(); + for ((row, hint), id) in usable.iter().zip(&hints).zip(ids) { + let Some(episode_id) = id else { + quarantine( + tx, + "legacy_podcast_episodes", + row.id, + "episode cannot be identified", + "no usable identity key or duplicate item", + )?; + continue; + }; + if let Some(first) = seen.insert(episode_id.clone(), row.id) { + quarantine( + tx, + "legacy_podcast_episodes", + row.id, + "duplicate episode", + &format!("same identity as legacy episode {first}"), + )?; + continue; + } + let meta = EpisodeMeta { + episode_id, + podcast_id: podcast_id.clone(), + guid: hint.guid.clone(), + title: bounded_title(&row.title), + description: row + .description + .as_deref() + .map(str::trim) + .filter(|text| !text.is_empty()) + .map(|text| bounded_text(text, MAX_DESCRIPTION_CHARS)), + season: None, + episode_number: None, + episode_type: EpisodeType::Full, + published_at_ms: row.published_at.map(secs_to_ms), + duration_ms: row + .duration_seconds + .filter(|secs| secs.is_finite() && *secs > 0.0) + .map(|secs| (secs * 1_000.0).round() as i64), + audio_url: row.audio_url.clone(), + mime: row.mime_type.clone().filter(|mime| { + !mime.is_empty() && mime.len() <= 128 && mime.bytes().all(|b| b.is_ascii_graphic()) + }), + byte_len: row.byte_len.and_then(|len| u64::try_from(len).ok()), + image_url: row.image_url.clone().filter(|url| is_http_url(url)), + link: row.link.clone().filter(|link| link.len() <= MAX_URL_BYTES), + }; + if let Err(error) = meta.validate() { + quarantine( + tx, + "legacy_podcast_episodes", + row.id, + "invalid episode metadata", + &error.to_string(), + )?; + continue; + } + let legacy_guid = hint.guid.is_none().then(|| row.guid.clone()); + let exists: bool = tx + .query_row( + "SELECT 1 FROM episodes WHERE episode_id = ?1", + [meta.episode_id.as_str()], + |_| Ok(true), + ) + .optional()? + .unwrap_or(false); + if exists { + continue; + } + apply::insert_episode( + tx, + &meta, + EpisodeSource::Feed, + true, + legacy_guid.as_deref(), + secs_to_ms(row.first_seen_at), + now, + )?; + } + Ok(()) +} + +fn bounded_title(title: &str) -> String { + let title = title.trim(); + if title.is_empty() { + "Untitled episode".to_string() + } else { + bounded_text(title, MAX_TITLE_CHARS) + } +} diff --git a/crates/furumi-library/src/podcasts/tests.rs b/crates/furumi-library/src/podcasts/tests.rs new file mode 100644 index 0000000..37e8d69 --- /dev/null +++ b/crates/furumi-library/src/podcasts/tests.rs @@ -0,0 +1,1760 @@ +use std::path::Path; + +use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot}; +use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; +use music_dht::podcast::{ + EpisodeId, EpisodeIdentityHints, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, + PodcastMeta, ProgressStatus, resolve_episode_ids, +}; +use rusqlite::{Connection, params}; + +use super::*; + +// --------------------------------------------------------------------------- +// Fixtures + +/// Deterministic stamps: `op_id = :`, `hlc_ms` from a counter. +struct TestStamps { + device: String, + seq: i64, + hlc_ms: i64, +} + +impl TestStamps { + fn new(device: &str, hlc_start: i64) -> Self { + Self { + device: device.to_string(), + seq: 0, + hlc_ms: hlc_start, + } + } +} + +impl StampSource for TestStamps { + fn device_id(&self) -> &str { + &self.device + } + + fn next_stamp(&mut self) -> Result { + self.seq += 1; + self.hlc_ms += 1_000; + Ok(OpStamp::new( + format!("{}:{}", self.device, self.seq), + self.device.clone(), + self.hlc_ms, + )) + } +} + +fn wire(ops: &[PodcastLocalOp]) -> Vec { + ops.iter() + .enumerate() + .map(|(index, op)| op.clone().into_wire(index as i64 + 1)) + .collect() +} + +fn url(value: &str) -> FeedUrl { + FeedUrl::parse(value).unwrap() +} + +fn meta(feed_url: &str, title: &str) -> PodcastMeta { + PodcastMeta { + feed_url: url(feed_url), + title: title.to_string(), + author: Some("Host".to_string()), + description: Some("About the show".to_string()), + link: Some("https://example.test/".to_string()), + image_url: None, + } +} + +fn episode_ref(podcast_url: &str, guid: &str) -> PodcastEpisodeRef { + let feed_url = url(podcast_url); + let podcast_id = PodcastId::from_feed_url(&feed_url); + PodcastEpisodeRef { + episode_id: EpisodeId::from_guid(&podcast_id, guid).unwrap(), + podcast_id, + feed_url, + podcast_title: "Show".to_string(), + episode_title: format!("Episode {guid}"), + audio_url: format!("https://cdn.example.test/{guid}.mp3"), + mime: Some("audio/mpeg".to_string()), + duration_ms: Some(3_600_000), + published_at_ms: Some(1_700_000_000_000), + season: None, + episode_number: None, + image_url: None, + progress_generation: None, + } +} + +struct Item<'a> { + guid: Option<&'a str>, + title: &'a str, + link: Option<&'a str>, + audio: &'a str, + date: Option<&'a str>, + season: Option, + number: Option, + description: Option<&'a str>, +} + +impl<'a> Item<'a> { + fn new(guid: Option<&'a str>, title: &'a str, audio: &'a str) -> Self { + Self { + guid, + title, + link: None, + audio, + date: None, + season: None, + number: None, + description: None, + } + } +} + +fn rss(title: &str, new_feed_url: Option<&str>, items: &[Item<'_>]) -> String { + let mut body = String::new(); + for item in items { + body.push_str(""); + body.push_str(&format!("{}", item.title)); + if let Some(guid) = item.guid { + body.push_str(&format!("{guid}")); + } + if let Some(link) = item.link { + body.push_str(&format!("{link}")); + } + if let Some(date) = item.date { + body.push_str(&format!("{date}")); + } + if let Some(season) = item.season { + body.push_str(&format!("{season}")); + } + if let Some(number) = item.number { + body.push_str(&format!("{number}")); + } + if let Some(description) = item.description { + body.push_str(&format!("{description}")); + } + body.push_str(&format!( + "", + item.audio + )); + body.push_str("\n"); + } + let new_feed = new_feed_url + .map(|url| format!("{url}")) + .unwrap_or_default(); + format!( + r#" + +{title}https://example.test/{new_feed} +{body}"# + ) +} + +fn parse(xml: &str, base: &FeedUrl) -> ParsedFeed { + parse_feed_with_base(xml.as_bytes(), Some(base.as_str())).unwrap() +} + +fn refresh( + store: &PodcastStore, + stamps: &mut TestStamps, + podcast_id: &PodcastId, + feed: &ParsedFeed, + fetched_from: &FeedUrl, + moved_permanently: bool, +) -> RefreshOutcome { + store + .apply_feed( + stamps, + &FeedRefresh { + podcast_id, + feed, + fetched_from, + moved_permanently, + }, + ) + .unwrap() +} + +/// Every permutation of `items` (Heap's algorithm). +fn permutations(items: &[T]) -> Vec> { + fn heap(k: usize, items: &mut Vec, out: &mut Vec>) { + if k <= 1 { + out.push(items.clone()); + return; + } + heap(k - 1, items, out); + for i in 0..k - 1 { + if k.is_multiple_of(2) { + items.swap(i, k - 1); + } else { + items.swap(0, k - 1); + } + heap(k - 1, items, out); + } + } + let mut items = items.to_vec(); + let mut out = Vec::new(); + heap(items.len(), &mut items, &mut out); + out +} + +fn sync_state(store: &PodcastStore) -> SyncSnapshot { + store.export_snapshot().unwrap() +} + +fn tables(conn: &Connection) -> Vec { + let mut statement = conn + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") + .unwrap(); + statement + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>>() + .unwrap() +} + +#[test] +fn store_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +} + +// --------------------------------------------------------------------------- +// Migrations + +/// The prototype migration of the original podcast PR, verbatim +/// (`furumi-tui` `src/podcasts/store.rs`, `MIGRATIONS[0]`). +const PROTOTYPE_MIGRATION_V1: &str = " + CREATE TABLE IF NOT EXISTS podcasts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_url TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + author TEXT, + description TEXT, + link TEXT, + image_url TEXT, + cover_url TEXT, + cover_path TEXT, + etag TEXT, + last_modified TEXT, + added_at INTEGER NOT NULL, + last_checked_at INTEGER, + last_error TEXT + ); + CREATE TABLE IF NOT EXISTS podcast_episodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE, + guid TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT, + published_at INTEGER, + duration_seconds REAL, + audio_url TEXT NOT NULL, + mime_type TEXT, + byte_len INTEGER, + link TEXT, + image_url TEXT, + first_seen_at INTEGER NOT NULL, + UNIQUE (podcast_id, guid) + ); + CREATE INDEX IF NOT EXISTS podcast_episodes_by_date + ON podcast_episodes (podcast_id, published_at DESC, id DESC); + "; + +/// Creates a database exactly as the prototype `Store::open` did. +fn fabricate_prototype(path: &Path) -> Connection { + let conn = Connection::open(path).unwrap(); + conn.pragma_update(None, "foreign_keys", true).unwrap(); + let _: String = conn + .query_row("PRAGMA journal_mode=WAL", [], |row| row.get(0)) + .unwrap(); + conn.execute_batch(&format!( + "BEGIN IMMEDIATE; {PROTOTYPE_MIGRATION_V1}; PRAGMA user_version = 1; COMMIT;" + )) + .unwrap(); + conn +} + +#[allow(clippy::too_many_arguments)] +fn prototype_episode( + conn: &Connection, + podcast: i64, + guid: &str, + title: &str, + published_at: Option, + audio_url: &str, + link: Option<&str>, +) -> i64 { + conn.execute( + "INSERT INTO podcast_episodes + (podcast_id, guid, title, description, published_at, duration_seconds, + audio_url, mime_type, byte_len, link, image_url, first_seen_at) + VALUES (?1, ?2, ?3, 'Notes', ?4, 1801.5, ?5, 'audio/mpeg', 1000, ?6, NULL, 1700000500)", + params![podcast, guid, title, published_at, audio_url, link], + ) + .unwrap(); + conn.last_insert_rowid() +} + +#[test] +fn empty_file_migrates_to_the_current_schema_and_reopens() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested").join(PODCAST_DB_FILE); + let store = PodcastStore::open(&path).unwrap(); + assert_eq!(store.schema_version().unwrap(), PODCAST_SCHEMA_VERSION); + assert_eq!(store.path(), Some(path.as_path())); + { + let conn = store.lock(); + let mode: String = conn + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .unwrap(); + assert_eq!(mode, "wal"); + let fk: i64 = conn + .query_row("PRAGMA foreign_keys", [], |row| row.get(0)) + .unwrap(); + assert_eq!(fk, 1); + let names = tables(&conn); + for table in [ + "podcasts", + "podcast_feed_aliases", + "episodes", + "podcast_seasons", + "podcast_subscriptions", + "podcast_shares", + "episode_progress", + "podcast_listen_sessions", + "podcast_fetch_state", + "legacy_quarantine", + "legacy_podcasts", + "legacy_podcast_episodes", + ] { + assert!(names.iter().any(|name| name == table), "missing {table}"); + } + } + assert!( + store + .list_podcasts(PodcastListFilter::All) + .unwrap() + .is_empty() + ); + drop(store); + // Reopening is idempotent. + let store = PodcastStore::open(&path).unwrap(); + assert_eq!(store.schema_version().unwrap(), PODCAST_SCHEMA_VERSION); + assert!(store.legacy_quarantine().unwrap().is_empty()); +} + +#[test] +fn empty_prototype_database_migrates() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(PODCAST_DB_FILE); + drop(fabricate_prototype(&path)); + let store = PodcastStore::open(&path).unwrap(); + assert_eq!(store.schema_version().unwrap(), PODCAST_SCHEMA_VERSION); + assert!( + store + .list_podcasts(PodcastListFilter::All) + .unwrap() + .is_empty() + ); + let conn = store.lock(); + let names = tables(&conn); + assert!(names.contains(&"legacy_podcasts".to_string())); + assert!(!names.iter().any(|name| name == "podcast_episodes")); +} + +#[test] +fn intermediate_version_one_written_by_this_store_migrates() { + // A store interrupted after migration 1 (or created by it) continues. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(PODCAST_DB_FILE); + { + let mut conn = Connection::open(&path).unwrap(); + let tx = conn.transaction().unwrap(); + tx.execute_batch(super::schema::PROTOTYPE_SCHEMA_V1_FOR_TESTS) + .unwrap(); + tx.pragma_update(None, "user_version", 1).unwrap(); + tx.commit().unwrap(); + } + let store = PodcastStore::open(&path).unwrap(); + assert_eq!(store.schema_version().unwrap(), PODCAST_SCHEMA_VERSION); +} + +#[test] +fn prototype_database_is_imported_without_losing_rows() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(PODCAST_DB_FILE); + let conn = fabricate_prototype(&path); + conn.execute( + "INSERT INTO podcasts (feed_url, title, author, description, link, image_url, + cover_url, cover_path, etag, last_modified, added_at, + last_checked_at, last_error) + VALUES ('HTTPS://Example.TEST:443/feed.xml?token=Ab', 'Show', 'Host', 'About', + 'https://example.test/', 'https://example.test/cover.jpg', + 'https://example.test/cover.jpg', '/covers/1.jpg', '\"etag-1\"', + 'Tue, 05 Mar 2024 10:00:00 GMT', 1700000000, 1700000600, NULL)", + [], + ) + .unwrap(); + let show = conn.last_insert_rowid(); + let with_guid = prototype_episode( + &conn, + show, + "guid-1", + "First", + Some(1_700_000_100), + "https://cdn.example.test/1.mp3", + Some("https://example.test/1"), + ); + // The prototype stored the enclosure URL as guid for items without one. + let without_guid = prototype_episode( + &conn, + show, + "https://cdn.example.test/2.mp3", + "Second", + Some(1_700_000_200), + "https://cdn.example.test/2.mp3", + Some("https://example.test/2"), + ); + let bad_audio = prototype_episode( + &conn, + show, + "guid-3", + "Third", + None, + "ftp://cdn.example.test/3.mp3", + None, + ); + conn.execute( + "INSERT INTO podcasts (feed_url, title, added_at, last_checked_at, last_error) + VALUES ('not a feed url', 'Broken', 1700000000, 1700000600, 'HTTP 500')", + [], + ) + .unwrap(); + let broken = conn.last_insert_rowid(); + prototype_episode( + &conn, + broken, + "b-1", + "Kept", + None, + "https://cdn.example.test/b1.mp3", + None, + ); + // Same canonical URL as `show`: merges into the same podcast. + conn.execute( + "INSERT INTO podcasts (feed_url, title, added_at) + VALUES ('https://example.test/feed.xml?token=Ab#fragment', 'Show again', 1700000900)", + [], + ) + .unwrap(); + let duplicate = conn.last_insert_rowid(); + prototype_episode( + &conn, + duplicate, + "guid-1", + "First (dup)", + Some(1_700_000_100), + "https://cdn.example.test/1.mp3", + None, + ); + drop(conn); + + let store = PodcastStore::open(&path).unwrap(); + assert_eq!(store.schema_version().unwrap(), PODCAST_SCHEMA_VERSION); + + let canonical = url("https://example.test/feed.xml?token=Ab"); + let podcast_id = PodcastId::from_feed_url(&canonical); + let podcasts = store.list_podcasts(PodcastListFilter::Subscribed).unwrap(); + assert_eq!(podcasts.len(), 1); + let summary = &podcasts[0]; + assert_eq!(summary.podcast_id, podcast_id); + assert_eq!(summary.meta.feed_url, canonical); + assert_eq!(summary.meta.title, "Show"); + assert!(summary.subscribed); + assert!(!summary.shared, "imported subscriptions are private"); + assert_eq!(summary.episode_count, 2); + assert_eq!(summary.unplayed_count, 2); + + let register = store.subscription(&podcast_id).unwrap().unwrap(); + assert_eq!(register.generation, 0); + assert_eq!(register.hlc_ms, 1_700_000_000_000); + assert_eq!( + register.op_id, + format!("{LEGACY_IMPORT_OP_PREFIX}{podcast_id}") + ); + assert_eq!(register.feed_aliases, vec![canonical.clone()]); + + // Episode ids follow the contract identity rule. + let guid_id = EpisodeId::from_guid(&podcast_id, "guid-1").unwrap(); + let expected_no_guid = resolve_episode_ids( + &podcast_id, + Some("https://example.test/"), + &[EpisodeIdentityHints { + guid: None, + enclosure_url: Some("https://cdn.example.test/2.mp3".to_string()), + link: Some("https://example.test/2".to_string()), + published_at_ms: Some(1_700_000_200_000), + title: Some("Second".to_string()), + }], + &[], + )[0] + .clone() + .unwrap(); + let first = store.episode(&guid_id).unwrap().unwrap(); + assert_eq!(first.episode.title, "First"); + assert_eq!(first.episode.published_at_ms, Some(1_700_000_100_000)); + assert_eq!(first.episode.duration_ms, Some(1_801_500)); + assert!(first.progress.is_none()); + let second = store.episode(&expected_no_guid).unwrap().unwrap(); + assert_eq!(second.episode.guid, None); + + let state = store.fetch_state(&podcast_id).unwrap(); + assert_eq!(state.etag.as_deref(), Some("\"etag-1\"")); + assert_eq!(state.checked_at_ms, Some(1_700_000_600_000)); + assert_eq!(state.last_success_at_ms, Some(1_700_000_600_000)); + assert_eq!(state.cover_path.as_deref(), Some("/covers/1.jpg")); + assert_eq!(state.consecutive_failures, 0); + + let quarantine = store.legacy_quarantine().unwrap(); + let ids: Vec<(String, i64)> = quarantine + .iter() + .map(|entry| (entry.legacy_table.clone(), entry.legacy_id)) + .collect(); + assert!(ids.contains(&("legacy_podcasts".to_string(), broken))); + assert!(ids.contains(&("legacy_podcast_episodes".to_string(), bad_audio))); + + // Nothing was dropped: every prototype row is still in a legacy table. + { + let conn = store.lock(); + let podcasts: i64 = conn + .query_row("SELECT COUNT(*) FROM legacy_podcasts", [], |row| row.get(0)) + .unwrap(); + let episodes: i64 = conn + .query_row("SELECT COUNT(*) FROM legacy_podcast_episodes", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!((podcasts, episodes), (3, 5)); + let legacy_guid: Option = conn + .query_row( + "SELECT legacy_guid FROM episodes WHERE episode_id = ?1", + [expected_no_guid.as_str()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + legacy_guid.as_deref(), + Some("https://cdn.example.test/2.mp3") + ); + } + // The imported subscription is part of the snapshot; progress is empty. + let snapshot = sync_state(&store); + assert_eq!(snapshot.podcast_subscriptions.len(), 1); + assert!(snapshot.podcast_progress.is_empty()); + let _ = (with_guid, without_guid); + + // A later refresh whose feed has a real GUID equal to the stored enclosure + // URL re-keys the imported episode instead of duplicating it. + let mut stamps = TestStamps::new("device-a", 1_800_000_000_000); + let feed = parse( + &rss( + "Show", + None, + &[ + Item::new(Some("guid-1"), "First", "https://cdn.example.test/1.mp3"), + Item::new( + Some("https://cdn.example.test/2.mp3"), + "Second", + "https://cdn.example.test/2.mp3", + ), + ], + ), + &canonical, + ); + let outcome = refresh(&store, &mut stamps, &podcast_id, &feed, &canonical, false); + assert_eq!(outcome.new_episodes, 0); + let rekeyed = EpisodeId::from_guid(&podcast_id, "https://cdn.example.test/2.mp3").unwrap(); + assert!(store.episode(&rekeyed).unwrap().is_some()); + assert!(store.episode(&expected_no_guid).unwrap().is_none()); + assert_eq!( + store.podcast(&podcast_id).unwrap().unwrap().episode_count, + 2 + ); +} + +#[test] +fn newer_schema_is_refused_without_modifying_the_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(PODCAST_DB_FILE); + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE future (x INTEGER); INSERT INTO future VALUES (1); + PRAGMA user_version = 3;", + ) + .unwrap(); + } + let before = std::fs::read(&path).unwrap(); + let error = PodcastStore::open(&path).unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&PodcastStoreError::NewerSchema { + found: 3, + supported: PODCAST_SCHEMA_VERSION + }) + ); + assert!(error.to_string().contains("newer")); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert!(!dir.path().join(format!("{PODCAST_DB_FILE}-wal")).exists()); +} + +#[test] +fn foreign_databases_are_refused_without_modification() { + let dir = tempfile::tempdir().unwrap(); + // A library database passed by mistake. + let library = dir.path().join("library.db"); + { + let conn = Connection::open(&library).unwrap(); + conn.execute_batch("CREATE TABLE tracks (id INTEGER PRIMARY KEY);") + .unwrap(); + } + let before = std::fs::read(&library).unwrap(); + let error = PodcastStore::open(&library).unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(PodcastStoreError::UnrecognizedDatabase(_)) + )); + assert_eq!(std::fs::read(&library).unwrap(), before); + + // Version 1 whose `podcasts` table is not the prototype's. + let odd = dir.path().join("odd.sqlite3"); + { + let conn = Connection::open(&odd).unwrap(); + conn.execute_batch( + "CREATE TABLE podcasts (id INTEGER PRIMARY KEY, url TEXT); + PRAGMA user_version = 1;", + ) + .unwrap(); + } + let error = PodcastStore::open(&odd).unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(PodcastStoreError::UnrecognizedDatabase(_)) + )); +} + +// --------------------------------------------------------------------------- +// Local actions + +const FEED: &str = "https://example.test/feed.xml"; + +#[test] +fn subscribe_is_idempotent_and_private_until_shared() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let outcome = store.subscribe(&mut stamps, &meta(FEED, "Show")).unwrap(); + assert!(!outcome.already_subscribed); + assert_eq!(outcome.ops.len(), 1); + assert_eq!(outcome.podcast_id, PodcastId::from_feed_url(&url(FEED))); + let SyncOpPayload::PodcastSubscriptionSet { + subscribed, + generation, + feed_aliases, + .. + } = &outcome.ops[0].payload + else { + panic!("subscription op expected"); + }; + assert!(*subscribed); + assert_eq!(*generation, 1); + assert_eq!(feed_aliases, &vec![url(FEED)]); + + let again = store.subscribe(&mut stamps, &meta(FEED, "Show")).unwrap(); + assert!(again.already_subscribed && again.ops.is_empty()); + assert_eq!(stamps.seq, 1, "no stamp drawn for a no-op"); + + let summary = store.podcast(&outcome.podcast_id).unwrap().unwrap(); + assert!(summary.subscribed && !summary.shared); + let shared = store + .set_shared(&mut stamps, &outcome.podcast_id, true) + .unwrap(); + assert_eq!(shared.len(), 1); + assert!(store.podcast(&outcome.podcast_id).unwrap().unwrap().shared); + assert!( + store + .set_shared(&mut stamps, &outcome.podcast_id, true) + .unwrap() + .is_empty() + ); + + // Sharing an unknown podcast is refused. + let other = PodcastId::from_feed_url(&url("https://other.test/")); + let error = store.set_shared(&mut stamps, &other, true).unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(PodcastStoreError::NotSubscribed(_)) + )); +} + +#[test] +fn unsubscribe_keeps_progress_and_history_and_resubscribe_continues() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let podcast_id = store + .subscribe(&mut stamps, &meta(FEED, "Show")) + .unwrap() + .podcast_id; + store.set_shared(&mut stamps, &podcast_id, true).unwrap(); + let episode = episode_ref(FEED, "e1"); + let session = store + .start_session(&mut stamps, &episode, "s1", 0, None) + .unwrap() + .session; + store.checkpoint(&mut stamps, &session, 600_000).unwrap(); + let listened = ListenSession { + session_id: "s1".to_string(), + episode_id: episode.episode_id.clone(), + podcast_id: podcast_id.clone(), + device_id: "device-a".to_string(), + started_at_ms: 10, + ended_at_ms: 600_010, + start_position_ms: 0, + end_position_ms: 600_000, + listened_ms: 600_000, + completed: false, + }; + assert!( + store + .record_listen_session(&mut stamps, &listened, None) + .unwrap() + .is_some() + ); + + let ops = store.unsubscribe(&mut stamps, &podcast_id).unwrap(); + assert_eq!(ops.len(), 2); + assert!(matches!( + ops[0].payload, + SyncOpPayload::PodcastSubscriptionSet { + subscribed: false, + .. + } + )); + assert!(matches!( + ops[1].payload, + SyncOpPayload::PodcastShareSet { shared: false, .. } + )); + let summary = store.podcast(&podcast_id).unwrap().unwrap(); + assert!(!summary.subscribed && !summary.shared); + assert!( + store + .list_podcasts(PodcastListFilter::Subscribed) + .unwrap() + .is_empty() + ); + assert_eq!( + store.list_podcasts(PodcastListFilter::All).unwrap().len(), + 1 + ); + assert_eq!( + store.resume_position_ms(&episode.episode_id).unwrap(), + 600_000 + ); + assert_eq!(store.listen_history(None, 10).unwrap().len(), 1); + assert!( + store + .unsubscribe(&mut stamps, &podcast_id) + .unwrap() + .is_empty() + ); + + // Subscribing again (even through an alias) reuses the podcast and its state. + let again = store.subscribe(&mut stamps, &meta(FEED, "Show")).unwrap(); + assert_eq!(again.podcast_id, podcast_id); + assert_eq!( + store.resume_position_ms(&episode.episode_id).unwrap(), + 600_000 + ); + let summary = store.podcast(&podcast_id).unwrap().unwrap(); + assert!(summary.subscribed); + assert!(!summary.shared, "a new subscription is private again"); +} + +#[test] +fn progress_actions_follow_the_generation_rules() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let episode = episode_ref(FEED, "e1"); + let started = store + .start_session(&mut stamps, &episode, "s1", 5_000, Some(4)) + .unwrap(); + assert_eq!(started.record.generation, 5, "max(local, carried) + 1"); + assert_eq!(started.record.session_id, "s1"); + assert_eq!(started.record.device_id, "device-a"); + let session = started.session; + let CheckpointOutcome::Written(checkpoint) = + store.checkpoint(&mut stamps, &session, 65_000).unwrap() + else { + panic!("checkpoint expected"); + }; + assert_eq!(checkpoint.record.generation, 5); + assert_eq!( + store.resume_position_ms(&episode.episode_id).unwrap(), + 65_000 + ); + + let seek = store.seek(&mut stamps, &session, 10_000).unwrap(); + assert_eq!(seek.record.generation, 6); + let session = seek.session; + assert_eq!(session.generation, 6); + + // Another device took over at a greater generation: the old session stops. + let remote = SyncOpWire { + op_id: "device-b:1".to_string(), + origin_device_id: "device-b".to_string(), + seq: 1, + hlc_ms: 1, + payload: SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms: 1_000, + status: ProgressStatus::InProgress, + generation: 7, + session_id: "s2".to_string(), + episode: Some(episode.clone()), + }, + }; + assert_eq!( + store.apply_remote_op(&remote).unwrap(), + ApplyOutcome::Changed + ); + assert!(matches!( + store.checkpoint(&mut stamps, &session, 70_000).unwrap(), + CheckpointOutcome::Superseded { .. } + )); + assert_eq!( + store.resume_position_ms(&episode.episode_id).unwrap(), + 1_000 + ); + + let played = store.mark_played(&mut stamps, &episode).unwrap(); + assert_eq!(played.record.generation, 8); + assert_eq!(played.record.status, ProgressStatus::Completed); + assert_eq!(store.resume_position_ms(&episode.episode_id).unwrap(), 0); + assert_eq!( + store.completed_listen_count(&episode.episode_id).unwrap(), + 0, + "mark played invents no session" + ); + let unplayed = store.mark_unplayed(&mut stamps, &episode).unwrap(); + assert_eq!(unplayed.record.status, ProgressStatus::NotStarted); + let restart = store.restart(&mut stamps, &episode, "s3").unwrap(); + assert_eq!( + (restart.record.position_ms, restart.record.status), + (0, ProgressStatus::InProgress) + ); + let done = store + .complete_session(&mut stamps, &restart.session, 3_590_000) + .unwrap(); + assert_eq!(done.record.status, ProgressStatus::Completed); + assert!(matches!( + store + .checkpoint(&mut stamps, &done.session, 3_595_000) + .unwrap(), + CheckpointOutcome::AlreadyCompleted { .. } + )); + // The episode learned through sync is in the catalog. + let reference = store.episode_ref(&episode.episode_id).unwrap().unwrap(); + assert_eq!(reference.progress_generation, Some(done.record.generation)); + assert!( + store + .episode_track(&episode.episode_id) + .unwrap() + .unwrap() + .is_podcast() + ); +} + +#[test] +fn listen_sessions_are_deduplicated_and_counted_once() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let episode = episode_ref(FEED, "e1"); + let mut session = ListenSession { + session_id: "s1".to_string(), + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + device_id: "device-a".to_string(), + started_at_ms: 0, + ended_at_ms: 10_000, + start_position_ms: 0, + end_position_ms: 10_000, + listened_ms: 10_000, + completed: false, + }; + // Below the history threshold: nothing recorded. + assert!( + store + .record_listen_session(&mut stamps, &session, Some(&episode)) + .unwrap() + .is_none() + ); + session.listened_ms = 3_600_000; + session.ended_at_ms = 3_600_000; + session.completed = true; + let op = store + .record_listen_session(&mut stamps, &session, Some(&episode)) + .unwrap() + .unwrap(); + assert!( + store + .record_listen_session(&mut stamps, &session, Some(&episode)) + .unwrap() + .is_none() + ); + let op = op.into_wire(1); + assert_eq!(store.apply_remote_op(&op).unwrap(), ApplyOutcome::Unchanged); + assert_eq!( + store.completed_listen_count(&episode.episode_id).unwrap(), + 1 + ); + let history = store.listen_history(Some(&episode.podcast_id), 10).unwrap(); + assert_eq!(history.len(), 1); + assert_eq!(history[0].episode_title.as_deref(), Some("Episode e1")); + + // Another device's session with a foreign device id is refused locally. + let mut foreign = session.clone(); + foreign.session_id = "s2".to_string(); + foreign.device_id = "device-b".to_string(); + assert!( + store + .record_listen_session(&mut stamps, &foreign, None) + .is_err() + ); +} + +#[test] +fn remote_ops_are_validated_and_non_podcast_ops_ignored() { + let store = PodcastStore::open_in_memory().unwrap(); + let episode = episode_ref(FEED, "e1"); + let mut bad = SyncOpWire { + op_id: "device-b:1".to_string(), + origin_device_id: "device-b".to_string(), + seq: 1, + hlc_ms: 1, + payload: SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms: -5, + status: ProgressStatus::InProgress, + generation: 1, + session_id: "s".to_string(), + episode: None, + }, + }; + assert!(matches!( + store.apply_remote_op(&bad).unwrap(), + ApplyOutcome::Rejected(_) + )); + bad.payload = SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms: 5, + status: ProgressStatus::InProgress, + generation: u64::MAX, + session_id: "s".to_string(), + episode: None, + }; + assert!(matches!( + store.apply_remote_op(&bad).unwrap(), + ApplyOutcome::Rejected(_) + )); + assert!(store.progress(&episode.episode_id).unwrap().is_none()); + bad.payload = SyncOpPayload::PlaylistDeleted { + playlist_id: "p".to_string(), + }; + assert_eq!(store.apply_remote_op(&bad).unwrap(), ApplyOutcome::Ignored); +} + +// --------------------------------------------------------------------------- +// Convergence + +fn assert_converges(ops: &[SyncOpWire], extra_snapshots: &[SyncSnapshot]) -> SyncSnapshot { + let reference = PodcastStore::open_in_memory().unwrap(); + for op in ops { + reference.apply_remote_op(op).unwrap(); + } + let expected = sync_state(&reference); + for order in permutations(ops) { + let store = PodcastStore::open_in_memory().unwrap(); + for op in &order { + store.apply_remote_op(op).unwrap(); + } + // Duplicate delivery changes nothing. + for op in order.iter().rev() { + assert_ne!(store.apply_remote_op(op).unwrap(), ApplyOutcome::Changed); + } + assert_eq!(sync_state(&store), expected); + } + // Recovery through the snapshot gives the same state. + let restored = PodcastStore::open_in_memory().unwrap(); + restored.apply_snapshot(&expected).unwrap(); + assert_eq!(sync_state(&restored), expected); + // Snapshot first, then the operations again (and stale snapshots). + for snapshot in extra_snapshots { + restored.apply_snapshot(snapshot).unwrap(); + } + for op in ops { + restored.apply_remote_op(op).unwrap(); + } + assert_eq!(sync_state(&restored), expected); + expected +} + +#[test] +fn subscription_share_and_alias_ops_converge_in_every_order() { + let a = PodcastStore::open_in_memory().unwrap(); + let b = PodcastStore::open_in_memory().unwrap(); + // Device A's clock runs ten minutes ahead. + let mut stamps_a = TestStamps::new("device-a", 600_000); + let mut stamps_b = TestStamps::new("device-b", 0); + + let subscribe = a.subscribe(&mut stamps_a, &meta(FEED, "Show")).unwrap(); + let podcast_id = subscribe.podcast_id.clone(); + let shared = a.set_shared(&mut stamps_a, &podcast_id, true).unwrap(); + let snapshot_after_share = sync_state(&a); + for op in wire(&subscribe.ops).iter().chain(&wire(&shared)) { + b.apply_remote_op(op).unwrap(); + } + // B saw everything and unsubscribes with a slower clock. + let unsubscribe = b.unsubscribe(&mut stamps_b, &podcast_id).unwrap(); + // A learns a redirect target while offline. + let feed = parse( + &rss( + "Show", + None, + &[Item::new( + Some("e1"), + "One", + "https://cdn.example.test/1.mp3", + )], + ), + &url("https://new.example.test/feed.xml"), + ); + let moved = refresh( + &a, + &mut stamps_a, + &podcast_id, + &feed, + &url("https://new.example.test/feed.xml"), + true, + ); + assert_eq!(moved.ops.len(), 1); + + let mut ops = Vec::new(); + ops.extend(wire(&subscribe.ops)); + ops.extend(wire(&shared)); + ops.extend(wire(&unsubscribe)); + ops.extend(wire(&moved.ops)); + assert_eq!(ops.len(), 5); + let state = assert_converges(&ops, &[snapshot_after_share]); + + // Unsubscribed on every device despite A's fast clock; share revoked. + let store = PodcastStore::open_in_memory().unwrap(); + store.apply_snapshot(&state).unwrap(); + let summary = store.podcast(&podcast_id).unwrap().unwrap(); + assert!(!summary.subscribed && !summary.shared); + let aliases = store.feed_aliases(&podcast_id).unwrap(); + assert!(aliases.contains(&url("https://new.example.test/feed.xml"))); + assert_eq!( + store + .resolve_podcast_id(&url("https://new.example.test/feed.xml")) + .unwrap(), + podcast_id + ); +} + +#[test] +fn progress_and_session_ops_converge_in_every_order() { + let a = PodcastStore::open_in_memory().unwrap(); + let b = PodcastStore::open_in_memory().unwrap(); + let mut stamps_a = TestStamps::new("device-a", 600_000); + let mut stamps_b = TestStamps::new("device-b", 0); + let episode = episode_ref(FEED, "e1"); + + let start = a + .start_session(&mut stamps_a, &episode, "s-a", 0, None) + .unwrap(); + let CheckpointOutcome::Written(checkpoint) = a + .checkpoint(&mut stamps_a, &start.session, 120_000) + .unwrap() + else { + panic!("checkpoint expected"); + }; + b.apply_remote_op(&start.op.clone().into_wire(1)).unwrap(); + // B seeks back after observing A's session, with a slower clock. + let b_start = b + .start_session(&mut stamps_b, &episode, "s-b", 0, None) + .unwrap(); + let seek = b.seek(&mut stamps_b, &b_start.session, 30_000).unwrap(); + // A's late checkpoint of the old session (it has not seen B). + let CheckpointOutcome::Written(late) = a + .checkpoint(&mut stamps_a, &start.session, 240_000) + .unwrap() + else { + panic!("checkpoint expected"); + }; + let session = |listened_ms: i64, completed: bool| ListenSession { + session_id: "s-a".to_string(), + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + device_id: "device-a".to_string(), + started_at_ms: 0, + ended_at_ms: listened_ms, + start_position_ms: 0, + end_position_ms: listened_ms, + listened_ms, + completed, + }; + let first = a + .record_listen_session(&mut stamps_a, &session(120_000, false), Some(&episode)) + .unwrap() + .unwrap(); + let grown = a + .record_listen_session(&mut stamps_a, &session(240_000, false), Some(&episode)) + .unwrap() + .unwrap(); + + let ops: Vec = [start.op, checkpoint.op, seek.op, late.op, first, grown] + .into_iter() + .enumerate() + .map(|(index, op)| op.into_wire(index as i64 + 1)) + .collect(); + let state = assert_converges(&ops, &[]); + // B's seek (greater generation) beats A's later-stamped checkpoint. + assert_eq!(state.podcast_progress.len(), 1); + assert_eq!(state.podcast_progress[0].position_ms, 30_000); + assert_eq!(state.podcast_progress[0].device_id, "device-b"); + assert_eq!(state.podcast_listen_sessions.len(), 1); + assert_eq!(state.podcast_listen_sessions[0].listened_ms, 240_000); +} + +#[test] +fn snapshot_import_never_loses_newer_local_state() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let episode = episode_ref(FEED, "e1"); + store.subscribe(&mut stamps, &meta(FEED, "Show")).unwrap(); + let old = sync_state(&store); + let start = store + .start_session(&mut stamps, &episode, "s1", 0, None) + .unwrap(); + store.seek(&mut stamps, &start.session, 42_000).unwrap(); + let podcast_id = episode.podcast_id.clone(); + store.unsubscribe(&mut stamps, &podcast_id).unwrap(); + let newer = sync_state(&store); + let stats = store.apply_snapshot(&old).unwrap(); + assert_eq!(stats.changed, 0); + assert_eq!(sync_state(&store), newer); + assert_eq!( + store.resume_position_ms(&episode.episode_id).unwrap(), + 42_000 + ); + + // Over-limit sections are ignored as a whole. + let huge = SyncSnapshot { + podcast_shares: vec![ + music_dht::podcast::ShareRecord { + podcast_id: podcast_id.clone(), + shared: true, + generation: 99, + hlc_ms: 1, + op_id: "x".to_string(), + }; + music_dht::podcast::MAX_SNAPSHOT_SHARES + 1 + ], + ..SyncSnapshot::default() + }; + let stats = store.apply_snapshot(&huge).unwrap(); + assert!(stats.ignored_over_limits); + assert_eq!(sync_state(&store), newer); + // Invalid records are rejected one by one. + let invalid = SyncSnapshot { + podcast_shares: vec![music_dht::podcast::ShareRecord { + podcast_id, + shared: true, + generation: 99, + hlc_ms: 1, + op_id: String::new(), + }], + ..SyncSnapshot::default() + }; + assert_eq!(store.apply_snapshot(&invalid).unwrap().rejected, 1); + assert_eq!(sync_state(&store), newer); +} + +#[test] +fn fetch_state_and_catalog_are_not_in_the_snapshot() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let podcast_id = store + .subscribe(&mut stamps, &meta(FEED, "Show")) + .unwrap() + .podcast_id; + store + .record_fetch_success( + &podcast_id, + 5, + Some("\"secret-etag\""), + Some("lm-value"), + Some(99), + ) + .unwrap(); + assert_eq!( + store + .record_fetch_failure(&podcast_id, 6, "connection refused", Some(100)) + .unwrap(), + 1 + ); + store + .set_cover_cache( + &podcast_id, + Some("https://example.test/c.jpg"), + Some("/tmp/cover-path"), + ) + .unwrap(); + let state = store.fetch_state(&podcast_id).unwrap(); + assert_eq!(state.etag.as_deref(), Some("\"secret-etag\"")); + assert_eq!(state.last_success_at_ms, Some(5)); + assert_eq!(state.last_error.as_deref(), Some("connection refused")); + let feed = parse( + &rss( + "Show", + None, + &[Item { + description: Some("catalog-only-description"), + ..Item::new(Some("e1"), "One", "https://cdn.example.test/1.mp3") + }], + ), + &url(FEED), + ); + refresh(&store, &mut stamps, &podcast_id, &feed, &url(FEED), false); + + let mut snapshot = SyncSnapshot::default(); + store + .fill_snapshot( + &mut snapshot, + music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES, + ) + .unwrap(); + let json = serde_json::to_string(&snapshot).unwrap(); + for secret in [ + "secret-etag", + "lm-value", + "connection refused", + "cover-path", + "catalog-only-description", + ] { + assert!(!json.contains(secret), "{secret} leaked into the snapshot"); + } + assert_eq!(snapshot.podcast_subscriptions.len(), 1); + + assert_eq!( + store.podcasts_due_for_refresh(99, 10).unwrap(), + Vec::<(PodcastId, FeedUrl)>::new() + ); + assert_eq!(store.podcasts_due_for_refresh(100, 10).unwrap().len(), 1); + store + .record_fetch_not_modified(&podcast_id, 7, None) + .unwrap(); + let state = store.fetch_state(&podcast_id).unwrap(); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.etag.as_deref(), Some("\"secret-etag\"")); +} + +// --------------------------------------------------------------------------- +// Feed refresh + +#[test] +fn feed_refresh_keeps_ids_and_progress_when_metadata_changes() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let feed_url = url(FEED); + let podcast_id = store + .subscribe(&mut stamps, &meta(FEED, "Show")) + .unwrap() + .podcast_id; + let v1 = parse( + &rss( + "Show", + None, + &[ + Item { + season: Some(1), + number: Some(1), + date: Some("Mon, 01 Jan 2024 10:00:00 GMT"), + ..Item::new( + Some("g1"), + "Guid one", + "https://cdn.example.test/g1.mp3?v=1", + ) + }, + Item { + link: Some("https://example.test/ep/2"), + season: Some(1), + number: Some(2), + date: Some("Mon, 08 Jan 2024 10:00:00 GMT"), + ..Item::new(None, "Linked two", "https://cdn.example.test/2.mp3?v=1") + }, + Item { + date: Some("Mon, 15 Jan 2024 10:00:00 GMT"), + ..Item::new(None, "Dated three", "https://cdn.example.test/3.mp3?v=1") + }, + Item::new( + Some("g4"), + "Leaves the feed", + "https://cdn.example.test/4.mp3", + ), + ], + ), + &feed_url, + ); + let first = refresh(&store, &mut stamps, &podcast_id, &v1, &feed_url, false); + assert_eq!(first.new_episodes, 4); + assert!(first.ops.is_empty()); + let mut episodes = store + .episodes(&EpisodeQuery::new(podcast_id.clone())) + .unwrap(); + assert_eq!(episodes.len(), 4); + episodes.sort_by(|a, b| a.episode.title.cmp(&b.episode.title)); + let ids: Vec = episodes + .iter() + .map(|item| item.episode.episode_id.clone()) + .collect(); + let progress_before: Vec = ids + .iter() + .enumerate() + .map(|(index, id)| { + let reference = store.episode_ref(id).unwrap().unwrap(); + let start = store + .start_session(&mut stamps, &reference, &format!("s{index}"), 0, None) + .unwrap(); + store + .seek(&mut stamps, &start.session, 10_000 * (index as i64 + 1)) + .unwrap(); + 10_000 * (index as i64 + 1) + }) + .collect(); + let progress_snapshot = sync_state(&store).podcast_progress; + + // Titles, audio URLs and dates of identified items change; one item leaves. + let v2 = parse( + &rss( + "Show (renamed)", + None, + &[ + Item { + season: Some(1), + number: Some(1), + date: Some("Mon, 01 Jan 2024 10:00:00 GMT"), + ..Item::new( + Some("g1"), + "Guid one, remastered", + "https://cdn2.example.test/g1.mp3", + ) + }, + Item { + link: Some("https://example.test/ep/2"), + season: Some(1), + number: Some(2), + date: Some("Mon, 08 Jan 2024 10:00:00 GMT"), + ..Item::new( + None, + "Linked two (edited)", + "https://cdn2.example.test/2.mp3", + ) + }, + Item { + date: Some("Mon, 15 Jan 2024 10:00:00 GMT"), + ..Item::new(None, "Dated three", "https://cdn2.example.test/3.mp3") + }, + ], + ), + &feed_url, + ); + let second = refresh(&store, &mut stamps, &podcast_id, &v2, &feed_url, false); + assert_eq!(second.new_episodes, 0); + assert_eq!(second.updated_episodes, 3); + let after = store + .episodes(&EpisodeQuery::new(podcast_id.clone())) + .unwrap(); + assert_eq!(after.len(), 4, "the episode that left the feed is kept"); + for (index, id) in ids.iter().enumerate() { + let item = store.episode(id).unwrap().unwrap(); + assert_eq!(item.resume_position_ms(), progress_before[index]); + if item.episode.guid.as_deref() == Some("g4") { + assert!(!item.in_feed); + } else { + assert!(item.in_feed); + assert!( + item.episode + .audio_url + .starts_with("https://cdn2.example.test/") + ); + } + } + assert_eq!(sync_state(&store).podcast_progress, progress_snapshot); + assert_eq!( + store.podcast(&podcast_id).unwrap().unwrap().meta.title, + "Show (renamed)" + ); + // The register keeps the subscription metadata: refreshes never write it. + assert_eq!( + store + .subscription(&podcast_id) + .unwrap() + .unwrap() + .podcast + .title, + "Show" + ); +} + +#[test] +fn feed_refresh_records_aliases_once() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let old_url = url(FEED); + let podcast_id = store + .subscribe(&mut stamps, &meta(FEED, "Show")) + .unwrap() + .podcast_id; + let items = [Item::new( + Some("g1"), + "One", + "https://cdn.example.test/1.mp3", + )]; + let temporary = url("https://mirror.example.test/feed.xml"); + let feed = parse(&rss("Show", None, &items), &temporary); + let outcome = refresh(&store, &mut stamps, &podcast_id, &feed, &temporary, false); + assert!( + outcome.ops.is_empty(), + "temporary redirects are not aliases" + ); + assert_eq!(outcome.feed_url.as_ref(), Some(&old_url)); + + let moved = url("https://new.example.test/feed.xml"); + let feed = parse( + &rss("Show", Some("https://newest.example.test/rss?k=1"), &items), + &moved, + ); + let outcome = refresh(&store, &mut stamps, &podcast_id, &feed, &moved, true); + assert_eq!(outcome.ops.len(), 1); + let newest = url("https://newest.example.test/rss?k=1"); + assert_eq!(outcome.feed_url.as_ref(), Some(&newest)); + assert!(matches!( + outcome.ops[0].payload, + SyncOpPayload::PodcastFeedAliasesAdded { .. } + )); + let again = refresh(&store, &mut stamps, &podcast_id, &feed, &newest, false); + assert!( + again.ops.is_empty(), + "no op when the alias set does not change" + ); + for alias in [&old_url, &moved, &newest] { + assert_eq!(store.resolve_podcast_id(alias).unwrap(), podcast_id); + } + // Subscribing through an alias resolves to the same podcast. + let outcome = store + .subscribe( + &mut stamps, + &meta("https://new.example.test/feed.xml", "Show"), + ) + .unwrap(); + assert_eq!(outcome.podcast_id, podcast_id); + assert!(outcome.already_subscribed); + // Refreshing an unknown podcast fails. + let unknown = PodcastId::from_feed_url(&url("https://unknown.test/")); + assert!( + store + .apply_feed( + &mut stamps, + &FeedRefresh { + podcast_id: &unknown, + feed: &feed, + fetched_from: &newest, + moved_permanently: false, + }, + ) + .is_err() + ); +} + +#[test] +fn episodes_learned_through_sync_keep_their_ids_after_refresh() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-b", 1_000); + let feed_url = url(FEED); + let podcast_id = PodcastId::from_feed_url(&feed_url); + // Device A identified a GUID-less episode by its link; B learns it from + // a progress operation before it ever fetches the feed. + let item = EpisodeIdentityHints { + guid: None, + enclosure_url: Some("https://cdn.example.test/x.mp3".to_string()), + link: Some("https://example.test/ep/x".to_string()), + published_at_ms: None, + title: Some("X".to_string()), + }; + let id = resolve_episode_ids(&podcast_id, Some("https://example.test/"), &[item], &[])[0] + .clone() + .unwrap(); + let reference = PodcastEpisodeRef { + episode_id: id.clone(), + episode_title: "X".to_string(), + audio_url: "https://cdn.example.test/x.mp3".to_string(), + ..episode_ref(FEED, "unused") + }; + let op = SyncOpWire { + op_id: "device-a:9".to_string(), + origin_device_id: "device-a".to_string(), + seq: 9, + hlc_ms: 9, + payload: SyncOpPayload::PodcastProgressSet { + episode_id: id.clone(), + podcast_id: podcast_id.clone(), + position_ms: 77_000, + status: ProgressStatus::InProgress, + generation: 3, + session_id: "s".to_string(), + episode: Some(reference), + }, + }; + assert_eq!(store.apply_remote_op(&op).unwrap(), ApplyOutcome::Changed); + assert!( + store + .list_podcasts(PodcastListFilter::Subscribed) + .unwrap() + .is_empty() + ); + store.subscribe(&mut stamps, &meta(FEED, "Show")).unwrap(); + // The feed now has a different link for the item, but the same enclosure. + let feed = parse( + &rss( + "Show", + None, + &[Item { + link: Some("https://example.test/ep/x-renamed"), + ..Item::new(None, "X (renamed)", "https://cdn.example.test/x.mp3") + }], + ), + &feed_url, + ); + let outcome = refresh(&store, &mut stamps, &podcast_id, &feed, &feed_url, false); + assert_eq!((outcome.new_episodes, outcome.updated_episodes), (0, 1)); + let item = store.episode(&id).unwrap().unwrap(); + assert_eq!(item.episode.title, "X (renamed)"); + assert_eq!(item.resume_position_ms(), 77_000); +} + +// --------------------------------------------------------------------------- +// Queries + +#[test] +fn seasons_filters_sorting_and_search() { + let store = PodcastStore::open_in_memory().unwrap(); + let mut stamps = TestStamps::new("device-a", 1_000); + let feed_url = url(FEED); + let podcast_id = store + .subscribe(&mut stamps, &meta(FEED, "Show")) + .unwrap() + .podcast_id; + let feed = parse( + &rss( + "Show", + None, + &[ + Item { + season: Some(2), + number: Some(1), + date: Some("Mon, 01 Apr 2024 10:00:00 GMT"), + description: Some("Café talk"), + ..Item::new( + Some("s2e1"), + "Season two opener", + "https://cdn.example.test/a.mp3", + ) + }, + Item { + season: Some(1), + number: Some(2), + date: Some("Mon, 08 Jan 2024 10:00:00 GMT"), + ..Item::new(Some("s1e2"), "Second", "https://cdn.example.test/b.mp3") + }, + Item { + season: Some(1), + number: Some(1), + date: Some("Mon, 01 Jan 2024 10:00:00 GMT"), + ..Item::new(Some("s1e1"), "Pilot", "https://cdn.example.test/c.mp3") + }, + Item::new( + Some("bonus"), + "Bonus without season", + "https://cdn.example.test/d.mp3", + ), + ], + ), + &feed_url, + ); + refresh(&store, &mut stamps, &podcast_id, &feed, &feed_url, false); + let pilot = EpisodeId::from_guid(&podcast_id, "s1e1").unwrap(); + let reference = store.episode_ref(&pilot).unwrap().unwrap(); + store.mark_played(&mut stamps, &reference).unwrap(); + let second = store + .episode_ref(&EpisodeId::from_guid(&podcast_id, "s1e2").unwrap()) + .unwrap() + .unwrap(); + store + .start_session(&mut stamps, &second, "s", 1_000, None) + .unwrap(); + + let seasons = store.seasons(&podcast_id).unwrap(); + let shape: Vec<(Option, u64, u64)> = seasons + .iter() + .map(|season| (season.season, season.episode_count, season.unplayed_count)) + .collect(); + assert_eq!(shape, vec![(Some(1), 2, 1), (Some(2), 1, 1), (None, 1, 1)]); + let summary = store.podcast(&podcast_id).unwrap().unwrap(); + assert_eq!( + ( + summary.episode_count, + summary.unplayed_count, + summary.in_progress_count, + summary.season_count + ), + (4, 3, 1, 2) + ); + + let titles = |query: EpisodeQuery| -> Vec { + store + .episodes(&query) + .unwrap() + .into_iter() + .map(|item| item.episode.title) + .collect() + }; + let base = EpisodeQuery::new(podcast_id.clone()); + assert_eq!( + titles(base.clone()), + vec![ + "Season two opener", + "Second", + "Pilot", + "Bonus without season" + ] + ); + assert_eq!( + titles(EpisodeQuery { + sort: EpisodeSort::EpisodeNumber, + ..base.clone() + }), + vec![ + "Pilot", + "Second", + "Season two opener", + "Bonus without season" + ] + ); + assert_eq!( + titles(EpisodeQuery { + season: SeasonFilter::Season(1), + sort: EpisodeSort::OldestFirst, + ..base.clone() + }), + vec!["Pilot", "Second"] + ); + assert_eq!( + titles(EpisodeQuery { + season: SeasonFilter::NoSeason, + ..base.clone() + }), + vec!["Bonus without season"] + ); + assert_eq!( + titles(EpisodeQuery { + played: PlayedFilter::Played, + ..base.clone() + }), + vec!["Pilot"] + ); + assert_eq!( + titles(EpisodeQuery { + played: PlayedFilter::InProgress, + ..base.clone() + }), + vec!["Second"] + ); + assert_eq!( + titles(EpisodeQuery { + played: PlayedFilter::NotStarted, + ..base.clone() + }) + .len(), + 2 + ); + assert_eq!( + titles(EpisodeQuery { + search: Some("CAFE".to_string()), + ..base.clone() + }), + Vec::::new(), + "normalization keeps accents distinct from their ASCII form" + ); + assert_eq!( + titles(EpisodeQuery { + search: Some("CAFÉ".to_string()), + ..base.clone() + }), + vec!["Season two opener"] + ); + assert_eq!( + titles(EpisodeQuery { + search: Some("pilot".to_string()), + limit: Some(1), + ..base.clone() + }), + vec!["Pilot"] + ); + assert_eq!( + titles(EpisodeQuery { + limit: Some(2), + offset: 1, + ..base + }), + vec!["Second", "Pilot"] + ); +} + +#[test] +fn concurrent_openers_migrate_once() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(PODCAST_DB_FILE); + drop(fabricate_prototype(&path)); + let handles: Vec<_> = (0..4) + .map(|_| { + let path = path.clone(); + std::thread::spawn(move || { + PodcastStore::open(&path).map(|store| store.schema_version()) + }) + }) + .collect(); + for handle in handles { + assert_eq!( + handle.join().unwrap().unwrap().unwrap(), + PODCAST_SCHEMA_VERSION + ); + } +}