diff --git a/src/federation/devices.rs b/src/federation/devices.rs index e7db3dd..4a800cf 100644 --- a/src/federation/devices.rs +++ b/src/federation/devices.rs @@ -1782,20 +1782,7 @@ async fn record_local_op( ); } - let now = now_ms(); - let row = sqlx::query( - "UPDATE furumusic__fed_device_identity - SET local_seq = local_seq + 1, - last_hlc_ms = GREATEST($2, last_hlc_ms + 1) - WHERE user_id = $1 - RETURNING local_seq, last_hlc_ms", - ) - .bind(user_id) - .bind(now) - .fetch_one(pool) - .await?; - let seq: i64 = row.get("local_seq"); - let hlc_ms: i64 = row.get("last_hlc_ms"); + let (seq, hlc_ms) = reserve_local_stamp(pool, user_id).await?; let op_id = format!("{}:{seq}", identity.device_id); let op = SyncOpWire { op_id, @@ -1805,6 +1792,40 @@ async fn record_local_op( payload, }; insert_op(pool, user_id, &op).await?; + advance_vector(pool, user_id, &op.origin_device_id, op.seq).await?; + let _ = apply_op(pool, Arc::new(PlayerDeviceHub::default()), user_id, &op).await?; + Ok(()) +} + +/// Reserves the next local `(seq, hlc_ms)` of the account's web device. +/// +/// `hlc_ms` is the wall clock, but always past the previous local +/// operation. Inside a transaction the identity row stays locked until it +/// ends, so local operations of one account are stamped in commit order. +async fn reserve_local_stamp<'e>( + db: impl sqlx::PgExecutor<'e>, + user_id: i64, +) -> Result<(i64, i64)> { + let row = sqlx::query( + "UPDATE furumusic__fed_device_identity + SET local_seq = local_seq + 1, + last_hlc_ms = GREATEST($2, last_hlc_ms + 1) + WHERE user_id = $1 + RETURNING local_seq, last_hlc_ms", + ) + .bind(user_id) + .bind(now_ms()) + .fetch_one(db) + .await?; + Ok((row.get("local_seq"), row.get("last_hlc_ms"))) +} + +async fn advance_vector<'e>( + db: impl sqlx::PgExecutor<'e>, + user_id: i64, + device_id: &str, + seq: i64, +) -> Result<()> { sqlx::query( "INSERT INTO furumusic__fed_sync_vector (user_id, device_id, max_seq) VALUES ($1, $2, $3) @@ -1812,14 +1833,53 @@ async fn record_local_op( max_seq = GREATEST(furumusic__fed_sync_vector.max_seq, EXCLUDED.max_seq)", ) .bind(user_id) - .bind(&op.origin_device_id) - .bind(op.seq) - .execute(pool) + .bind(device_id) + .bind(seq) + .execute(db) .await?; - let _ = apply_op(pool, Arc::new(PlayerDeviceHub::default()), user_id, &op).await?; Ok(()) } +/// Device id of the account's web device, creating the identity on first +/// use. Local podcast actions stamp their operations with it. +#[cfg_attr(not(test), allow(dead_code))] // podcast UI and RSS job: later commits +pub(crate) async fn local_device_id(pool: &sqlx::PgPool, user_id: i64) -> Result { + Ok(ensure_identity(pool, user_id, "").await?.device_id) +} + +/// Appends a local operation of `device_id` (the account's web device, see +/// [`local_device_id`]) to the sync log inside the caller's transaction. +/// +/// The stamp (`seq`, `hlc_ms`, `op_id = :`) is reserved here; +/// `make` receives the `op_id`. The caller applies the returned operation +/// itself in the same transaction, so the log and the materialized state +/// change together (the podcast store does this for its local actions). +#[cfg_attr(not(test), allow(dead_code))] // podcast UI and RSS job: later commits +pub(crate) async fn append_local_op( + conn: &mut sqlx::PgConnection, + user_id: i64, + device_id: &str, + make: impl FnOnce(&str) -> SyncOpPayload, +) -> Result { + let (seq, hlc_ms) = reserve_local_stamp(&mut *conn, user_id).await?; + let op_id = format!("{device_id}:{seq}"); + let payload = make(&op_id); + let op = SyncOpWire { + op_id, + origin_device_id: device_id.to_string(), + seq, + hlc_ms, + payload, + }; + anyhow::ensure!( + insert_op(&mut *conn, user_id, &op).await?, + "local operation {} already exists", + op.op_id + ); + advance_vector(&mut *conn, user_id, device_id, seq).await?; + Ok(op) +} + async fn apply_ops( pool: &sqlx::PgPool, hub: Arc, @@ -1837,23 +1897,17 @@ async fn apply_ops( } else if insert_op(pool, user_id, &op).await? { let _ = apply_op(pool, Arc::clone(&hub), user_id, &op).await?; } - sqlx::query( - "INSERT INTO furumusic__fed_sync_vector (user_id, device_id, max_seq) - VALUES ($1, $2, $3) - ON CONFLICT (user_id, device_id) DO UPDATE SET - max_seq = GREATEST(furumusic__fed_sync_vector.max_seq, EXCLUDED.max_seq)", - ) - .bind(user_id) - .bind(&op.origin_device_id) - .bind(op.seq) - .execute(pool) - .await?; + advance_vector(pool, user_id, &op.origin_device_id, op.seq).await?; } Ok(()) } /// Appends an operation to the log; `false` when it was already stored. -async fn insert_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire) -> Result { +async fn insert_op<'e>( + db: impl sqlx::PgExecutor<'e>, + user_id: i64, + op: &SyncOpWire, +) -> Result { let inserted = sqlx::query( "INSERT INTO furumusic__fed_sync_ops (user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms, @@ -1870,7 +1924,7 @@ async fn insert_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire) -> Result .bind(op.hlc_ms) .bind(now_ms()) .bind(op.payload.is_tombstone()) - .execute(pool) + .execute(db) .await? .rows_affected(); Ok(inserted > 0) diff --git a/src/main.rs b/src/main.rs index d3a02c4..0c41cc4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod metrics; mod music; mod oidc; mod player; +mod podcasts; mod scheduler; mod similarity; mod torrents; diff --git a/src/music/mod.rs b/src/music/mod.rs index 8295f96..e8323e7 100644 --- a/src/music/mod.rs +++ b/src/music/mod.rs @@ -2942,6 +2942,76 @@ pub mod db_migrations { .build()]; } + // -- M0056: PostgreSQL podcast store (device-sync protocol v4) --------- + + // Per-account podcast catalog, synced registers and local fetch state; + // see `crate::podcasts` and `crate::podcasts::schema`. + #[cot::db::migrations::migration_op] + async fn create_podcast_store(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> { + for sql in crate::podcasts::schema::CREATE_PODCAST_STORE { + ctx.db.raw(sql).await?; + } + Ok(()) + } + + #[cot::db::migrations::migration_op] + async fn drop_podcast_store(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> { + for sql in crate::podcasts::schema::DROP_PODCAST_STORE { + ctx.db.raw(sql).await?; + } + Ok(()) + } + + #[derive(Debug, Copy, Clone)] + pub struct M0056CreatePodcastStore; + impl migrations::Migration for M0056CreatePodcastStore { + const APP_NAME: &'static str = "furumusic"; + const MIGRATION_NAME: &'static str = "m_0056_create_podcast_store"; + const DEPENDENCIES: &'static [migrations::MigrationDependency] = + &[migrations::MigrationDependency::migration( + "furumusic", + "m_0055_create_fed_podcast_relay", + )]; + const OPERATIONS: &'static [Operation] = &[Operation::custom(create_podcast_store) + .backwards(drop_podcast_store) + .build()]; + } + + // -- M0057: import the 0.13.0 podcast relay state into the store -------- + + // The relay table stays (unused) until a later cleanup release, so a + // rollback to 0.13.0 still finds the state it relayed. + #[cot::db::migrations::migration_op] + async fn import_podcast_relay(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> { + for sql in crate::podcasts::schema::import_statements("TRUE") { + ctx.db.raw(&sql).await?; + } + Ok(()) + } + + #[cot::db::migrations::migration_op] + async fn keep_podcast_store_rows( + _ctx: migrations::MigrationContext<'_>, + ) -> cot::db::Result<()> { + // The imported rows go with the tables (`m_0056` backwards). + Ok(()) + } + + #[derive(Debug, Copy, Clone)] + pub struct M0057ImportPodcastRelay; + impl migrations::Migration for M0057ImportPodcastRelay { + const APP_NAME: &'static str = "furumusic"; + const MIGRATION_NAME: &'static str = "m_0057_import_podcast_relay"; + const DEPENDENCIES: &'static [migrations::MigrationDependency] = + &[migrations::MigrationDependency::migration( + "furumusic", + "m_0056_create_podcast_store", + )]; + const OPERATIONS: &'static [Operation] = &[Operation::custom(import_podcast_relay) + .backwards(keep_podcast_store_rows) + .build()]; + } + pub const MIGRATIONS: &[&SyncDynMigration] = &[ &M0006CreateMediaFile, &M0007CreateArtist, @@ -2987,5 +3057,7 @@ pub mod db_migrations { &M0052AddImportPlaylistDestinations, &M0054AddTypedPlaybackQueue, &M0055CreateFedPodcastRelay, + &M0056CreatePodcastStore, + &M0057ImportPodcastRelay, ]; } diff --git a/src/podcasts/actions.rs b/src/podcasts/actions.rs new file mode 100644 index 0000000..ae16c9b --- /dev/null +++ b/src/podcasts/actions.rs @@ -0,0 +1,594 @@ +//! Local user actions: they build the device-sync operation, append it to +//! the account's sync log with its stamp and apply it through the shared +//! code path, all in one transaction. + +use anyhow::Result; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire}; +use music_dht::podcast::{ + EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord, + ProgressStatus, effective_share, merge_feed_aliases, merge_listen_session, + next_podcast_generation, resolve_podcast_id, +}; +use sqlx::{PgConnection, Row}; + +use super::apply::{self, ApplyOutcome}; +use super::{PodcastStore, PodcastStoreError, invalid}; +use crate::federation::devices; + +/// Result of [`PodcastStore::subscribe`]. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SubscribeOutcome { + /// Podcast the URL resolved to (an existing one when the URL is a known + /// feed URL or alias). + pub(crate) podcast_id: PodcastId, + /// The podcast was already subscribed; no operation was emitted. + pub(crate) already_subscribed: bool, + /// Operations appended to the sync log (and applied). + pub(crate) ops: Vec, +} + +/// A listening session of this account as seen by the progress register. +/// +/// Returned by explicit actions; periodic checkpoints of the session use its +/// `generation` ([`PodcastStore::checkpoint`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProgressSession { + /// Episode being played. + pub(crate) episode: PodcastEpisodeRef, + /// Session id (also the `ListenSession::session_id`). + pub(crate) session_id: String, + /// Generation of the session's latest explicit action. + pub(crate) generation: u64, +} + +/// A progress register written by a local action. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ProgressWrite { + /// The register as written. + pub(crate) record: ProgressRecord, + /// The session after the action (its generation for later checkpoints). + pub(crate) session: ProgressSession, + /// Operation appended to the sync log. + pub(crate) op: SyncOpWire, +} + +/// Result of [`PodcastStore::checkpoint`]. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum CheckpointOutcome { + /// The checkpoint was written. + Written(Box), + /// The register holds a greater generation (another device or an + /// explicit action took over). Nothing was written: stop writing at the + /// old generation, or continue locally with an explicit action + /// ([`PodcastStore::start_session`] / [`PodcastStore::seek`]). + Superseded { + /// Current register. + current: ProgressRecord, + }, + /// This session already completed the episode at its generation; a + /// checkpoint would turn it back into "in progress". Nothing was written. + AlreadyCompleted { + /// Current register. + current: ProgressRecord, + }, +} + +#[derive(Debug, Clone, Copy)] +enum ExplicitAction { + Start(i64), + Seek(i64), + Restart, + Complete(i64), + MarkPlayed, + MarkUnplayed, +} + +/// Validates the payload (with a provisional stamp), appends it to the sync +/// log with the next stamp of `device_id` and applies exactly that operation +/// through [`apply::apply_op`]. `make` receives the operation id, which +/// explicit actions without a session use as their session id. +pub(crate) async fn emit( + conn: &mut PgConnection, + user_id: i64, + device_id: &str, + make: impl Fn(&str) -> SyncOpPayload, +) -> Result { + SyncOpWire { + op_id: "provisional".to_string(), + origin_device_id: device_id.to_string(), + seq: 0, + hlc_ms: 0, + payload: make("provisional"), + } + .validate_podcast() + .map_err(invalid)?; + let op = devices::append_local_op(conn, user_id, device_id, &make).await?; + match apply::apply_op(conn, user_id, &op).await? { + ApplyOutcome::Rejected(reason) => Err(invalid(reason)), + _ => Ok(op), + } +} + +/// Resolves a feed URL against every known URL of every podcast of the +/// account: the alias sets, the catalog fetch URLs and the registers' +/// `podcast.feed_url` (`resolve_podcast_id`). +pub(crate) async fn resolve_podcast( + conn: &mut PgConnection, + user_id: i64, + url: &FeedUrl, +) -> Result { + let rows = sqlx::query( + "SELECT podcast_id FROM furumusic__podcast_feed_alias + WHERE user_id = $1 AND feed_url = $2 + UNION SELECT podcast_id FROM furumusic__podcast + WHERE user_id = $1 AND feed_url = $2 + UNION SELECT podcast_id FROM furumusic__podcast_subscription + WHERE user_id = $1 AND podcast_json->>'feed_url' = $2", + ) + .bind(user_id) + .bind(url.as_str()) + .fetch_all(&mut *conn) + .await?; + let ids = rows + .iter() + .map(|row| super::podcast_id_col(row, "podcast_id")) + .collect::>>()?; + Ok(resolve_podcast_id(url, ids.iter().map(|id| (id, url)))) +} + +async fn catalog_meta( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let row = sqlx::query( + "SELECT feed_url, title, author, description, link, image_url + FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + let Some(row) = row else { + return Ok(None); + }; + Ok(Some(PodcastMeta { + feed_url: super::feed_url_col(&row, "feed_url")?, + title: row.try_get("title")?, + author: row.try_get("author")?, + description: row.try_get("description")?, + link: row.try_get("link")?, + image_url: row.try_get("image_url")?, + })) +} + +async fn share_payload( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, + shared: bool, +) -> Result { + let subscription = apply::load_subscription_register(conn, user_id, podcast_id).await?; + let share = apply::load_share(conn, user_id, podcast_id).await?; + Ok(SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id.clone(), + shared, + generation: next_podcast_generation(subscription.as_ref(), share.as_ref()), + }) +} + +/// The reference carried by operations: identity and display only. +fn op_ref(episode: &PodcastEpisodeRef) -> PodcastEpisodeRef { + let mut episode = episode.clone(); + episode.progress_generation = None; + episode +} + +impl PodcastStore { + /// Subscribes to the podcast of `podcast.feed_url` (explicit user action). + /// + /// The URL is resolved against known feed URLs and aliases first, so + /// subscribing again, through a redirect target or from someone else's + /// card reuses the existing podcast with its progress and history. When + /// the podcast is already subscribed nothing is emitted. Otherwise one + /// `podcast_subscription_set` with generation + /// `next_podcast_generation(..)` and the known aliases is appended and + /// applied. The new subscription is private until + /// [`PodcastStore::set_shared`]. + /// + /// Pass the channel metadata of the fetched feed + /// (`ParsedFeed::to_podcast_meta(url)`), then ingest the episodes with + /// [`PodcastStore::apply_feed`]. + pub(crate) async fn subscribe( + &self, + user_id: i64, + podcast: &PodcastMeta, + ) -> Result { + let meta = podcast.bounded(); + meta.validate().map_err(invalid)?; + let device_id = devices::local_device_id(&self.pool, user_id).await?; + let mut tx = self.begin_write(user_id).await?; + let podcast_id = resolve_podcast(&mut tx, user_id, &meta.feed_url).await?; + let current = apply::load_subscription_register(&mut tx, user_id, &podcast_id).await?; + if current.as_ref().is_some_and(|record| record.subscribed) { + return Ok(SubscribeOutcome { + podcast_id, + already_subscribed: true, + ops: Vec::new(), + }); + } + let share = apply::load_share(&mut tx, user_id, &podcast_id).await?; + let generation = next_podcast_generation(current.as_ref(), share.as_ref()); + let aliases = merge_feed_aliases( + &podcast_id, + &apply::load_aliases(&mut tx, user_id, &podcast_id).await?, + std::slice::from_ref(&meta.feed_url), + ); + let op = emit(&mut tx, user_id, &device_id, |_| { + SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id.clone(), + subscribed: true, + podcast: meta.clone(), + feed_aliases: aliases.clone(), + generation, + } + }) + .await?; + tx.commit().await?; + Ok(SubscribeOutcome { + podcast_id, + already_subscribed: false, + ops: vec![op], + }) + } + + /// Unsubscribes (explicit user action). + /// + /// Emits `podcast_subscription_set { subscribed: false }` followed by + /// `podcast_share_set { shared: false }`, each with the next podcast + /// generation. Progress, sessions and the catalog are kept, so + /// subscribing again continues where the user stopped. Does nothing when + /// the podcast is not subscribed. + pub(crate) async fn unsubscribe( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result> { + let device_id = devices::local_device_id(&self.pool, user_id).await?; + let mut tx = self.begin_write(user_id).await?; + let Some(current) = apply::load_subscription_register(&mut tx, user_id, podcast_id).await? + else { + return Ok(Vec::new()); + }; + if !current.subscribed { + return Ok(Vec::new()); + } + let share = apply::load_share(&mut tx, user_id, podcast_id).await?; + let generation = next_podcast_generation(Some(¤t), share.as_ref()); + let mut meta = catalog_meta(&mut tx, user_id, podcast_id) + .await? + .map(|meta| meta.bounded()) + .unwrap_or_else(|| current.podcast.clone()); + meta.feed_url = current.podcast.feed_url.clone(); + let aliases = current.feed_aliases.clone(); + let unsubscribe = emit(&mut tx, user_id, &device_id, |_| { + SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id.clone(), + subscribed: false, + podcast: meta.clone(), + feed_aliases: aliases.clone(), + generation, + } + }) + .await?; + let revoke_payload = share_payload(&mut tx, user_id, podcast_id, false).await?; + let revoke = emit(&mut tx, user_id, &device_id, |_| revoke_payload.clone()).await?; + tx.commit().await?; + Ok(vec![unsubscribe, revoke]) + } + + /// Sets the decision to publish the podcast to the federation. + /// + /// Sharing needs an active subscription and is a no-op when the podcast + /// is already effectively shared (`effective_share`). Revoking is a no-op + /// when no share decision says `true`. The write uses the next podcast + /// generation, so it orders after the subscription it observed. + pub(crate) async fn set_shared( + &self, + user_id: i64, + podcast_id: &PodcastId, + shared: bool, + ) -> Result> { + let device_id = devices::local_device_id(&self.pool, user_id).await?; + let mut tx = self.begin_write(user_id).await?; + let subscription = apply::load_subscription_register(&mut tx, user_id, podcast_id).await?; + let share = apply::load_share(&mut tx, user_id, podcast_id).await?; + if shared { + if !subscription + .as_ref() + .is_some_and(|record| record.subscribed) + { + return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into()); + } + if effective_share(subscription.as_ref(), share.as_ref()) { + return Ok(Vec::new()); + } + } else if !share.as_ref().is_some_and(|record| record.shared) { + return Ok(Vec::new()); + } + let payload = share_payload(&mut tx, user_id, podcast_id, shared).await?; + let op = emit(&mut tx, user_id, &device_id, |_| payload.clone()).await?; + tx.commit().await?; + Ok(vec![op]) + } + + async fn explicit_progress( + &self, + user_id: i64, + episode: &PodcastEpisodeRef, + session_id: Option<&str>, + carried_generation: Option, + action: ExplicitAction, + ) -> Result { + episode.validate().map_err(invalid)?; + let device_id = devices::local_device_id(&self.pool, user_id).await?; + let mut tx = self.begin_write(user_id).await?; + let current = apply::load_progress(&mut tx, user_id, &episode.episode_id).await?; + let max_seen = current + .as_ref() + .map(|record| record.generation) + .into_iter() + .chain(carried_generation) + .max(); + let generation = ProgressRecord::explicit_generation(max_seen); + let current_position = current.as_ref().map_or(0, |record| record.position_ms); + let (position_ms, status) = match action { + ExplicitAction::Start(position) | ExplicitAction::Seek(position) => { + (position, ProgressStatus::InProgress) + } + ExplicitAction::Restart => (0, ProgressStatus::InProgress), + ExplicitAction::Complete(position) => (position, ProgressStatus::Completed), + ExplicitAction::MarkPlayed => (current_position, ProgressStatus::Completed), + ExplicitAction::MarkUnplayed => (0, ProgressStatus::NotStarted), + }; + let reference = op_ref(episode); + let op = emit(&mut tx, user_id, &device_id, |op_id| { + SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms, + status, + generation, + session_id: session_id.unwrap_or(op_id).to_string(), + episode: Some(reference.clone()), + } + }) + .await?; + tx.commit().await?; + let record = op.podcast_progress().expect("progress payload"); + Ok(ProgressWrite { + session: ProgressSession { + episode: episode.clone(), + session_id: record.session_id.clone(), + generation, + }, + record, + op, + }) + } + + /// Starts a listening session at `position_ms` (explicit action). + /// + /// The generation is `explicit_generation(max(local, carried_generation))`; + /// pass `PodcastEpisodeRef::progress_generation` of a queue item received + /// through a handoff as `carried_generation`. Ordinary Play resumes from + /// [`PodcastStore::resume_position_ms`]. + pub(crate) async fn start_session( + &self, + user_id: i64, + episode: &PodcastEpisodeRef, + session_id: &str, + position_ms: i64, + carried_generation: Option, + ) -> Result { + self.explicit_progress( + user_id, + episode, + Some(session_id), + carried_generation, + ExplicitAction::Start(position_ms), + ) + .await + } + + /// Periodic checkpoint (or pause/stop save) of a running session. + /// + /// Keeps the session generation. When the register already holds a + /// greater generation nothing is written ([`CheckpointOutcome::Superseded`]). + pub(crate) async fn checkpoint( + &self, + user_id: i64, + session: &ProgressSession, + position_ms: i64, + ) -> Result { + let episode = &session.episode; + let device_id = devices::local_device_id(&self.pool, user_id).await?; + let mut tx = self.begin_write(user_id).await?; + if let Some(current) = apply::load_progress(&mut tx, user_id, &episode.episode_id).await? { + if current.generation > session.generation { + return Ok(CheckpointOutcome::Superseded { current }); + } + if current.generation == session.generation + && current.session_id == session.session_id + && current.status == ProgressStatus::Completed + { + return Ok(CheckpointOutcome::AlreadyCompleted { current }); + } + } + let reference = op_ref(episode); + let op = emit(&mut tx, user_id, &device_id, |_| { + SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms, + status: ProgressStatus::InProgress, + generation: session.generation, + session_id: session.session_id.clone(), + episode: Some(reference.clone()), + } + }) + .await?; + tx.commit().await?; + let record = op.podcast_progress().expect("progress payload"); + Ok(CheckpointOutcome::Written(Box::new(ProgressWrite { + record, + session: session.clone(), + op, + }))) + } + + /// User seek during a session (explicit action; the returned session + /// carries the new generation for later checkpoints). + pub(crate) async fn seek( + &self, + user_id: i64, + session: &ProgressSession, + position_ms: i64, + ) -> Result { + self.explicit_progress( + user_id, + &session.episode, + Some(&session.session_id), + Some(session.generation), + ExplicitAction::Seek(position_ms), + ) + .await + } + + /// The session reached the end (`reaches_completion` or end of stream): + /// writes `completed` with an explicit generation. Record the session + /// itself with [`PodcastStore::record_listen_session`]. + pub(crate) async fn complete_session( + &self, + user_id: i64, + session: &ProgressSession, + position_ms: i64, + ) -> Result { + self.explicit_progress( + user_id, + &session.episode, + Some(&session.session_id), + Some(session.generation), + ExplicitAction::Complete(position_ms), + ) + .await + } + + /// "Start over": position 0, in progress, explicit generation. + pub(crate) async fn restart( + &self, + user_id: i64, + episode: &PodcastEpisodeRef, + session_id: &str, + ) -> Result { + self.explicit_progress( + user_id, + episode, + Some(session_id), + None, + ExplicitAction::Restart, + ) + .await + } + + /// "Mark played": status completed, explicit generation. Creates no + /// listening session and invents no listening time. + pub(crate) async fn mark_played( + &self, + user_id: i64, + episode: &PodcastEpisodeRef, + ) -> Result { + self.explicit_progress(user_id, episode, None, None, ExplicitAction::MarkPlayed) + .await + } + + /// "Mark unplayed": status not started at position 0, explicit generation. + pub(crate) async fn mark_unplayed( + &self, + user_id: i64, + episode: &PodcastEpisodeRef, + ) -> Result { + self.explicit_progress(user_id, episode, None, None, ExplicitAction::MarkUnplayed) + .await + } + + /// Records (or grows) a listening session of this account's device. + /// + /// Sessions below the history threshold (`ListenSession::should_record`) + /// and versions that do not grow the stored session emit nothing. + /// `session.device_id` must be the account's web device id. The episode + /// reference defaults to the catalog's. + pub(crate) async fn record_listen_session( + &self, + user_id: i64, + session: &ListenSession, + episode: Option<&PodcastEpisodeRef>, + ) -> Result> { + let device_id = devices::local_device_id(&self.pool, user_id).await?; + if session.device_id != device_id { + return Err(invalid( + "session device_id differs from the local device id", + )); + } + session.validate().map_err(invalid)?; + if !session.should_record() { + return Ok(None); + } + let mut tx = self.begin_write(user_id).await?; + if let Some(current) = apply::load_session(&mut tx, user_id, &session.session_id).await? { + let merged = merge_listen_session(¤t, session).map_err(invalid)?; + if merged == current { + return Ok(None); + } + } + let reference = match episode { + Some(episode) => Some(op_ref(episode)), + None => super::queries::episode_ref_in(&mut tx, user_id, &session.episode_id) + .await? + .map(|reference| op_ref(&reference)), + }; + let op = emit(&mut tx, user_id, &device_id, |_| { + SyncOpPayload::PodcastListenRecorded { + session: session.clone(), + episode: reference.clone(), + } + }) + .await?; + tx.commit().await?; + Ok(Some(op)) + } + + /// Resolves a feed URL to a podcast id: the existing podcast when the + /// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise + /// the id a first subscription through this URL would derive. + pub(crate) async fn resolve_podcast_id( + &self, + user_id: i64, + url: &FeedUrl, + ) -> Result { + let mut conn = self.pool.acquire().await?; + resolve_podcast(&mut conn, user_id, url).await + } + + /// The episode reference for queueing or progress actions (with + /// `progress_generation` set to the stored register's generation), or + /// `None` when the episode is unknown. + pub(crate) async fn episode_ref( + &self, + user_id: i64, + episode_id: &EpisodeId, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + super::queries::episode_ref_in(&mut conn, user_id, episode_id).await + } +} diff --git a/src/podcasts/apply.rs b/src/podcasts/apply.rs new file mode 100644 index 0000000..ad42958 --- /dev/null +++ b/src/podcasts/apply.rs @@ -0,0 +1,868 @@ +//! The single code path that applies podcast records to the store. +//! +//! Remote operations, local actions and snapshot sections all end here. +//! Records come from the `SyncOpWire` materializers of +//! `music_dht::device_sync` and are merged with the functions of +//! `music_dht::podcast`, so applying is idempotent and order-independent. + +use anyhow::Result; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot}; +use music_dht::podcast::{ + EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode, + ListenSession, PodcastEpisodeRef, PodcastError, PodcastId, ProgressRecord, ShareRecord, + SubscriptionRecord, merge_feed_aliases, merge_listen_session, +}; +use sqlx::{PgConnection, Row}; + +use super::{ + PodcastStore, PodcastStoreError, SESSION_COLUMNS, episode_id_col, episode_search_norm, + feed_url_col, generation_from_sql, generation_to_sql, invalid, now_ms, podcast_meta_col, + progress_columns, progress_from_row, session_from_row, status_to_sql, +}; + +/// Result of applying one operation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ApplyOutcome { + /// Stored state changed. + Changed, + /// The operation was already reflected (duplicate, or older than the + /// stored register). + Unchanged, + /// Not a podcast operation; nothing to do here. + Ignored, + /// The operation violates the contract and was skipped. Acknowledge it + /// anyway so the log keeps moving (`PODCASTS.md`). + Rejected(String), +} + +/// Counters returned by [`PodcastStore::apply_snapshot`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct SnapshotApplyStats { + /// Records that changed stored state. + pub(crate) changed: usize, + /// Valid records already reflected locally. + pub(crate) unchanged: usize, + /// Records that failed validation and were skipped. + pub(crate) rejected: usize, + /// Whether the podcast sections were ignored for exceeding the counts. + pub(crate) ignored_over_limits: bool, +} + +/// Where an episode row came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EpisodeSource { + /// Parsed from the podcast's feed by this account. + Feed, + /// Learned from a `PodcastEpisodeRef` (sync operation or queue item). + Sync, +} + +impl EpisodeSource { + fn as_sql(self) -> &'static str { + match self { + Self::Feed => "feed", + Self::Sync => "sync", + } + } +} + +pub(crate) async fn load_subscription_register( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let row = sqlx::query( + "SELECT subscribed, podcast_json, generation, hlc_ms, op_id + FROM furumusic__podcast_subscription WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + let Some(row) = row else { + return Ok(None); + }; + let record = SubscriptionRecord { + podcast_id: podcast_id.clone(), + subscribed: row.try_get("subscribed")?, + podcast: podcast_meta_col(&row, "podcast_json")?, + feed_aliases: load_aliases(conn, user_id, podcast_id).await?, + generation: generation_from_sql(row.try_get("generation")?), + hlc_ms: row.try_get("hlc_ms")?, + op_id: row.try_get("op_id")?, + }; + Ok(Some(record)) +} + +pub(crate) async fn load_aliases( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let rows = sqlx::query( + "SELECT feed_url FROM furumusic__podcast_feed_alias + WHERE user_id = $1 AND podcast_id = $2 ORDER BY feed_url", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_all(&mut *conn) + .await?; + Ok(rows + .iter() + .map(|row| feed_url_col(row, "feed_url")) + .collect::>>()?) +} + +pub(crate) fn share_from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { + Ok(ShareRecord { + podcast_id: super::podcast_id_col(row, "podcast_id")?, + shared: row.try_get("shared")?, + generation: generation_from_sql(row.try_get("generation")?), + hlc_ms: row.try_get("hlc_ms")?, + op_id: row.try_get("op_id")?, + }) +} + +pub(crate) async fn load_share( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let row = sqlx::query( + "SELECT podcast_id, shared, generation, hlc_ms, op_id + FROM furumusic__podcast_share WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + Ok(row.as_ref().map(share_from_row).transpose()?) +} + +pub(crate) async fn load_progress( + conn: &mut PgConnection, + user_id: i64, + episode_id: &EpisodeId, +) -> Result> { + let row = sqlx::query(&format!( + "SELECT {} FROM furumusic__podcast_progress p + WHERE p.user_id = $1 AND p.episode_id = $2", + progress_columns("p", "") + )) + .bind(user_id) + .bind(episode_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + Ok(row + .as_ref() + .map(|row| progress_from_row(row, "")) + .transpose()?) +} + +pub(crate) async fn load_session( + conn: &mut PgConnection, + user_id: i64, + session_id: &str, +) -> Result> { + let row = sqlx::query(&format!( + "SELECT {SESSION_COLUMNS} FROM furumusic__podcast_listen_session s + WHERE s.user_id = $1 AND s.session_id = $2" + )) + .bind(user_id) + .bind(session_id) + .fetch_optional(&mut *conn) + .await?; + Ok(row.as_ref().map(session_from_row).transpose()?) +} + +/// Replaces the alias set of a podcast with `merged` when it differs. +async fn store_aliases( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, + merged: &[FeedUrl], +) -> Result { + let current = load_aliases(conn, user_id, podcast_id).await?; + if current == merged { + return Ok(false); + } + sqlx::query("DELETE FROM furumusic__podcast_feed_alias WHERE user_id = $1 AND podcast_id = $2") + .bind(user_id) + .bind(podcast_id.as_str()) + .execute(&mut *conn) + .await?; + let urls: Vec<&str> = merged.iter().map(FeedUrl::as_str).collect(); + sqlx::query( + "INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url) + SELECT $1, $2, url FROM UNNEST($3::text[]) AS url", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .bind(&urls) + .execute(&mut *conn) + .await?; + Ok(true) +} + +/// Merges feed URLs into the bounded grow-only alias set of a podcast. +/// +/// Returns whether the stored set changed. +pub(crate) async fn merge_aliases( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, + urls: &[FeedUrl], +) -> Result { + let current = load_aliases(conn, user_id, podcast_id).await?; + let merged = merge_feed_aliases(podcast_id, ¤t, urls); + store_aliases(conn, user_id, podcast_id, &merged).await +} + +/// Inserts the catalog row of a podcast if it does not exist yet. +pub(crate) async fn ensure_podcast_row( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, + feed_url: &FeedUrl, + title: &str, + now: i64, +) -> Result<()> { + sqlx::query( + "INSERT INTO furumusic__podcast + (user_id, podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms) + VALUES ($1, $2, $3, $4, FALSE, $5, $5) + ON CONFLICT (user_id, podcast_id) DO NOTHING", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .bind(feed_url.as_str()) + .bind(title) + .bind(now) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Applies a subscription register record (operation or snapshot). +/// +/// The register is last-writer-wins ([`SubscriptionRecord::supersedes`]); +/// the alias set is a bounded grow-only set merged independently, so the +/// stored aliases are the bounded union of every URL seen (what +/// `merge_subscription` computes). The catalog row is created from the +/// register metadata and follows the winning register until this account +/// has refreshed the feed itself. +pub(crate) async fn apply_subscription_record( + conn: &mut PgConnection, + user_id: i64, + record: &SubscriptionRecord, + now: i64, +) -> Result { + let generation = generation_to_sql(record.generation)?; + let podcast_id = &record.podcast_id; + let current = load_subscription_register(conn, user_id, podcast_id).await?; + let wins = current + .as_ref() + .is_none_or(|current| record.supersedes(current)); + let mut changed = false; + if wins { + sqlx::query( + "INSERT INTO furumusic__podcast_subscription + (user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (user_id, podcast_id) DO UPDATE SET + subscribed = EXCLUDED.subscribed, podcast_json = EXCLUDED.podcast_json, + generation = EXCLUDED.generation, hlc_ms = EXCLUDED.hlc_ms, + op_id = EXCLUDED.op_id", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .bind(record.subscribed) + .bind(serde_json::to_value(&record.podcast)?) + .bind(generation) + .bind(record.hlc_ms) + .bind(&record.op_id) + .execute(&mut *conn) + .await?; + changed = true; + } + let mut urls = record.feed_aliases.clone(); + urls.push(record.podcast.feed_url.clone()); + changed |= merge_aliases(conn, user_id, podcast_id, &urls).await?; + + ensure_podcast_row( + conn, + user_id, + podcast_id, + &record.podcast.feed_url, + &record.podcast.title, + now, + ) + .await?; + if wins { + let meta = &record.podcast; + sqlx::query( + "UPDATE furumusic__podcast + SET feed_url = $3, title = $4, author = $5, + description = COALESCE($6, description), link = $7, image_url = $8, + updated_at_ms = $9 + WHERE user_id = $1 AND podcast_id = $2 AND NOT meta_from_feed", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .bind(meta.feed_url.as_str()) + .bind(&meta.title) + .bind(&meta.author) + .bind(&meta.description) + .bind(&meta.link) + .bind(&meta.image_url) + .bind(now) + .execute(&mut *conn) + .await?; + } + Ok(changed) +} + +/// Applies a share register record. +pub(crate) async fn apply_share_record( + conn: &mut PgConnection, + user_id: i64, + record: &ShareRecord, +) -> Result { + let generation = generation_to_sql(record.generation)?; + let current = load_share(conn, user_id, &record.podcast_id).await?; + if current + .as_ref() + .is_some_and(|current| !record.supersedes(current)) + { + return Ok(false); + } + sqlx::query( + "INSERT INTO furumusic__podcast_share + (user_id, podcast_id, shared, generation, hlc_ms, op_id) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (user_id, podcast_id) DO UPDATE SET + shared = EXCLUDED.shared, generation = EXCLUDED.generation, + hlc_ms = EXCLUDED.hlc_ms, op_id = EXCLUDED.op_id", + ) + .bind(user_id) + .bind(record.podcast_id.as_str()) + .bind(record.shared) + .bind(generation) + .bind(record.hlc_ms) + .bind(&record.op_id) + .execute(&mut *conn) + .await?; + Ok(true) +} + +/// Applies a progress register record. Never touches the catalog. +pub(crate) async fn apply_progress_record( + conn: &mut PgConnection, + user_id: i64, + record: &ProgressRecord, +) -> Result { + let generation = generation_to_sql(record.generation)?; + let current = load_progress(conn, user_id, &record.episode_id).await?; + if current + .as_ref() + .is_some_and(|current| !record.supersedes(current)) + { + return Ok(false); + } + sqlx::query( + "INSERT INTO furumusic__podcast_progress + (user_id, episode_id, podcast_id, position_ms, status, generation, session_id, + hlc_ms, device_id, op_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (user_id, episode_id) DO UPDATE SET + podcast_id = EXCLUDED.podcast_id, position_ms = EXCLUDED.position_ms, + status = EXCLUDED.status, generation = EXCLUDED.generation, + session_id = EXCLUDED.session_id, hlc_ms = EXCLUDED.hlc_ms, + device_id = EXCLUDED.device_id, op_id = EXCLUDED.op_id", + ) + .bind(user_id) + .bind(record.episode_id.as_str()) + .bind(record.podcast_id.as_str()) + .bind(record.position_ms) + .bind(status_to_sql(record.status)) + .bind(generation) + .bind(&record.session_id) + .bind(record.hlc_ms) + .bind(&record.device_id) + .bind(&record.op_id) + .execute(&mut *conn) + .await?; + Ok(true) +} + +/// Adds or grows a listening session (grow-only set keyed by `session_id`). +pub(crate) async fn apply_listen_session( + conn: &mut PgConnection, + user_id: i64, + session: &ListenSession, +) -> Result { + let current = load_session(conn, user_id, &session.session_id).await?; + let merged = match ¤t { + Some(current) => merge_listen_session(current, session).map_err(invalid)?, + None => session.clone(), + }; + if current.as_ref() == Some(&merged) { + return Ok(false); + } + sqlx::query( + "INSERT INTO furumusic__podcast_listen_session + (user_id, session_id, episode_id, podcast_id, device_id, started_at_ms, + ended_at_ms, start_position_ms, end_position_ms, listened_ms, completed) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (user_id, session_id) DO UPDATE SET + started_at_ms = EXCLUDED.started_at_ms, ended_at_ms = EXCLUDED.ended_at_ms, + start_position_ms = EXCLUDED.start_position_ms, + end_position_ms = EXCLUDED.end_position_ms, + listened_ms = EXCLUDED.listened_ms, completed = EXCLUDED.completed", + ) + .bind(user_id) + .bind(&merged.session_id) + .bind(merged.episode_id.as_str()) + .bind(merged.podcast_id.as_str()) + .bind(&merged.device_id) + .bind(merged.started_at_ms) + .bind(merged.ended_at_ms) + .bind(merged.start_position_ms) + .bind(merged.end_position_ms) + .bind(merged.listened_ms) + .bind(merged.completed) + .execute(&mut *conn) + .await?; + Ok(true) +} + +pub(crate) fn episode_type_to_sql(kind: EpisodeType) -> &'static str { + match kind { + EpisodeType::Full => "full", + EpisodeType::Trailer => "trailer", + EpisodeType::Bonus => "bonus", + } +} + +/// Inserts an episode row. +pub(crate) async fn insert_episode( + conn: &mut PgConnection, + user_id: i64, + meta: &EpisodeMeta, + source: EpisodeSource, + in_feed: bool, + now: i64, +) -> Result<()> { + sqlx::query( + "INSERT INTO furumusic__podcast_episode + (user_id, episode_id, podcast_id, guid, title, description, season, + episode_number, episode_type, published_at_ms, duration_ms, audio_url, mime, + byte_len, image_url, link, source, in_feed, search_norm, first_seen_at_ms, + updated_at_ms) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, + $18, $19, $20, $20)", + ) + .bind(user_id) + .bind(meta.episode_id.as_str()) + .bind(meta.podcast_id.as_str()) + .bind(&meta.guid) + .bind(&meta.title) + .bind(&meta.description) + .bind(meta.season.map(i64::from)) + .bind(meta.episode_number.map(i64::from)) + .bind(episode_type_to_sql(meta.episode_type)) + .bind(meta.published_at_ms) + .bind(meta.duration_ms) + .bind(&meta.audio_url) + .bind(&meta.mime) + .bind(meta.byte_len.and_then(|len| i64::try_from(len).ok())) + .bind(&meta.image_url) + .bind(&meta.link) + .bind(source.as_sql()) + .bind(in_feed) + .bind(episode_search_norm( + &meta.title, + meta.description.as_deref(), + )) + .bind(now) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Episode metadata of a reference learned through sync. +pub(crate) fn meta_from_ref(episode: &PodcastEpisodeRef) -> EpisodeMeta { + EpisodeMeta { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + guid: None, + title: episode.episode_title.clone(), + description: None, + season: episode.season, + episode_number: episode.episode_number, + episode_type: EpisodeType::default(), + published_at_ms: episode.published_at_ms, + duration_ms: episode.duration_ms, + audio_url: episode.audio_url.clone(), + mime: episode.mime.clone(), + byte_len: None, + image_url: episode.image_url.clone(), + link: None, + } +} + +/// Records an episode learned through sync when the catalog does not know it. +/// +/// Feed metadata always wins: an existing row is left untouched. The +/// podcast catalog row is created from the reference when missing (no +/// register is written). +pub(crate) async fn ensure_episode_from_ref( + conn: &mut PgConnection, + user_id: i64, + episode: &PodcastEpisodeRef, +) -> Result<()> { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM furumusic__podcast_episode + WHERE user_id = $1 AND episode_id = $2)", + ) + .bind(user_id) + .bind(episode.episode_id.as_str()) + .fetch_one(&mut *conn) + .await?; + if exists { + return Ok(()); + } + let now = now_ms(); + ensure_podcast_row( + conn, + user_id, + &episode.podcast_id, + &episode.feed_url, + &episode.podcast_title, + now, + ) + .await?; + insert_episode( + conn, + user_id, + &meta_from_ref(episode), + EpisodeSource::Sync, + false, + now, + ) + .await +} + +/// Known episodes of a podcast with the identity hints they were last seen with. +pub(crate) async fn known_episodes( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let rows = sqlx::query( + "SELECT episode_id, guid, audio_url, link, published_at_ms, title + FROM furumusic__podcast_episode + WHERE user_id = $1 AND podcast_id = $2 ORDER BY episode_id", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_all(&mut *conn) + .await?; + let known = rows + .iter() + .map(|row| { + Ok(KnownEpisode { + episode_id: episode_id_col(row, "episode_id")?, + hints: EpisodeIdentityHints { + guid: row.try_get("guid")?, + enclosure_url: Some(row.try_get("audio_url")?), + link: row.try_get("link")?, + published_at_ms: row.try_get("published_at_ms")?, + title: Some(row.try_get("title")?), + }, + }) + }) + .collect::>>()?; + Ok(known) +} + +/// Applies one podcast operation inside the caller's transaction (which +/// holds the account lock). +pub(crate) async fn apply_op( + conn: &mut PgConnection, + user_id: i64, + op: &SyncOpWire, +) -> Result { + if !op.payload.is_podcast() { + return Ok(ApplyOutcome::Ignored); + } + if let Err(error) = op.validate_podcast() { + return Ok(ApplyOutcome::Rejected(error.to_string())); + } + let result = match &op.payload { + SyncOpPayload::PodcastSubscriptionSet { .. } => { + let record = op.podcast_subscription().expect("subscription payload"); + apply_subscription_record(conn, user_id, &record, now_ms()).await + } + SyncOpPayload::PodcastShareSet { .. } => { + apply_share_record(conn, user_id, &op.podcast_share().expect("share payload")).await + } + SyncOpPayload::PodcastFeedAliasesAdded { .. } => { + let (podcast_id, urls) = op.podcast_feed_aliases().expect("alias payload"); + merge_aliases(conn, user_id, podcast_id, urls).await + } + SyncOpPayload::PodcastProgressSet { episode, .. } => { + let record = op.podcast_progress().expect("progress payload"); + if let Err(error) = generation_to_sql(record.generation) { + return Ok(ApplyOutcome::Rejected(error.to_string())); + } + if let Some(episode) = episode { + ensure_episode_from_ref(conn, user_id, episode).await?; + } + apply_progress_record(conn, user_id, &record).await + } + SyncOpPayload::PodcastListenRecorded { session, episode } => { + if let Some(current) = load_session(conn, user_id, &session.session_id).await? + && let Err(error) = merge_listen_session(¤t, session) + { + return Ok(ApplyOutcome::Rejected(error.to_string())); + } + if let Some(episode) = episode { + ensure_episode_from_ref(conn, user_id, episode).await?; + } + apply_listen_session(conn, user_id, session).await + } + _ => return Ok(ApplyOutcome::Ignored), + }; + match result { + Ok(true) => Ok(ApplyOutcome::Changed), + Ok(false) => Ok(ApplyOutcome::Unchanged), + Err(error) => match error.downcast_ref::() { + Some(PodcastStoreError::Invalid(reason)) => Ok(ApplyOutcome::Rejected(reason.clone())), + _ => Err(error), + }, + } +} + +fn record_outcome(stats: &mut SnapshotApplyStats, result: Result) -> Result<()> { + match result { + Ok(true) => stats.changed += 1, + Ok(false) => stats.unchanged += 1, + Err(error) => match error.downcast_ref::() { + Some(PodcastStoreError::Invalid(_)) => stats.rejected += 1, + _ => return Err(error), + }, + } + Ok(()) +} + +fn check(result: Result<(), PodcastError>) -> Result<()> { + result.map_err(invalid) +} + +impl PodcastStore { + /// Applies a device-sync operation received from a trusted device. + /// + /// Non-podcast operations return [`ApplyOutcome::Ignored`]. Podcast + /// operations are validated (`SyncOpWire::validate_podcast`) and merged + /// with the contract functions in one transaction; applying the same + /// operations in any order, with duplicates, gives the same state. + pub(crate) async fn apply_remote_op( + &self, + user_id: i64, + op: &SyncOpWire, + ) -> Result { + if !op.payload.is_podcast() { + return Ok(ApplyOutcome::Ignored); + } + let mut tx = self.begin_write(user_id).await?; + let outcome = apply_op(&mut tx, user_id, op).await?; + tx.commit().await?; + Ok(outcome) + } + + /// Merges the podcast sections of a sync snapshot. + /// + /// Sections exceeding the contract counts are ignored + /// (`SyncSnapshot::podcast_state_within_limits`); every record is + /// validated and merged with the same functions as operations, so a + /// snapshot never replaces newer local state. One transaction. + pub(crate) async fn apply_snapshot( + &self, + user_id: i64, + snapshot: &SyncSnapshot, + ) -> Result { + let mut stats = SnapshotApplyStats::default(); + if !snapshot.has_podcast_state() { + return Ok(stats); + } + if !snapshot.podcast_state_within_limits() { + stats.ignored_over_limits = true; + return Ok(stats); + } + let mut tx = self.begin_write(user_id).await?; + let now = now_ms(); + for record in &snapshot.podcast_subscriptions { + let mut record = record.clone(); + record.normalize(); + let result = match check(record.validate()) { + Ok(()) => apply_subscription_record(&mut tx, user_id, &record, now).await, + Err(error) => Err(error), + }; + record_outcome(&mut stats, result)?; + } + for record in &snapshot.podcast_shares { + let result = match check(record.validate()) { + Ok(()) => apply_share_record(&mut tx, user_id, record).await, + Err(error) => Err(error), + }; + record_outcome(&mut stats, result)?; + } + for record in &snapshot.podcast_progress { + let result = match check(record.validate()) { + Ok(()) => apply_progress_record(&mut tx, user_id, record).await, + Err(error) => Err(error), + }; + record_outcome(&mut stats, result)?; + } + for session in &snapshot.podcast_listen_sessions { + let result = match check(session.validate()) { + Ok(()) => apply_listen_session(&mut tx, user_id, session).await, + Err(error) => Err(error), + }; + record_outcome(&mut stats, result)?; + } + tx.commit().await?; + Ok(stats) + } + + /// Fills the podcast sections of `snapshot` with this account's + /// registers and sessions, bounded by + /// `SyncSnapshot::bound_podcast_state_to(budget_bytes)`. + /// + /// Records are read newest first and capped by the per-section counts + /// before the byte budget is applied. Fetch state and the catalog are + /// never part of the snapshot. Send it only to v4 peers + /// (`SyncSnapshot::for_peer`). + pub(crate) async fn fill_snapshot( + &self, + user_id: i64, + snapshot: &mut SyncSnapshot, + budget_bytes: usize, + ) -> Result<()> { + use music_dht::podcast::{ + MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES, + MAX_SNAPSHOT_SUBSCRIPTIONS, + }; + // One consistent view of every section. + let mut tx = self.pool.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + .execute(&mut *tx) + .await?; + let rows = sqlx::query( + "SELECT s.podcast_id, s.subscribed, s.podcast_json, s.generation, s.hlc_ms, s.op_id, + COALESCE((SELECT array_agg(a.feed_url ORDER BY a.feed_url) + FROM furumusic__podcast_feed_alias a + WHERE a.user_id = s.user_id AND a.podcast_id = s.podcast_id), + '{}') AS aliases + FROM furumusic__podcast_subscription s + WHERE s.user_id = $1 + ORDER BY s.generation DESC, s.hlc_ms DESC, s.op_id DESC, s.podcast_id DESC + LIMIT $2", + ) + .bind(user_id) + .bind(MAX_SNAPSHOT_SUBSCRIPTIONS as i64) + .fetch_all(&mut *tx) + .await?; + let subscriptions = rows + .iter() + .map(|row| { + let aliases: Vec = row.try_get("aliases")?; + Ok(SubscriptionRecord { + podcast_id: super::podcast_id_col(row, "podcast_id")?, + subscribed: row.try_get("subscribed")?, + podcast: podcast_meta_col(row, "podcast_json")?, + feed_aliases: aliases + .into_iter() + .map(FeedUrl::try_from) + .collect::>() + .map_err(|error| sqlx::Error::Decode(Box::new(error)))?, + generation: generation_from_sql(row.try_get("generation")?), + hlc_ms: row.try_get("hlc_ms")?, + op_id: row.try_get("op_id")?, + }) + }) + .collect::>>()?; + let shares = sqlx::query( + "SELECT podcast_id, shared, generation, hlc_ms, op_id FROM furumusic__podcast_share + WHERE user_id = $1 + ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC + LIMIT $2", + ) + .bind(user_id) + .bind(MAX_SNAPSHOT_SHARES as i64) + .fetch_all(&mut *tx) + .await? + .iter() + .map(share_from_row) + .collect::>>()?; + let progress = sqlx::query(&format!( + "SELECT {} FROM furumusic__podcast_progress p + WHERE p.user_id = $1 + ORDER BY p.generation DESC, p.hlc_ms DESC, p.op_id DESC, p.episode_id DESC + LIMIT $2", + progress_columns("p", "") + )) + .bind(user_id) + .bind(MAX_SNAPSHOT_PROGRESS as i64) + .fetch_all(&mut *tx) + .await? + .iter() + .map(|row| progress_from_row(row, "")) + .collect::>>()?; + let sessions = sqlx::query(&format!( + "SELECT {SESSION_COLUMNS} FROM furumusic__podcast_listen_session s + WHERE s.user_id = $1 + ORDER BY s.ended_at_ms DESC, s.session_id DESC + LIMIT $2" + )) + .bind(user_id) + .bind(MAX_SNAPSHOT_LISTEN_SESSIONS as i64) + .fetch_all(&mut *tx) + .await? + .iter() + .map(session_from_row) + .collect::>>()?; + tx.commit().await?; + snapshot.podcast_subscriptions = subscriptions; + snapshot.podcast_shares = shares; + snapshot.podcast_progress = progress; + snapshot.podcast_listen_sessions = sessions; + snapshot.bound_podcast_state_to(budget_bytes); + Ok(()) + } + + /// A snapshot holding only this account's podcast sections, bounded + /// with the default budget (`MAX_PODCAST_SNAPSHOT_BYTES`). + pub(crate) async fn export_snapshot(&self, user_id: i64) -> Result { + let mut snapshot = SyncSnapshot::default(); + self.fill_snapshot( + user_id, + &mut snapshot, + music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES, + ) + .await?; + Ok(snapshot) + } + + /// Progress register of an episode (also for episodes the catalog does + /// not know yet). + pub(crate) async fn progress( + &self, + user_id: i64, + episode_id: &EpisodeId, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + load_progress(&mut conn, user_id, episode_id).await + } +} diff --git a/src/podcasts/mod.rs b/src/podcasts/mod.rs new file mode 100644 index 0000000..df00fa9 --- /dev/null +++ b/src/podcasts/mod.rs @@ -0,0 +1,273 @@ +//! PostgreSQL podcast store of the web server. +//! +//! [`PodcastStore`] implements the podcast contract of `music_dht::podcast` +//! (normative description: `PODCASTS.md` of frid) on PostgreSQL, with the +//! same invariants as the SQLite reference store of `furumi-library`. Each +//! web account is one virtual trusted device, so every table is keyed by +//! `user_id`. The store owns no HTTP, scheduling or UI: RSS fetching and the +//! player use it. +//! +//! # Tables +//! +//! | Table | Key | Content | Synced | +//! | --- | --- | --- | --- | +//! | `furumusic__podcast` | `podcast_id` | local channel metadata, `feed_url` = URL this account fetches | no (catalog) | +//! | `furumusic__podcast_feed_alias` | `(podcast_id, feed_url)` | bounded grow-only alias set | yes, inside subscription records and `podcast_feed_aliases_added` | +//! | `furumusic__podcast_episode` | `episode_id` | `EpisodeMeta`, `source` (`feed`/`sync`), `in_feed` | no (catalog) | +//! | `furumusic__podcast_season` (view) | `(podcast_id, season)` | derived from the episodes | no | +//! | `furumusic__podcast_subscription` | `podcast_id` | subscription register | yes | +//! | `furumusic__podcast_share` | `podcast_id` | share register | yes | +//! | `furumusic__podcast_progress` | `episode_id` | progress register | yes | +//! | `furumusic__podcast_listen_session` | `session_id` | grow-only set of sessions | yes | +//! | `furumusic__podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, check times, errors, cover cache | **never** | +//! +//! # One apply path +//! +//! Remote operations ([`PodcastStore::apply_remote_op`]), snapshot sections +//! ([`PodcastStore::apply_snapshot`]) and local actions all end in the same +//! functions: the `SyncOpWire` materializers of `music_dht::device_sync` and +//! the `merge_*` rules of `music_dht::podcast`. Applying is therefore +//! idempotent and independent of delivery order, and a snapshot never +//! replaces newer state. +//! +//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...) +//! run in one transaction: they read the registers, append the operation to +//! the account's device-sync log with its stamp +//! ([`crate::federation::devices::append_local_op`]: next `seq`, `hlc_ms`, +//! `op_id = :`) and apply exactly that operation. Either both +//! the log and the registers change or neither does. Every write +//! transaction takes a per-account advisory lock, so concurrent writers of +//! one account serialize. +//! +//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the +//! alias set only; they never write a register, progress or a session. + +// The RSS refresh job and the podcast UI/HTTP API that call the local +// actions, queries and fetch state land in the following commits of this +// series; until then only sync and the tests use them. Remove this +// allowance with them. +#![allow(dead_code, unused_imports)] + +mod actions; +mod apply; +mod queries; +mod refresh; +pub(crate) mod schema; + +#[cfg(test)] +pub(crate) mod postgres_tests; +#[cfg(test)] +mod tests; + +use std::fmt; + +use anyhow::Result; +use music_dht::podcast::{ + EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord, + ProgressStatus, +}; +use sqlx::Row; +use sqlx::postgres::PgRow; + +pub(crate) use actions::{CheckpointOutcome, ProgressSession, ProgressWrite, SubscribeOutcome}; +pub(crate) use apply::{ApplyOutcome, SnapshotApplyStats}; +pub(crate) use queries::{ + EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter, + PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary, +}; +pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome}; + +/// Typed errors of [`PodcastStore`]; other failures are database errors. +/// +/// They are returned inside `anyhow::Error`; use `downcast_ref` to match. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum PodcastStoreError { + /// The podcast is unknown to this account. + UnknownPodcast(PodcastId), + /// The action needs an active subscription. + NotSubscribed(PodcastId), + /// Input violates the podcast contract. + Invalid(String), +} + +impl fmt::Display for PodcastStoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"), + Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"), + Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"), + } + } +} + +impl std::error::Error for PodcastStoreError {} + +impl From for PodcastStoreError { + fn from(error: PodcastError) -> Self { + Self::Invalid(error.to_string()) + } +} + +pub(crate) fn invalid(reason: impl ToString) -> anyhow::Error { + PodcastStoreError::Invalid(reason.to_string()).into() +} + +/// PostgreSQL store of podcast catalog, personal state and fetch state. +/// +/// Cheap to clone (the pool is shared). See the module documentation. +#[derive(Debug, Clone)] +pub(crate) struct PodcastStore { + pool: sqlx::PgPool, +} + +impl PodcastStore { + pub(crate) fn new(pool: sqlx::PgPool) -> Self { + Self { pool } + } + + /// Starts a write transaction holding the account's podcast lock. + async fn begin_write( + &self, + user_id: i64, + ) -> Result> { + let mut tx = self.pool.begin().await?; + lock_account(&mut tx, user_id).await?; + Ok(tx) + } +} + +/// Serializes podcast writers of one account until the transaction ends. +pub(crate) async fn lock_account(conn: &mut sqlx::PgConnection, user_id: i64) -> Result<()> { + sqlx::query( + "SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcasts:' || $1::text, 0))", + ) + .bind(user_id) + .execute(conn) + .await?; + Ok(()) +} + +pub(crate) fn now_ms() -> i64 { + chrono::Utc::now().timestamp_millis() +} + +/// Converts a contract generation to the `BIGINT` column value. +/// +/// Generations above `i64::MAX` cannot be stored; records carrying one are +/// rejected (no conforming client produces them). +pub(crate) fn generation_to_sql(generation: u64) -> Result { + i64::try_from(generation) + .map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string())) +} + +pub(crate) fn generation_from_sql(value: i64) -> u64 { + u64::try_from(value).unwrap_or(0) +} + +pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str { + match status { + ProgressStatus::NotStarted => "not_started", + ProgressStatus::InProgress => "in_progress", + ProgressStatus::Completed => "completed", + } +} + +pub(crate) fn status_from_sql(value: &str) -> ProgressStatus { + match value { + "in_progress" => ProgressStatus::InProgress, + "completed" => ProgressStatus::Completed, + _ => ProgressStatus::NotStarted, + } +} + +/// `u32` contract numbers (season, episode number) from a `BIGINT` column. +pub(crate) fn u32_from_sql(value: Option) -> Option { + value.and_then(|value| u32::try_from(value).ok()) +} + +fn decode_error(message: impl ToString) -> sqlx::Error { + sqlx::Error::Decode(Box::new(PodcastStoreError::Invalid(message.to_string()))) +} + +/// Parses a stored id; stored ids are validated on write. +pub(crate) fn podcast_id_col(row: &PgRow, column: &str) -> sqlx::Result { + PodcastId::try_from(row.try_get::(column)?).map_err(decode_error) +} + +pub(crate) fn episode_id_col(row: &PgRow, column: &str) -> sqlx::Result { + EpisodeId::try_from(row.try_get::(column)?).map_err(decode_error) +} + +pub(crate) fn feed_url_col(row: &PgRow, column: &str) -> sqlx::Result { + FeedUrl::try_from(row.try_get::(column)?).map_err(decode_error) +} + +pub(crate) fn podcast_meta_col(row: &PgRow, column: &str) -> sqlx::Result { + serde_json::from_value(row.try_get::(column)?).map_err(decode_error) +} + +/// Progress register columns of table alias `alias`, named `{prefix}...`. +pub(crate) fn progress_columns(alias: &str, prefix: &str) -> String { + [ + "episode_id", + "podcast_id", + "position_ms", + "status", + "generation", + "session_id", + "hlc_ms", + "device_id", + "op_id", + ] + .iter() + .map(|column| format!("{alias}.{column} AS {prefix}{column}")) + .collect::>() + .join(", ") +} + +/// Progress register selected with [`progress_columns`]. +pub(crate) fn progress_from_row(row: &PgRow, prefix: &str) -> sqlx::Result { + let column = |name: &str| format!("{prefix}{name}"); + Ok(ProgressRecord { + episode_id: episode_id_col(row, &column("episode_id"))?, + podcast_id: podcast_id_col(row, &column("podcast_id"))?, + position_ms: row.try_get(column("position_ms").as_str())?, + status: status_from_sql(&row.try_get::(column("status").as_str())?), + generation: generation_from_sql(row.try_get(column("generation").as_str())?), + session_id: row.try_get(column("session_id").as_str())?, + hlc_ms: row.try_get(column("hlc_ms").as_str())?, + device_id: row.try_get(column("device_id").as_str())?, + op_id: row.try_get(column("op_id").as_str())?, + }) +} + +/// Listening session columns of table alias `s` (plain names). +pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \ + s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \ + s.completed"; + +pub(crate) fn session_from_row(row: &PgRow) -> sqlx::Result { + Ok(ListenSession { + session_id: row.try_get("session_id")?, + episode_id: episode_id_col(row, "episode_id")?, + podcast_id: podcast_id_col(row, "podcast_id")?, + device_id: row.try_get("device_id")?, + started_at_ms: row.try_get("started_at_ms")?, + ended_at_ms: row.try_get("ended_at_ms")?, + start_position_ms: row.try_get("start_position_ms")?, + end_position_ms: row.try_get("end_position_ms")?, + listened_ms: row.try_get("listened_ms")?, + completed: row.try_get("completed")?, + }) +} + +/// Normalized search text of an episode: `normalize_name` of the title and +/// of the description, separated by a newline (the normalization never +/// produces one, so a needle cannot match across the two). +pub(crate) fn episode_search_norm(title: &str, description: Option<&str>) -> String { + format!( + "{}\n{}", + music_dht::normalize_name(title), + music_dht::normalize_name(description.unwrap_or_default()) + ) +} diff --git a/src/podcasts/postgres_tests.rs b/src/podcasts/postgres_tests.rs new file mode 100644 index 0000000..f626996 --- /dev/null +++ b/src/podcasts/postgres_tests.rs @@ -0,0 +1,1467 @@ +//! Podcast store against PostgreSQL. +//! +//! Ignored by default: set `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a +//! disposable database (for example a throwaway `postgres:16-alpine` +//! container). The tests migrate it with the production migrations, use +//! random account ids and remove their rows again; the migration test +//! creates and drops its own database next to it. + +use std::collections::BTreeSet; + +use cot::db::migrations::{MigrationEngine, SyncDynMigration}; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot, random_hex}; +use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; +use music_dht::podcast::{ + EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord, + ProgressStatus, ShareRecord, SubscriptionRecord, +}; + +use super::*; +use crate::player::queue_items::test_episode; + +const DATABASE_URL: &str = "FURU_DEVICE_SYNC_TEST_DATABASE_URL"; + +/// Every production migration (the list of `admin::App::migrations`). +fn all_migrations() -> Vec<&'static SyncDynMigration> { + [ + crate::config::db_migrations::MIGRATIONS, + crate::user::db_migrations::MIGRATIONS, + crate::music::db_migrations::MIGRATIONS, + crate::scheduler::db_migrations::MIGRATIONS, + crate::auth::db_migrations::MIGRATIONS, + crate::youtube::db_migrations::MIGRATIONS, + ] + .into_iter() + .flatten() + .copied() + .collect() +} + +async fn migrate(url: &str, migrations: Vec<&'static SyncDynMigration>) { + let database = cot::db::Database::new(url).await.unwrap(); + MigrationEngine::new(migrations) + .unwrap() + .run(&database) + .await + .unwrap(); + database.close().await.unwrap(); +} + +/// Pool on the test database, migrated once per test binary with the +/// production migrations (also used by the device-sync Postgres tests). +pub(crate) async fn test_pool() -> sqlx::PgPool { + static MIGRATED: tokio::sync::Mutex = tokio::sync::Mutex::const_new(false); + let url = std::env::var(DATABASE_URL).unwrap(); + let mut migrated = MIGRATED.lock().await; + if !*migrated { + migrate(&url, all_migrations()).await; + *migrated = true; + } + sqlx::PgPool::connect(&url).await.unwrap() +} + +fn random_user() -> i64 { + let bytes = blake3::hash(random_hex(8).as_bytes()); + 9_000_000_000 + + i64::from(u32::from_le_bytes(bytes.as_bytes()[..4].try_into().unwrap()) % 1_000_000_000) +} + +const ACCOUNT_TABLES: &[&str] = &[ + "furumusic__podcast_fetch_state", + "furumusic__podcast_listen_session", + "furumusic__podcast_progress", + "furumusic__podcast_share", + "furumusic__podcast_subscription", + "furumusic__podcast_episode", + "furumusic__podcast_feed_alias", + "furumusic__podcast", + "furumusic__fed_podcast_relay", + "furumusic__fed_device_identity", + "furumusic__fed_device", + "furumusic__fed_sync_ops", + "furumusic__fed_sync_vector", + "furumusic__fed_state_like", + "furumusic__fed_state_playlist", + "furumusic__fed_state_playlist_item", +]; + +async fn cleanup(pool: &sqlx::PgPool, users: impl IntoIterator) { + let users: Vec = users.into_iter().collect(); + for table in ACCOUNT_TABLES { + sqlx::query(&format!("DELETE FROM {table} WHERE user_id = ANY($1)")) + .bind(&users) + .execute(pool) + .await + .unwrap(); + } +} + +fn feed_url(path: &str) -> FeedUrl { + FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap() +} + +fn meta(path: &str, title: &str) -> PodcastMeta { + PodcastMeta { + feed_url: feed_url(path), + title: title.into(), + author: Some("Host".into()), + description: Some("Channel description".into()), + link: None, + image_url: None, + } +} + +fn op(origin: &str, seq: i64, hlc_ms: i64, payload: SyncOpPayload) -> SyncOpWire { + SyncOpWire { + op_id: format!("{origin}:{seq}"), + origin_device_id: origin.into(), + seq, + hlc_ms, + payload, + } +} + +fn session( + id: &str, + device: &str, + episode: &PodcastEpisodeRef, + listened_ms: i64, + completed: bool, +) -> ListenSession { + ListenSession { + session_id: id.into(), + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + device_id: device.into(), + started_at_ms: 1_000, + ended_at_ms: 1_000 + listened_ms, + start_position_ms: 0, + end_position_ms: listened_ms, + listened_ms, + completed, + } +} + +fn progress_op( + origin: &str, + seq: i64, + hlc_ms: i64, + episode: &PodcastEpisodeRef, + generation: u64, + position_ms: i64, +) -> SyncOpWire { + op( + origin, + seq, + hlc_ms, + SyncOpPayload::PodcastProgressSet { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms, + status: ProgressStatus::InProgress, + generation, + session_id: format!("{origin}-g{generation}"), + episode: Some(PodcastEpisodeRef { + progress_generation: None, + ..episode.clone() + }), + }, + ) +} + +/// Every permutation of `0..n` (Heap's algorithm). +fn permutations(n: usize) -> Vec> { + fn heap(k: usize, items: &mut Vec, out: &mut Vec>) { + if k <= 1 { + out.push(items.clone()); + return; + } + for i in 0..k { + heap(k - 1, items, out); + let j = if k.is_multiple_of(2) { i } else { 0 }; + items.swap(j, k - 1); + } + } + let mut out = Vec::new(); + heap(n, &mut (0..n).collect(), &mut out); + out +} + +async fn apply_all(store: &PodcastStore, user_id: i64, ops: &[SyncOpWire]) { + for op in ops { + let outcome = store.apply_remote_op(user_id, op).await.unwrap(); + assert!( + matches!(outcome, ApplyOutcome::Changed | ApplyOutcome::Unchanged), + "{outcome:?}" + ); + } +} + +/// The registers and sessions of one podcast and episode from three +/// devices, with a duplicate, a late checkpoint of an older session, a +/// concurrent offline action and a growing session. +fn convergence_ops() -> Vec { + let episode = test_episode(); + let podcast_id = episode.podcast_id.clone(); + let podcast = PodcastMeta { + feed_url: episode.feed_url.clone(), + title: episode.podcast_title.clone(), + author: None, + description: Some("dropped from snapshots".into()), + link: None, + image_url: None, + }; + let subscribe = op( + "tui_a", + 1, + 100, + SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id.clone(), + subscribed: true, + podcast: podcast.clone(), + feed_aliases: vec![], + generation: 1, + }, + ); + vec![ + subscribe.clone(), + // B saw g1 and unsubscribes with a slow clock: g2 still wins. + op( + "tui_b", + 1, + 50, + SyncOpPayload::PodcastSubscriptionSet { + podcast_id: podcast_id.clone(), + subscribed: false, + podcast, + feed_aliases: vec![], + generation: 2, + }, + ), + // A shared concurrently at g2: stored, but not effective. + op( + "tui_a", + 2, + 300, + SyncOpPayload::PodcastShareSet { + podcast_id: podcast_id.clone(), + shared: true, + generation: 2, + }, + ), + op( + "tui_c", + 1, + 400, + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id, + feed_aliases: vec![feed_url("moved.xml")], + }, + ), + subscribe, + // A late g3 checkpoint of an old session with a fast clock. + progress_op("tui_a", 3, 9_000, &episode, 3, 600_000), + // B and C both seek at g4 with the same clock: device id decides. + progress_op("tui_b", 2, 1_000, &episode, 4, 10_000), + progress_op("tui_c", 2, 1_000, &episode, 4, 20_000), + op( + "tui_a", + 4, + 9_100, + SyncOpPayload::PodcastListenRecorded { + session: session("s1", "tui_a", &episode, 40_000, false), + episode: None, + }, + ), + op( + "tui_a", + 5, + 9_200, + SyncOpPayload::PodcastListenRecorded { + session: session("s1", "tui_a", &episode, 90_000, true), + episode: None, + }, + ), + op( + "tui_b", + 3, + 1_100, + SyncOpPayload::PodcastListenRecorded { + session: session("s2", "tui_b", &episode, 31_000, false), + episode: Some(PodcastEpisodeRef { + progress_generation: None, + ..episode + }), + }, + ), + ] +} + +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_ops_converge_in_any_order_and_match_snapshot_import() { + let pool = test_pool().await; + let store = PodcastStore::new(pool.clone()); + let ops = convergence_ops(); + let reference_user = random_user(); + let mut users = vec![reference_user]; + apply_all(&store, reference_user, &ops).await; + let reference = store.export_snapshot(reference_user).await.unwrap(); + + // The contract outcome, independently of the order. + let episode = test_episode(); + assert_eq!(reference.podcast_subscriptions.len(), 1); + let subscription = &reference.podcast_subscriptions[0]; + assert!(!subscription.subscribed); + assert_eq!(subscription.generation, 2); + assert!(subscription.podcast.description.is_none()); + assert_eq!( + subscription.feed_urls(), + vec![episode.feed_url.clone(), feed_url("moved.xml")] + ); + assert!(reference.podcast_shares[0].shared); + let summary = store + .podcast(reference_user, &episode.podcast_id) + .await + .unwrap() + .unwrap(); + assert!(!summary.subscribed && !summary.shared); + assert_eq!(reference.podcast_progress.len(), 1); + assert_eq!(reference.podcast_progress[0].position_ms, 20_000); + assert_eq!(reference.podcast_progress[0].device_id, "tui_c"); + let sessions: BTreeSet<_> = reference + .podcast_listen_sessions + .iter() + .map(|session| { + ( + session.session_id.as_str(), + session.listened_ms, + session.completed, + ) + }) + .collect(); + assert_eq!( + sessions, + BTreeSet::from([("s1", 90_000, true), ("s2", 31_000, false)]) + ); + assert_eq!( + store + .completed_listen_count(reference_user, &episode.episode_id) + .await + .unwrap(), + 1 + ); + // The episode reference created a catalog row (never touching progress). + let item = store + .episode(reference_user, &episode.episode_id) + .await + .unwrap() + .unwrap(); + assert!(!item.in_feed); + assert_eq!(item.resume_position_ms(), 20_000); + + // Register operations: all 120 orders of the first five operations + // (subscription, unsubscription, share, aliases, duplicate) with the + // rest appended; progress and session operations: all 720 orders of + // the last six with the first five prepended. + let (registers, personal) = ops.split_at(5); + for order in permutations(registers.len()) { + let user_id = random_user(); + users.push(user_id); + let mut shuffled: Vec = order.iter().map(|&i| registers[i].clone()).collect(); + shuffled.extend_from_slice(personal); + apply_all(&store, user_id, &shuffled).await; + assert_eq!( + store.export_snapshot(user_id).await.unwrap(), + reference, + "{order:?}" + ); + } + for order in permutations(personal.len()) { + let user_id = random_user(); + users.push(user_id); + let mut shuffled = registers.to_vec(); + shuffled.extend(order.iter().map(|&i| personal[i].clone())); + apply_all(&store, user_id, &shuffled).await; + assert_eq!( + store.export_snapshot(user_id).await.unwrap(), + reference, + "{order:?}" + ); + } + + // A new device recovering from the snapshot reaches the same state, and + // the operations delivered afterwards change nothing. (Snapshots omit + // the channel description, so the winning subscription operation only + // adds it back to the stored register: the contract's tie-break by + // metadata. Every other operation is already reflected.) + let recovered = random_user(); + users.push(recovered); + let stats = store.apply_snapshot(recovered, &reference).await.unwrap(); + assert_eq!(stats.rejected, 0); + assert!(!stats.ignored_over_limits); + assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); + for op in ops.iter().rev() { + let expected = if op.op_id == "tui_b:1" { + ApplyOutcome::Changed + } else { + ApplyOutcome::Unchanged + }; + assert_eq!( + store.apply_remote_op(recovered, op).await.unwrap(), + expected, + "{}", + op.op_id + ); + } + assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); + // A stale snapshot never replaces newer state. + let stale = SyncSnapshot { + podcast_progress: vec![ProgressRecord { + position_ms: 1, + generation: 3, + hlc_ms: i64::MAX, + ..reference.podcast_progress[0].clone() + }], + ..SyncSnapshot::default() + }; + let stats = store.apply_snapshot(recovered, &stale).await.unwrap(); + assert_eq!((stats.changed, stats.unchanged), (0, 1)); + assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); + // Invalid operations are rejected without touching state. + let mut invalid = progress_op("tui_a", 9, 1, &episode, 9, 1); + if let SyncOpPayload::PodcastProgressSet { session_id, .. } = &mut invalid.payload { + session_id.clear(); + } + assert!(matches!( + store.apply_remote_op(recovered, &invalid).await.unwrap(), + ApplyOutcome::Rejected(_) + )); + let mut huge = progress_op("tui_a", 10, 1, &episode, u64::MAX, 1); + huge.op_id = "tui_a:10".into(); + assert!(matches!( + store.apply_remote_op(recovered, &huge).await.unwrap(), + ApplyOutcome::Rejected(_) + )); + assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); + + cleanup(&pool, users).await; +} + +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_local_actions_keep_progress_through_unsubscribe() { + let pool = test_pool().await; + let store = PodcastStore::new(pool.clone()); + let user_id = random_user(); + let device = crate::federation::devices::local_device_id(&pool, user_id) + .await + .unwrap(); + let episode = test_episode(); + let show = PodcastMeta { + feed_url: episode.feed_url.clone(), + ..meta("unused.xml", &episode.podcast_title) + }; + + let subscribed = store.subscribe(user_id, &show).await.unwrap(); + assert_eq!(subscribed.podcast_id, episode.podcast_id); + assert!(!subscribed.already_subscribed); + let [first] = subscribed.ops.as_slice() else { + panic!("one operation"); + }; + assert_eq!(first.origin_device_id, device); + assert_eq!(first.op_id, format!("{device}:{}", first.seq)); + // The operation is in the log with exactly this stamp; applying it again + // is a no-op. + let logged: serde_json::Value = sqlx::query_scalar( + "SELECT payload_json FROM furumusic__fed_sync_ops + WHERE user_id = $1 AND op_id = $2 AND hlc_ms = $3 AND kind = 'podcast_subscription_set'", + ) + .bind(user_id) + .bind(&first.op_id) + .bind(first.hlc_ms) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(logged, serde_json::to_value(&first.payload).unwrap()); + assert_eq!( + store.apply_remote_op(user_id, first).await.unwrap(), + ApplyOutcome::Unchanged + ); + assert!( + store + .subscribe(user_id, &show) + .await + .unwrap() + .already_subscribed + ); + assert_eq!( + store + .set_shared(user_id, &episode.podcast_id, true) + .await + .unwrap() + .len(), + 1 + ); + assert!( + store + .set_shared(user_id, &episode.podcast_id, true) + .await + .unwrap() + .is_empty() + ); + assert!( + store + .podcast(user_id, &episode.podcast_id) + .await + .unwrap() + .unwrap() + .shared + ); + + // Listening: an explicit start, checkpoints keep the generation. + let start = store + .start_session(user_id, &episode, "web-s1", 0, episode.progress_generation) + .await + .unwrap(); + assert_eq!(start.record.generation, 8, "max(local, carried 7) + 1"); + let CheckpointOutcome::Written(checkpoint) = store + .checkpoint(user_id, &start.session, 30_000) + .await + .unwrap() + else { + panic!("checkpoint written"); + }; + assert_eq!(checkpoint.record.generation, 8); + assert_eq!( + store + .resume_position_ms(user_id, &episode.episode_id) + .await + .unwrap(), + 30_000 + ); + // A seek is explicit: later checkpoints use its generation. + let seek = store + .seek(user_id, &checkpoint.session, 12_000) + .await + .unwrap(); + assert_eq!(seek.record.generation, 9); + assert_eq!(seek.session.generation, 9); + let start = seek; + // Another device seeks back at a greater generation with a slow clock: + // later checkpoints of the old session are refused. + store + .apply_remote_op(user_id, &progress_op("tui_b", 1, 1, &episode, 10, 5_000)) + .await + .unwrap(); + assert!(matches!( + store + .checkpoint(user_id, &start.session, 900_000) + .await + .unwrap(), + CheckpointOutcome::Superseded { .. } + )); + assert_eq!( + store + .resume_position_ms(user_id, &episode.episode_id) + .await + .unwrap(), + 5_000 + ); + let listened = ListenSession { + device_id: device.clone(), + ..session("web-s1", &device, &episode, 45_000, false) + }; + assert!( + store + .record_listen_session(user_id, &listened, None) + .await + .unwrap() + .is_some() + ); + assert!( + store + .record_listen_session(user_id, &listened, None) + .await + .unwrap() + .is_none() + ); + assert!( + store + .record_listen_session( + user_id, + &session("x", "tui_b", &episode, 45_000, false), + None + ) + .await + .is_err() + ); + + // Unsubscribing keeps progress and history and revokes the share. + let ops = store + .unsubscribe(user_id, &episode.podcast_id) + .await + .unwrap(); + assert_eq!(ops.len(), 2); + assert!( + store + .unsubscribe(user_id, &episode.podcast_id) + .await + .unwrap() + .is_empty() + ); + let summary = store + .podcast(user_id, &episode.podcast_id) + .await + .unwrap() + .unwrap(); + assert!(!summary.subscribed && !summary.shared); + assert_eq!( + store + .resume_position_ms(user_id, &episode.episode_id) + .await + .unwrap(), + 5_000 + ); + assert_eq!( + store.listen_history(user_id, None, 10).await.unwrap().len(), + 1 + ); + assert!( + store + .list_podcasts(user_id, PodcastListFilter::Subscribed) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .list_podcasts(user_id, PodcastListFilter::All) + .await + .unwrap() + .len(), + 1 + ); + + // Subscribing again resolves to the same podcast, continues the + // progress and is private again. + let again = store.subscribe(user_id, &show).await.unwrap(); + assert_eq!(again.podcast_id, episode.podcast_id); + let register = store + .subscription(user_id, &episode.podcast_id) + .await + .unwrap() + .unwrap(); + let share = store + .share(user_id, &episode.podcast_id) + .await + .unwrap() + .unwrap(); + assert!(register.subscribed); + assert!(register.generation > share.generation); + let summary = store + .podcast(user_id, &episode.podcast_id) + .await + .unwrap() + .unwrap(); + assert!(summary.subscribed && !summary.shared); + assert_eq!( + store + .resume_position_ms(user_id, &episode.episode_id) + .await + .unwrap(), + 5_000 + ); + + // Mark played writes progress only; restart and unplayed reset it. + let played = store.mark_played(user_id, &episode).await.unwrap(); + assert_eq!(played.record.status, ProgressStatus::Completed); + assert_eq!(played.record.generation, 11); + assert_eq!( + store.listen_history(user_id, None, 10).await.unwrap().len(), + 1 + ); + assert_eq!( + store + .resume_position_ms(user_id, &episode.episode_id) + .await + .unwrap(), + 0 + ); + let restarted = store.restart(user_id, &episode, "web-s2").await.unwrap(); + assert_eq!( + ( + restarted.record.position_ms, + restarted.record.status, + restarted.record.generation + ), + (0, ProgressStatus::InProgress, 12) + ); + let finished = store + .complete_session(user_id, &restarted.session, 1_790_000) + .await + .unwrap(); + assert_eq!( + (finished.record.status, finished.record.generation), + (ProgressStatus::Completed, 13) + ); + assert!(matches!( + store + .checkpoint(user_id, &finished.session, 1_795_000) + .await + .unwrap(), + CheckpointOutcome::AlreadyCompleted { .. } + )); + let mut played_filter = EpisodeQuery::new(episode.podcast_id.clone()); + played_filter.played = PlayedFilter::Played; + assert_eq!( + store.episodes(user_id, &played_filter).await.unwrap().len(), + 1 + ); + played_filter.played = PlayedFilter::Unplayed; + assert!( + store + .episodes(user_id, &played_filter) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .mark_unplayed(user_id, &episode) + .await + .unwrap() + .record + .status, + ProgressStatus::NotStarted + ); + played_filter.played = PlayedFilter::NotStarted; + let items = store.episodes(user_id, &played_filter).await.unwrap(); + assert_eq!(items[0].status(), ProgressStatus::NotStarted); + + // Local stamps: consecutive sequence numbers, strictly growing clock. + let stamps: Vec<(i64, i64)> = sqlx::query_as( + "SELECT seq, hlc_ms FROM furumusic__fed_sync_ops + WHERE user_id = $1 AND origin_device_id = $2 ORDER BY seq", + ) + .bind(user_id) + .bind(&device) + .fetch_all(&pool) + .await + .unwrap(); + assert!( + stamps + .windows(2) + .all(|pair| pair[1].0 == pair[0].0 + 1 && pair[1].1 > pair[0].1) + ); + // Re-applying the whole local log changes nothing. + let before = store.export_snapshot(user_id).await.unwrap(); + let payloads: Vec<(String, i64, i64, serde_json::Value)> = sqlx::query_as( + "SELECT op_id, seq, hlc_ms, payload_json FROM furumusic__fed_sync_ops + WHERE user_id = $1 AND kind LIKE 'podcast\\_%' ORDER BY hlc_ms", + ) + .bind(user_id) + .fetch_all(&pool) + .await + .unwrap(); + for (op_id, seq, hlc_ms, payload) in payloads { + let origin = op_id.rsplit_once(':').unwrap().0.to_string(); + let op = SyncOpWire { + op_id, + origin_device_id: origin, + seq, + hlc_ms, + payload: serde_json::from_value(payload).unwrap(), + }; + assert_eq!( + store.apply_remote_op(user_id, &op).await.unwrap(), + ApplyOutcome::Unchanged + ); + } + assert_eq!(store.export_snapshot(user_id).await.unwrap(), before); + + // Sharing needs a subscription. + let unknown = PodcastId::from_feed_url(&feed_url("unknown.xml")); + let error = store.set_shared(user_id, &unknown, true).await.unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(PodcastStoreError::NotSubscribed(_)) + )); + + cleanup(&pool, [user_id]).await; +} + +fn rss(items: &str) -> String { + format!( + r#" + + + Refresh Show + https://show.example/ + Channel text + Host + {items} + +"# + ) +} + +fn parse(xml: &str, url: &FeedUrl) -> ParsedFeed { + parse_feed_with_base(xml.as_bytes(), Some(url.as_str())).unwrap() +} + +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_feed_refresh_keeps_ids_and_progress() { + let pool = test_pool().await; + let store = PodcastStore::new(pool.clone()); + let user_id = random_user(); + let url = feed_url("refresh.xml"); + let podcast_id = PodcastId::from_feed_url(&url); + let v1 = parse( + &rss(r#" + g-1One1 + 1Mon, 01 Jan 2024 10:00:00 GMT + + Twohttps://show.example/ep2 + 12 + Mon, 08 Jan 2024 10:00:00 GMT + Talk about Ünïcode + + ThreeMon, 15 Jan 2024 10:00:00 GMT + "#), + &url, + ); + // What every device derives for this document. + let ids: Vec = v1 + .resolve_episode_ids(&podcast_id, &[]) + .into_iter() + .map(Option::unwrap) + .collect(); + + // Another device played the GUID-less "Two" before this account ever + // fetched the feed: its reference arrives with the progress. + let two = PodcastEpisodeRef { + podcast_id: podcast_id.clone(), + episode_id: ids[1].clone(), + feed_url: url.clone(), + podcast_title: "Refresh Show".into(), + episode_title: "Two".into(), + audio_url: "https://cdn.example/a2.mp3".into(), + mime: None, + duration_ms: None, + published_at_ms: None, + season: None, + episode_number: None, + image_url: None, + progress_generation: None, + }; + store + .apply_remote_op(user_id, &progress_op("tui_a", 1, 10, &two, 3, 61_000)) + .await + .unwrap(); + let subscribed = store + .subscribe(user_id, &v1.to_podcast_meta(url.clone())) + .await + .unwrap(); + assert_eq!(subscribed.podcast_id, podcast_id); + let first = store + .apply_feed( + user_id, + &FeedRefresh { + podcast_id: &podcast_id, + feed: &v1, + fetched_from: &url, + moved_permanently: false, + }, + ) + .await + .unwrap(); + assert_eq!((first.new_episodes, first.updated_episodes), (2, 1)); + assert!(first.ops.is_empty()); + let stored: BTreeSet = store + .episodes(user_id, &EpisodeQuery::new(podcast_id.clone())) + .await + .unwrap() + .into_iter() + .map(|item| item.episode.episode_id) + .collect(); + assert_eq!(stored, ids.iter().cloned().collect()); + let progress_before = store.progress(user_id, &ids[1]).await.unwrap().unwrap(); + + // The publisher renames episodes, moves their audio, drops "Three", + // adds "Four" and moves the feed permanently. + let moved = feed_url("moved-refresh.xml"); + let v2 = parse( + &rss(r#" + g-1One (remastered)1 + 1Mon, 01 Jan 2024 10:00:00 GMT + + Two, revisedhttps://show.example/ep2 + 12 + Mon, 08 Jan 2024 10:00:00 GMT + Talk about Ünïcode + + g-4Four2 + 1Mon, 22 Jan 2024 10:00:00 GMT + "#), + &moved, + ); + let refresh = FeedRefresh { + podcast_id: &podcast_id, + feed: &v2, + fetched_from: &moved, + moved_permanently: true, + }; + let second = store.apply_feed(user_id, &refresh).await.unwrap(); + assert_eq!( + ( + second.new_episodes, + second.updated_episodes, + second.unchanged_episodes + ), + (1, 2, 0) + ); + assert_eq!(second.ops.len(), 1, "aliases changed once"); + assert_eq!(second.feed_url.as_ref(), Some(&moved)); + let again = store.apply_feed(user_id, &refresh).await.unwrap(); + assert!(again.ops.is_empty()); + assert_eq!(again.unchanged_episodes, 3); + assert_eq!( + store.resolve_podcast_id(user_id, &moved).await.unwrap(), + podcast_id + ); + assert_eq!( + store.resolve_podcast_id(user_id, &url).await.unwrap(), + podcast_id + ); + + // Ids and progress survive; the dropped episode stays out of the feed. + let one = store.episode(user_id, &ids[0]).await.unwrap().unwrap(); + assert_eq!(one.episode.title, "One (remastered)"); + assert_eq!(one.episode.audio_url, "https://cdn2.example/a1.mp3"); + let two_item = store.episode(user_id, &ids[1]).await.unwrap().unwrap(); + assert_eq!(two_item.episode.audio_url, "https://cdn2.example/a2.mp3"); + assert_eq!(two_item.progress.as_ref(), Some(&progress_before)); + assert_eq!(two_item.resume_position_ms(), 61_000); + let three = store.episode(user_id, &ids[2]).await.unwrap().unwrap(); + assert!(!three.in_feed); + // The subscription register is untouched by refreshes. + let register = store + .subscription(user_id, &podcast_id) + .await + .unwrap() + .unwrap(); + assert_eq!(register.op_id, subscribed.ops[0].op_id); + assert!(register.feed_urls().contains(&moved)); + let reference = store.episode_ref(user_id, &ids[1]).await.unwrap().unwrap(); + assert_eq!(reference.progress_generation, Some(3)); + assert_eq!(reference.audio_url, "https://cdn2.example/a2.mp3"); + + // UI queries: counters, seasons, filters, search and sorting. + let summary = store.podcast(user_id, &podcast_id).await.unwrap().unwrap(); + assert_eq!( + ( + summary.episode_count, + summary.unplayed_count, + summary.in_progress_count, + summary.season_count + ), + (4, 4, 1, 2) + ); + assert_eq!(summary.meta.feed_url, moved); + let seasons = store.seasons(user_id, &podcast_id).await.unwrap(); + assert_eq!( + seasons + .iter() + .map(|season| (season.season, season.episode_count)) + .collect::>(), + vec![(Some(1), 2), (Some(2), 1), (None, 1)] + ); + let query = |configure: &dyn Fn(&mut EpisodeQuery)| { + let mut query = EpisodeQuery::new(podcast_id.clone()); + configure(&mut query); + query + }; + let titles = |items: Vec| { + items + .into_iter() + .map(|item| item.episode.title) + .collect::>() + }; + assert_eq!( + titles(store.episodes(user_id, &query(&|_| {})).await.unwrap()), + ["Four", "Three", "Two, revised", "One (remastered)"] + ); + assert_eq!( + titles( + store + .episodes(user_id, &query(&|q| q.sort = EpisodeSort::EpisodeNumber)) + .await + .unwrap() + ), + ["One (remastered)", "Two, revised", "Four", "Three"] + ); + assert_eq!( + titles( + store + .episodes(user_id, &query(&|q| q.season = SeasonFilter::Season(1))) + .await + .unwrap() + ), + ["Two, revised", "One (remastered)"] + ); + assert_eq!( + titles( + store + .episodes(user_id, &query(&|q| q.season = SeasonFilter::NoSeason)) + .await + .unwrap() + ), + ["Three"] + ); + assert_eq!( + titles( + store + .episodes(user_id, &query(&|q| q.played = PlayedFilter::InProgress)) + .await + .unwrap() + ), + ["Two, revised"] + ); + assert_eq!( + titles( + store + .episodes(user_id, &query(&|q| q.search = Some("ÜNÏCODE".into()))) + .await + .unwrap() + ), + ["Two, revised"] + ); + assert_eq!( + titles( + store + .episodes(user_id, &query(&|q| q.search = Some("remaster".into()))) + .await + .unwrap() + ), + ["One (remastered)"] + ); + assert_eq!( + titles( + store + .episodes( + user_id, + &query(&|q| { + q.limit = Some(1); + q.offset = 1; + }) + ) + .await + .unwrap() + ), + ["Three"] + ); + assert!( + store + .episodes(user_id, &query(&|q| q.search = Some("one two".into()))) + .await + .unwrap() + .is_empty() + ); + + // A refresh of a podcast this account does not know is refused. + let unknown = PodcastId::from_feed_url(&feed_url("never.xml")); + let error = store + .apply_feed( + user_id, + &FeedRefresh { + podcast_id: &unknown, + ..refresh + }, + ) + .await + .unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(PodcastStoreError::UnknownPodcast(_)) + )); + + cleanup(&pool, [user_id]).await; +} + +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_fetch_state_is_local_only() { + let pool = test_pool().await; + let store = PodcastStore::new(pool.clone()); + let user_id = random_user(); + let due_show = meta("due.xml", "Due"); + let later_show = meta("later.xml", "Later"); + let gone_show = meta("gone.xml", "Gone"); + let mut ids = Vec::new(); + for show in [&due_show, &later_show, &gone_show] { + ids.push(store.subscribe(user_id, show).await.unwrap().podcast_id); + } + store.unsubscribe(user_id, &ids[2]).await.unwrap(); + let before = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap(); + + let now = 1_800_000_000_000; + store + .record_fetch_success( + user_id, + &ids[1], + now, + Some("\"secret-etag\""), + Some("Tue"), + Some(now + 60_000), + ) + .await + .unwrap(); + assert_eq!( + store + .record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1)) + .await + .unwrap(), + 1 + ); + assert_eq!( + store + .record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1)) + .await + .unwrap(), + 2 + ); + store + .set_cover_cache( + user_id, + &ids[0], + Some("https://img.example/c.jpg"), + Some("/var/cache/cover-secret.jpg"), + ) + .await + .unwrap(); + let state = store.fetch_state(user_id, &ids[0]).await.unwrap(); + assert_eq!(state.consecutive_failures, 2); + assert_eq!(state.last_error.as_deref(), Some("boom: 503")); + assert_eq!( + state.cover_path.as_deref(), + Some("/var/cache/cover-secret.jpg") + ); + store + .record_fetch_not_modified(user_id, &ids[1], now + 1, Some(now + 60_000)) + .await + .unwrap(); + let later = store.fetch_state(user_id, &ids[1]).await.unwrap(); + assert_eq!(later.etag.as_deref(), Some("\"secret-etag\"")); + assert_eq!(later.last_success_at_ms, Some(now + 1)); + + // Fetch state changes nothing in the synced state. + let after = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap(); + assert_eq!(after, before); + for secret in ["secret-etag", "boom", "cover-secret", "Channel description"] { + assert!(!after.contains(secret), "{secret}"); + } + let ops: Vec = sqlx::query_scalar( + "SELECT payload_json::text FROM furumusic__fed_sync_ops WHERE user_id = $1", + ) + .bind(user_id) + .fetch_all(&pool) + .await + .unwrap(); + assert!( + ops.iter() + .all(|op| !op.contains("secret-etag") && !op.contains("cover-secret")) + ); + + // Scheduling: overdue first, future and unsubscribed podcasts skipped. + let due: Vec = store + .podcasts_due_for_refresh(now, 10_000) + .await + .unwrap() + .into_iter() + .filter(|(user, ..)| *user == user_id) + .map(|(_, id, _)| id) + .collect(); + assert_eq!(due, vec![ids[0].clone()]); + let unknown = PodcastId::from_feed_url(&feed_url("unknown-fetch.xml")); + assert!( + store + .record_fetch_failure(user_id, &unknown, now, "x", None) + .await + .is_err() + ); + + cleanup(&pool, [user_id]).await; +} + +/// Relay rows exactly as 0.13.0 stored them (`podcast_relay`): the contract +/// records as JSON, keyed by section and entity. +async fn insert_relay( + pool: &sqlx::PgPool, + user_id: i64, + section: &str, + key: &str, + value: serde_json::Value, +) { + sqlx::query( + "INSERT INTO furumusic__fed_podcast_relay + (user_id, section, record_key, record_json, updated_at_ms) + VALUES ($1, $2, $3, $4, 1700000000000)", + ) + .bind(user_id) + .bind(section) + .bind(key) + .bind(value) + .execute(pool) + .await + .unwrap(); +} + +async fn insert_logged_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire, kind: &str) { + sqlx::query( + "INSERT INTO furumusic__fed_sync_ops + (user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms, + received_at_ms, tombstone) + VALUES ($1, $2, $3, $4, $5, $6, $7, 1700000000000, FALSE)", + ) + .bind(user_id) + .bind(&op.op_id) + .bind(&op.origin_device_id) + .bind(op.seq) + .bind(kind) + .bind(serde_json::to_value(&op.payload).unwrap()) + .bind(op.hlc_ms) + .execute(pool) + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] +async fn postgres_podcast_migrations_import_the_relay_state() { + let admin_url = std::env::var(DATABASE_URL).unwrap(); + let admin = sqlx::PgPool::connect(&admin_url).await.unwrap(); + let name = format!("furu_podcast_migration_{}", random_hex(6)); + sqlx::query(&format!("CREATE DATABASE {name}")) + .execute(&admin) + .await + .unwrap(); + let (base, _) = admin_url.rsplit_once('/').unwrap(); + let url = format!("{base}/{name}"); + + // A 0.13.0 database: every migration up to the relay table. + let new_migrations = ["m_0056_create_podcast_store", "m_0057_import_podcast_relay"]; + let old: Vec<_> = all_migrations() + .into_iter() + .filter(|migration| { + !(migration.app_name() == "furumusic" && new_migrations.contains(&migration.name())) + }) + .collect(); + migrate(&url, old).await; + let pool = sqlx::PgPool::connect(&url).await.unwrap(); + + // State relayed by 0.13.0 for two accounts. + let episode = test_episode(); + let user_a = 1; + let user_b = 2; + let subscription = SubscriptionRecord { + podcast_id: episode.podcast_id.clone(), + subscribed: true, + podcast: PodcastMeta { + feed_url: episode.feed_url.clone(), + title: episode.podcast_title.clone(), + author: Some("Host".into()), + description: None, + link: None, + image_url: Some("https://img.example/show.jpg".into()), + }, + feed_aliases: vec![episode.feed_url.clone(), feed_url("old.xml")], + generation: 3, + hlc_ms: 30, + op_id: "tui_a:3".into(), + }; + let share = ShareRecord { + podcast_id: episode.podcast_id.clone(), + shared: false, + generation: 2, + hlc_ms: 20, + op_id: "tui_a:2".into(), + }; + let progress = ProgressRecord { + episode_id: episode.episode_id.clone(), + podcast_id: episode.podcast_id.clone(), + position_ms: 1_200_000, + status: ProgressStatus::InProgress, + generation: 9, + session_id: "tui_a-g9".into(), + hlc_ms: 90, + device_id: "tui_a".into(), + op_id: "tui_a:9".into(), + }; + let listened = session("s9", "tui_a", &episode, 120_000, true); + let pending_podcast = PodcastId::from_feed_url(&feed_url("pending.xml")); + insert_relay( + &pool, + user_a, + "subscription", + subscription.podcast_id.as_str(), + serde_json::to_value(&subscription).unwrap(), + ) + .await; + insert_relay( + &pool, + user_a, + "share", + share.podcast_id.as_str(), + serde_json::to_value(&share).unwrap(), + ) + .await; + insert_relay( + &pool, + user_a, + "progress", + progress.episode_id.as_str(), + serde_json::to_value(&progress).unwrap(), + ) + .await; + insert_relay( + &pool, + user_a, + "session", + "s9", + serde_json::to_value(&listened).unwrap(), + ) + .await; + insert_relay( + &pool, + user_a, + "aliases", + pending_podcast.as_str(), + serde_json::to_value(vec![feed_url("pending.xml")]).unwrap(), + ) + .await; + // An out-of-range generation (no conforming client) is skipped. + let huge = ProgressRecord { + episode_id: EpisodeId::from_guid(&episode.podcast_id, "huge").unwrap(), + generation: u64::MAX, + ..progress.clone() + }; + insert_relay( + &pool, + user_a, + "progress", + huge.episode_id.as_str(), + serde_json::to_value(&huge).unwrap(), + ) + .await; + insert_relay( + &pool, + user_b, + "progress", + progress.episode_id.as_str(), + serde_json::to_value(&ProgressRecord { + position_ms: 5, + ..progress.clone() + }) + .unwrap(), + ) + .await; + // The relayed operations kept the episode references (two versions). + let old_ref = PodcastEpisodeRef { + episode_title: "Old title".into(), + progress_generation: None, + ..episode.clone() + }; + insert_logged_op( + &pool, + user_a, + &progress_op("tui_a", 8, 80, &old_ref, 8, 1_000), + "podcast_progress_set", + ) + .await; + insert_logged_op( + &pool, + user_a, + &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000), + "podcast_progress_set", + ) + .await; + + // Upgrade. + migrate(&url, all_migrations()).await; + let store = PodcastStore::new(pool.clone()); + + let mut expected = SyncSnapshot { + podcast_subscriptions: vec![subscription.clone()], + podcast_shares: vec![share], + podcast_progress: vec![progress.clone()], + podcast_listen_sessions: vec![listened], + ..SyncSnapshot::default() + }; + expected.bound_podcast_state(); + assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected); + assert_eq!( + store + .export_snapshot(user_b) + .await + .unwrap() + .podcast_progress[0] + .position_ms, + 5 + ); + assert_eq!( + store.feed_aliases(user_a, &pending_podcast).await.unwrap(), + vec![feed_url("pending.xml")] + ); + // Catalog rows: the podcast from its register, the episode from the + // newest reference in the log. + let summary = store + .podcast(user_a, &episode.podcast_id) + .await + .unwrap() + .unwrap(); + assert!(summary.subscribed); + assert_eq!( + summary.meta.image_url.as_deref(), + Some("https://img.example/show.jpg") + ); + let item = store + .episode(user_a, &episode.episode_id) + .await + .unwrap() + .unwrap(); + assert_eq!(item.episode.title, episode.episode_title); + assert_eq!(item.episode.season, episode.season); + assert_eq!(item.resume_position_ms(), 1_200_000); + assert_eq!(item.completed_count, 1); + // Imported rows are searchable once their search text is computed. + let mut search = EpisodeQuery::new(episode.podcast_id.clone()); + search.search = Some("episode 42".into()); + assert_eq!(store.episodes(user_a, &search).await.unwrap().len(), 1); + // The store keeps working on imported rows, and the relay table stays. + assert_eq!( + store + .apply_remote_op(user_a, &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000)) + .await + .unwrap(), + ApplyOutcome::Unchanged + ); + let relay_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM furumusic__fed_podcast_relay") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(relay_rows, 7); + // Running the import again (or the migrations) changes nothing. + for sql in schema::import_statements("TRUE") { + sqlx::query(&sql).execute(&pool).await.unwrap(); + } + migrate(&url, all_migrations()).await; + assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected); + + pool.close().await; + sqlx::query(&format!("DROP DATABASE {name} WITH (FORCE)")) + .execute(&admin) + .await + .unwrap(); +} diff --git a/src/podcasts/queries.rs b/src/podcasts/queries.rs new file mode 100644 index 0000000..dad3f17 --- /dev/null +++ b/src/podcasts/queries.rs @@ -0,0 +1,664 @@ +//! Read-only queries for the podcast UI and the player. + +use anyhow::Result; +use music_dht::podcast::{ + EpisodeId, EpisodeMeta, EpisodeType, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, + PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, effective_share, +}; +use sqlx::postgres::PgRow; +use sqlx::{PgConnection, Postgres, QueryBuilder, Row}; + +use super::apply; +use super::{ + PodcastStore, SESSION_COLUMNS, episode_id_col, episode_search_norm, feed_url_col, + podcast_id_col, progress_columns, progress_from_row, session_from_row, u32_from_sql, +}; + +/// Which podcasts [`PodcastStore::list_podcasts`] returns. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum PodcastListFilter { + /// Podcasts with an active subscription. + #[default] + Subscribed, + /// Every podcast in the catalog, including unsubscribed ones and + /// podcasts only known from synced progress or queue items. + All, +} + +/// A podcast with its personal state and episode counters. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct PodcastSummary { + /// Podcast id. + pub(crate) podcast_id: PodcastId, + /// Local channel metadata; `feed_url` is the URL this account fetches. + pub(crate) meta: PodcastMeta, + /// Whether the subscription register says subscribed. + pub(crate) subscribed: bool, + /// Whether the podcast must currently be published (`effective_share`). + pub(crate) shared: bool, + /// Episodes in the catalog. + pub(crate) episode_count: u64, + /// Episodes not completed (no progress, not started or in progress). + pub(crate) unplayed_count: u64, + /// Episodes in progress. + pub(crate) in_progress_count: u64, + /// Newest publication time. + pub(crate) latest_published_at_ms: Option, + /// Numbered seasons (episodes without a season are not counted). + pub(crate) season_count: u64, +} + +/// One season group of a podcast. `season == None` groups the episodes +/// without `itunes:season`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SeasonSummary { + /// Season number; `None` for episodes without a season. + pub(crate) season: Option, + /// Episodes in the group. + pub(crate) episode_count: u64, + /// Episodes not completed. + pub(crate) unplayed_count: u64, + /// Oldest publication time. + pub(crate) first_published_at_ms: Option, + /// Newest publication time. + pub(crate) last_published_at_ms: Option, +} + +/// Season filter of an [`EpisodeQuery`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum SeasonFilter { + /// All episodes. + #[default] + Any, + /// Episodes of one season. + Season(u32), + /// Episodes without a season. + NoSeason, +} + +/// Listening-state filter of an [`EpisodeQuery`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum PlayedFilter { + /// All episodes. + #[default] + Any, + /// Not completed (not started or in progress). + Unplayed, + /// Never started or marked unplayed. + NotStarted, + /// In progress. + InProgress, + /// Completed. + Played, +} + +/// Episode ordering of an [`EpisodeQuery`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum EpisodeSort { + /// Newest publication first; undated episodes last. + #[default] + NewestFirst, + /// Oldest publication first; undated episodes last. + OldestFirst, + /// By season, then episode number, ascending; unnumbered last. + EpisodeNumber, + /// By season, then episode number, descending; unnumbered last. + EpisodeNumberDesc, +} + +impl EpisodeSort { + fn order_by(self) -> &'static str { + match self { + Self::NewestFirst => "e.published_at_ms IS NULL, e.published_at_ms DESC, e.episode_id", + Self::OldestFirst => "e.published_at_ms IS NULL, e.published_at_ms, e.episode_id", + Self::EpisodeNumber => { + "e.season IS NULL, e.season, e.episode_number IS NULL, e.episode_number, \ + e.published_at_ms IS NULL, e.published_at_ms, e.episode_id" + } + Self::EpisodeNumberDesc => { + "e.season IS NULL, e.season DESC, e.episode_number IS NULL, \ + e.episode_number DESC, e.published_at_ms IS NULL, e.published_at_ms DESC, \ + e.episode_id" + } + } + } +} + +#[cfg(test)] +pub(crate) fn order_by_for_tests(sort: EpisodeSort) -> &'static str { + sort.order_by() +} + +/// Parameters of [`PodcastStore::episodes`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct EpisodeQuery { + /// Podcast. + pub(crate) podcast_id: PodcastId, + /// Season filter. + pub(crate) season: SeasonFilter, + /// Listening-state filter. + pub(crate) played: PlayedFilter, + /// Text searched in titles and descriptions (`normalize_name`: Unicode + /// NFKC, case-insensitive substring match). Blank means no filter. + pub(crate) search: Option, + /// Ordering. + pub(crate) sort: EpisodeSort, + /// Maximum rows (`None`: all). + pub(crate) limit: Option, + /// Rows to skip. + pub(crate) offset: usize, +} + +impl EpisodeQuery { + /// All episodes of a podcast, newest first. + pub(crate) fn new(podcast_id: PodcastId) -> Self { + Self { + podcast_id, + season: SeasonFilter::Any, + played: PlayedFilter::Any, + search: None, + sort: EpisodeSort::NewestFirst, + limit: None, + offset: 0, + } + } +} + +/// An episode with its personal state. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct EpisodeListItem { + /// Episode metadata (feed or synced reference). + pub(crate) episode: EpisodeMeta, + /// Progress register, if any. + pub(crate) progress: Option, + /// Distinct completed listening sessions (`completed_listen_count`). + pub(crate) completed_count: u64, + /// Whether the latest refresh of the feed listed the episode. + pub(crate) in_feed: bool, +} + +impl EpisodeListItem { + /// Listening status (`NotStarted` without a register). + pub(crate) fn status(&self) -> ProgressStatus { + self.progress + .as_ref() + .map_or(ProgressStatus::NotStarted, |progress| progress.status) + } + + /// Position ordinary Play resumes from. + pub(crate) fn resume_position_ms(&self) -> i64 { + self.progress + .as_ref() + .map_or(0, ProgressRecord::resume_position_ms) + } +} + +/// A listening session in history, with display titles when known. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PodcastListenEntry { + /// The session. + pub(crate) session: ListenSession, + /// Episode title from the catalog. + pub(crate) episode_title: Option, + /// Podcast title from the catalog. + pub(crate) podcast_title: Option, +} + +const EPISODE_COLUMNS: &str = "e.episode_id, e.podcast_id, e.guid, e.title, e.description, \ + e.season, e.episode_number, e.episode_type, e.published_at_ms, e.duration_ms, \ + e.audio_url, e.mime, e.byte_len, e.image_url, e.link"; + +const NOT_COMPLETED: &str = "COALESCE(g.status, 'not_started') <> 'completed'"; + +fn episode_from_row(row: &PgRow) -> sqlx::Result { + Ok(EpisodeMeta { + episode_id: episode_id_col(row, "episode_id")?, + podcast_id: podcast_id_col(row, "podcast_id")?, + guid: row.try_get("guid")?, + title: row.try_get("title")?, + description: row.try_get("description")?, + season: u32_from_sql(row.try_get("season")?), + episode_number: u32_from_sql(row.try_get("episode_number")?), + episode_type: EpisodeType::parse(&row.try_get::("episode_type")?), + published_at_ms: row.try_get("published_at_ms")?, + duration_ms: row.try_get("duration_ms")?, + audio_url: row.try_get("audio_url")?, + mime: row.try_get("mime")?, + byte_len: row + .try_get::, _>("byte_len")? + .and_then(|len| u64::try_from(len).ok()), + image_url: row.try_get("image_url")?, + link: row.try_get("link")?, + }) +} + +fn list_item_from_row(row: &PgRow) -> sqlx::Result { + let progress = if row.try_get::, _>("g_episode_id")?.is_some() { + Some(progress_from_row(row, "g_")?) + } else { + None + }; + Ok(EpisodeListItem { + episode: episode_from_row(row)?, + in_feed: row.try_get("in_feed")?, + progress, + completed_count: count(row.try_get("completed_count")?), + }) +} + +fn count(value: i64) -> u64 { + u64::try_from(value).unwrap_or(0) +} + +pub(crate) async fn episode_meta_in( + conn: &mut PgConnection, + user_id: i64, + episode_id: &EpisodeId, +) -> Result> { + let row = sqlx::query(&format!( + "SELECT {EPISODE_COLUMNS} FROM furumusic__podcast_episode e + WHERE e.user_id = $1 AND e.episode_id = $2" + )) + .bind(user_id) + .bind(episode_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + Ok(row.as_ref().map(episode_from_row).transpose()?) +} + +pub(crate) async fn podcast_meta_in( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let row = sqlx::query( + "SELECT feed_url, title, author, description, link, image_url + FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + let Some(row) = row else { + return Ok(None); + }; + Ok(Some(PodcastMeta { + feed_url: feed_url_col(&row, "feed_url")?, + title: row.try_get("title")?, + author: row.try_get("author")?, + description: row.try_get("description")?, + link: row.try_get("link")?, + image_url: row.try_get("image_url")?, + })) +} + +/// Queue reference of a catalog episode, with `progress_generation` set to +/// the generation of its progress register (handoff hint). +pub(crate) async fn episode_ref_in( + conn: &mut PgConnection, + user_id: i64, + episode_id: &EpisodeId, +) -> Result> { + let Some(episode) = episode_meta_in(conn, user_id, episode_id).await? else { + return Ok(None); + }; + let Some(podcast) = podcast_meta_in(conn, user_id, &episode.podcast_id).await? else { + return Ok(None); + }; + let mut reference = episode.to_ref(&podcast); + reference.progress_generation = apply::load_progress(conn, user_id, episode_id) + .await? + .map(|progress| progress.generation); + Ok(Some(reference)) +} + +async fn summary_in( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let Some(meta) = podcast_meta_in(conn, user_id, podcast_id).await? else { + return Ok(None); + }; + let row = sqlx::query(&format!( + "SELECT COUNT(*) AS episode_count, + COUNT(*) FILTER (WHERE {NOT_COMPLETED}) AS unplayed_count, + COUNT(*) FILTER (WHERE g.status = 'in_progress') AS in_progress_count, + MAX(e.published_at_ms) AS latest, + (SELECT COUNT(DISTINCT e2.season) FROM furumusic__podcast_episode e2 + WHERE e2.user_id = $1 AND e2.podcast_id = $2) AS season_count + FROM furumusic__podcast_episode e + LEFT JOIN furumusic__podcast_progress g + ON g.user_id = e.user_id AND g.episode_id = e.episode_id + WHERE e.user_id = $1 AND e.podcast_id = $2" + )) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_one(&mut *conn) + .await?; + let subscription = apply::load_subscription_register(conn, user_id, podcast_id).await?; + let share = apply::load_share(conn, user_id, podcast_id).await?; + Ok(Some(PodcastSummary { + podcast_id: podcast_id.clone(), + meta, + subscribed: subscription + .as_ref() + .is_some_and(|record| record.subscribed), + shared: effective_share(subscription.as_ref(), share.as_ref()), + episode_count: count(row.try_get("episode_count")?), + unplayed_count: count(row.try_get("unplayed_count")?), + in_progress_count: count(row.try_get("in_progress_count")?), + latest_published_at_ms: row.try_get("latest")?, + season_count: count(row.try_get("season_count")?), + })) +} + +/// Computes the cached search text of episodes that do not have it yet +/// (rows imported by the `m_0057` migration). +async fn backfill_search_norm( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result<()> { + let rows = sqlx::query( + "SELECT episode_id, title, description FROM furumusic__podcast_episode + WHERE user_id = $1 AND podcast_id = $2 AND search_norm IS NULL", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_all(&mut *conn) + .await?; + for row in rows { + let title: String = row.try_get("title")?; + let description: Option = row.try_get("description")?; + sqlx::query( + "UPDATE furumusic__podcast_episode SET search_norm = $3 + WHERE user_id = $1 AND episode_id = $2 AND search_norm IS NULL", + ) + .bind(user_id) + .bind(row.try_get::("episode_id")?) + .bind(episode_search_norm(&title, description.as_deref())) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + +impl PodcastStore { + /// Podcasts with subscription state and counters, sorted by title. + pub(crate) async fn list_podcasts( + &self, + user_id: i64, + filter: PodcastListFilter, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + let join = match filter { + PodcastListFilter::Subscribed => { + "JOIN furumusic__podcast_subscription r + ON r.user_id = p.user_id AND r.podcast_id = p.podcast_id AND r.subscribed" + } + PodcastListFilter::All => "", + }; + let rows = sqlx::query(&format!( + "SELECT p.podcast_id FROM furumusic__podcast p {join} + WHERE p.user_id = $1 + ORDER BY lower(p.title), p.podcast_id" + )) + .bind(user_id) + .fetch_all(&mut *conn) + .await?; + let ids = rows + .iter() + .map(|row| podcast_id_col(row, "podcast_id")) + .collect::>>()?; + let mut out = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(summary) = summary_in(&mut conn, user_id, &id).await? { + out.push(summary); + } + } + Ok(out) + } + + /// One podcast with subscription state and counters. + pub(crate) async fn podcast( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + summary_in(&mut conn, user_id, podcast_id).await + } + + /// The subscription register (with the stored alias set), if any. + pub(crate) async fn subscription( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + apply::load_subscription_register(&mut conn, user_id, podcast_id).await + } + + /// The share register, if any. + pub(crate) async fn share( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + apply::load_share(&mut conn, user_id, podcast_id).await + } + + /// The bounded, synced feed alias set of a podcast. + pub(crate) async fn feed_aliases( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + apply::load_aliases(&mut conn, user_id, podcast_id).await + } + + /// Season groups of a podcast: numbered seasons ascending, then the + /// group of episodes without a season. + pub(crate) async fn seasons( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result> { + let rows = sqlx::query(&format!( + "SELECT s.season, s.episode_count, s.first_published_at_ms, s.last_published_at_ms, + (SELECT COUNT(*) FROM furumusic__podcast_episode e + LEFT JOIN furumusic__podcast_progress g + ON g.user_id = e.user_id AND g.episode_id = e.episode_id + WHERE e.user_id = s.user_id AND e.podcast_id = s.podcast_id + AND e.season IS NOT DISTINCT FROM s.season + AND {NOT_COMPLETED}) AS unplayed_count + FROM furumusic__podcast_season s + WHERE s.user_id = $1 AND s.podcast_id = $2 + ORDER BY s.season IS NULL, s.season" + )) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_all(&self.pool) + .await?; + let seasons = rows + .iter() + .map(|row| { + Ok(SeasonSummary { + season: u32_from_sql(row.try_get("season")?), + episode_count: count(row.try_get("episode_count")?), + first_published_at_ms: row.try_get("first_published_at_ms")?, + last_published_at_ms: row.try_get("last_published_at_ms")?, + unplayed_count: count(row.try_get("unplayed_count")?), + }) + }) + .collect::>>()?; + Ok(seasons) + } + + /// Episodes of a podcast or season with their progress. + pub(crate) async fn episodes( + &self, + user_id: i64, + query: &EpisodeQuery, + ) -> Result> { + let mut conn = self.pool.acquire().await?; + let needle = query + .search + .as_deref() + .map(music_dht::normalize_name) + .filter(|needle| !needle.is_empty()); + if needle.is_some() { + backfill_search_norm(&mut conn, user_id, &query.podcast_id).await?; + } + let mut sql: QueryBuilder<'_, Postgres> = QueryBuilder::new(format!( + "SELECT {EPISODE_COLUMNS}, e.in_feed, {}, + (SELECT COUNT(*) FROM furumusic__podcast_listen_session s + WHERE s.user_id = e.user_id AND s.episode_id = e.episode_id + AND s.completed) AS completed_count + FROM furumusic__podcast_episode e + LEFT JOIN furumusic__podcast_progress g + ON g.user_id = e.user_id AND g.episode_id = e.episode_id + WHERE e.user_id = ", + progress_columns("g", "g_") + )); + sql.push_bind(user_id); + sql.push(" AND e.podcast_id = "); + sql.push_bind(query.podcast_id.as_str().to_string()); + match query.season { + SeasonFilter::Any => {} + SeasonFilter::Season(season) => { + sql.push(" AND e.season = "); + sql.push_bind(i64::from(season)); + } + SeasonFilter::NoSeason => { + sql.push(" AND e.season IS NULL"); + } + } + match query.played { + PlayedFilter::Any => {} + PlayedFilter::Unplayed => { + sql.push(format!(" AND {NOT_COMPLETED}")); + } + PlayedFilter::NotStarted => { + sql.push(" AND COALESCE(g.status, 'not_started') = 'not_started'"); + } + PlayedFilter::InProgress => { + sql.push(" AND g.status = 'in_progress'"); + } + PlayedFilter::Played => { + sql.push(" AND g.status = 'completed'"); + } + } + if let Some(needle) = needle { + sql.push(" AND strpos(e.search_norm, "); + sql.push_bind(needle); + sql.push(") > 0"); + } + sql.push(format!(" ORDER BY {}", query.sort.order_by())); + if let Some(limit) = query.limit { + sql.push(" LIMIT "); + sql.push_bind(i64::try_from(limit).unwrap_or(i64::MAX)); + } + sql.push(" OFFSET "); + sql.push_bind(i64::try_from(query.offset).unwrap_or(i64::MAX)); + let rows = sql.build().fetch_all(&mut *conn).await?; + Ok(rows + .iter() + .map(list_item_from_row) + .collect::>>()?) + } + + /// One episode with its progress. + pub(crate) async fn episode( + &self, + user_id: i64, + episode_id: &EpisodeId, + ) -> Result> { + let row = sqlx::query(&format!( + "SELECT {EPISODE_COLUMNS}, e.in_feed, {}, + (SELECT COUNT(*) FROM furumusic__podcast_listen_session s + WHERE s.user_id = e.user_id AND s.episode_id = e.episode_id + AND s.completed) AS completed_count + FROM furumusic__podcast_episode e + LEFT JOIN furumusic__podcast_progress g + ON g.user_id = e.user_id AND g.episode_id = e.episode_id + WHERE e.user_id = $1 AND e.episode_id = $2", + progress_columns("g", "g_") + )) + .bind(user_id) + .bind(episode_id.as_str()) + .fetch_optional(&self.pool) + .await?; + Ok(row.as_ref().map(list_item_from_row).transpose()?) + } + + /// Position ordinary Play resumes from + /// (`ProgressRecord::resume_position_ms`, 0 without progress). + pub(crate) async fn resume_position_ms( + &self, + user_id: i64, + episode_id: &EpisodeId, + ) -> Result { + Ok(self + .progress(user_id, episode_id) + .await? + .map_or(0, |progress| progress.resume_position_ms())) + } + + /// Listening history (sessions that `should_record`), newest first. + pub(crate) async fn listen_history( + &self, + user_id: i64, + podcast_id: Option<&PodcastId>, + limit: usize, + ) -> Result> { + let rows = sqlx::query(&format!( + "SELECT {SESSION_COLUMNS}, e.title AS episode_title, p.title AS podcast_title + FROM furumusic__podcast_listen_session s + LEFT JOIN furumusic__podcast_episode e + ON e.user_id = s.user_id AND e.episode_id = s.episode_id + LEFT JOIN furumusic__podcast p + ON p.user_id = s.user_id AND p.podcast_id = s.podcast_id + WHERE s.user_id = $1 AND (s.completed OR s.listened_ms >= $2) + AND ($3::text IS NULL OR s.podcast_id = $3) + ORDER BY s.ended_at_ms DESC, s.session_id DESC + LIMIT $4" + )) + .bind(user_id) + .bind(music_dht::podcast::MIN_RECORDED_EPISODE_LISTEN_MS) + .bind(podcast_id.map(PodcastId::as_str)) + .bind(i64::try_from(limit).unwrap_or(i64::MAX)) + .fetch_all(&self.pool) + .await?; + Ok(rows + .iter() + .map(|row| { + Ok(PodcastListenEntry { + session: session_from_row(row)?, + episode_title: row.try_get("episode_title")?, + podcast_title: row.try_get("podcast_title")?, + }) + }) + .collect::>>()?) + } + + /// Number of distinct completed sessions of an episode. + pub(crate) async fn completed_listen_count( + &self, + user_id: i64, + episode_id: &EpisodeId, + ) -> Result { + let total: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM furumusic__podcast_listen_session + WHERE user_id = $1 AND episode_id = $2 AND completed", + ) + .bind(user_id) + .bind(episode_id.as_str()) + .fetch_one(&self.pool) + .await?; + Ok(count(total)) + } +} diff --git a/src/podcasts/refresh.rs b/src/podcasts/refresh.rs new file mode 100644 index 0000000..6cf6899 --- /dev/null +++ b/src/podcasts/refresh.rs @@ -0,0 +1,472 @@ +//! Feed refresh ingestion and the local-only fetch state. + +use std::collections::BTreeSet; + +use anyhow::Result; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire}; +use music_dht::podcast::feed::ParsedFeed; +use music_dht::podcast::{EpisodeMeta, FeedUrl, PodcastId, merge_feed_aliases}; +use sqlx::{PgConnection, Row}; + +use super::apply::{self, EpisodeSource}; +use super::{PodcastStore, PodcastStoreError, episode_search_norm, now_ms}; +use crate::federation::devices; + +/// A fetched and parsed feed document for [`PodcastStore::apply_feed`]. +#[derive(Debug, Clone, Copy)] +pub(crate) struct FeedRefresh<'a> { + /// Podcast the feed belongs to. + pub(crate) podcast_id: &'a PodcastId, + /// The parsed document (`music_dht::podcast::feed::parse_feed_with_base` + /// with `fetched_from` as base). + pub(crate) feed: &'a ParsedFeed, + /// URL the document was finally served from, after redirects. + pub(crate) fetched_from: &'a FeedUrl, + /// Whether every redirect leading to `fetched_from` was permanent + /// (301/308). Temporary redirects do not move the feed. + pub(crate) moved_permanently: bool, +} + +/// Result of [`PodcastStore::apply_feed`]. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct RefreshOutcome { + /// Episodes seen for the first time. + pub(crate) new_episodes: usize, + /// Known episodes whose metadata changed (title, audio URL, ...). + pub(crate) updated_episodes: usize, + /// Known episodes without metadata changes. + pub(crate) unchanged_episodes: usize, + /// Items without identity or with invalid metadata. + pub(crate) skipped_items: usize, + /// Feed URLs learned from a permanent redirect or `itunes:new-feed-url`. + pub(crate) learned_aliases: Vec, + /// URL the podcast is fetched from after this refresh. + pub(crate) feed_url: Option, + /// `podcast_feed_aliases_added`, only when the learned URLs changed the + /// stored alias set (already appended to the sync log). + pub(crate) ops: Vec, +} + +/// Local-only fetch state of a podcast. Never part of sync output. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct FetchState { + /// HTTP `ETag` validator of the last successful fetch. + pub(crate) etag: Option, + /// HTTP `Last-Modified` validator of the last successful fetch. + pub(crate) last_modified: Option, + /// Last fetch attempt, unix milliseconds. + pub(crate) checked_at_ms: Option, + /// Last successful fetch (including "not modified"). + pub(crate) last_success_at_ms: Option, + /// When the scheduler should check again (`None`: due now). + pub(crate) next_check_at_ms: Option, + /// Error of the last attempt; `None` after a success. + pub(crate) last_error: Option, + /// Failed attempts since the last success. + pub(crate) consecutive_failures: u32, + /// Image URL the cached cover was downloaded from. + pub(crate) cover_url: Option, + /// Local path of the cached cover. + pub(crate) cover_path: Option, +} + +async fn podcast_fetch_url( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, +) -> Result> { + let row = sqlx::query( + "SELECT feed_url FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + Ok(row + .as_ref() + .map(|row| super::feed_url_col(row, "feed_url")) + .transpose()?) +} + +async fn fetch_state_in( + conn: &mut PgConnection, + user_id: i64, + podcast_id: &PodcastId, + lock: bool, +) -> Result { + let row = sqlx::query(&format!( + "SELECT etag, last_modified, checked_at_ms, last_success_at_ms, next_check_at_ms, + last_error, consecutive_failures, cover_url, cover_path + FROM furumusic__podcast_fetch_state WHERE user_id = $1 AND podcast_id = $2{}", + if lock { " FOR UPDATE" } else { "" } + )) + .bind(user_id) + .bind(podcast_id.as_str()) + .fetch_optional(&mut *conn) + .await?; + let Some(row) = row else { + return Ok(FetchState::default()); + }; + Ok(FetchState { + etag: row.try_get("etag")?, + last_modified: row.try_get("last_modified")?, + checked_at_ms: row.try_get("checked_at_ms")?, + last_success_at_ms: row.try_get("last_success_at_ms")?, + next_check_at_ms: row.try_get("next_check_at_ms")?, + last_error: row.try_get("last_error")?, + consecutive_failures: u32::try_from(row.try_get::("consecutive_failures")?) + .unwrap_or(u32::MAX), + cover_url: row.try_get("cover_url")?, + cover_path: row.try_get("cover_path")?, + }) +} + +/// Inserts or updates a feed episode; `None` when it is new, otherwise +/// whether its metadata changed. +async fn upsert_feed_episode( + conn: &mut PgConnection, + user_id: i64, + meta: &EpisodeMeta, + now: i64, +) -> Result> { + let Some(stored) = super::queries::episode_meta_in(conn, user_id, &meta.episode_id).await? + else { + apply::insert_episode(conn, user_id, meta, EpisodeSource::Feed, true, now).await?; + return Ok(None); + }; + let changed = stored != *meta; + sqlx::query( + "UPDATE furumusic__podcast_episode + SET podcast_id = $3, guid = $4, title = $5, description = $6, season = $7, + episode_number = $8, episode_type = $9, published_at_ms = $10, + duration_ms = $11, audio_url = $12, mime = $13, byte_len = $14, + image_url = $15, link = $16, source = 'feed', in_feed = TRUE, + search_norm = $17, + updated_at_ms = CASE WHEN $18 THEN $19 ELSE updated_at_ms END + WHERE user_id = $1 AND episode_id = $2", + ) + .bind(user_id) + .bind(meta.episode_id.as_str()) + .bind(meta.podcast_id.as_str()) + .bind(&meta.guid) + .bind(&meta.title) + .bind(&meta.description) + .bind(meta.season.map(i64::from)) + .bind(meta.episode_number.map(i64::from)) + .bind(apply::episode_type_to_sql(meta.episode_type)) + .bind(meta.published_at_ms) + .bind(meta.duration_ms) + .bind(&meta.audio_url) + .bind(&meta.mime) + .bind(meta.byte_len.and_then(|len| i64::try_from(len).ok())) + .bind(&meta.image_url) + .bind(&meta.link) + .bind(episode_search_norm( + &meta.title, + meta.description.as_deref(), + )) + .bind(changed) + .bind(now) + .execute(&mut *conn) + .await?; + Ok(Some(changed)) +} + +impl PodcastStore { + /// Ingests a fetched feed document for a podcast of the account. + /// + /// * Channel metadata of the catalog row is replaced by the feed's. + /// * Episode ids are resolved with the contract identity rule + /// (`ParsedFeed::resolve_episode_ids`) against every stored episode of + /// the podcast, including episodes learned through sync, so a changed + /// title, audio URL or a GUID-less item keeps its id. Known episodes + /// get the new metadata (including the audio URL); new ones are added; + /// episodes missing from the document are kept (`in_feed = false`). + /// * A permanent redirect target and `itunes:new-feed-url` become feed + /// aliases and the new fetch URL. When they change the stored alias + /// set, one `podcast_feed_aliases_added` operation is appended. + /// * Subscription, share and progress registers and listening sessions + /// are never touched, so a refresh cannot resurrect an unsubscribed + /// podcast or move anyone's position. + /// + /// Record the HTTP outcome separately with + /// [`PodcastStore::record_fetch_success`] and friends. + pub(crate) async fn apply_feed( + &self, + user_id: i64, + refresh: &FeedRefresh<'_>, + ) -> Result { + let podcast_id = refresh.podcast_id; + let feed = refresh.feed; + let now = now_ms(); + let device_id = devices::local_device_id(&self.pool, user_id).await?; + let mut tx = self.begin_write(user_id).await?; + let current_url = match podcast_fetch_url(&mut tx, user_id, podcast_id).await? { + Some(url) => url, + None => { + let Some(register) = + apply::load_subscription_register(&mut tx, user_id, podcast_id).await? + else { + return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into()); + }; + apply::apply_subscription_record(&mut tx, user_id, ®ister, now).await?; + register.podcast.feed_url + } + }; + + let known = apply::known_episodes(&mut tx, user_id, podcast_id).await?; + let ids = feed.resolve_episode_ids(podcast_id, &known); + sqlx::query( + "UPDATE furumusic__podcast_episode SET in_feed = FALSE + WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .execute(&mut *tx) + .await?; + let mut outcome = RefreshOutcome::default(); + for (item, id) in feed.episodes.iter().zip(ids) { + let Some(id) = id else { + outcome.skipped_items += 1; + continue; + }; + let meta = item.to_meta(podcast_id, id); + if meta.validate().is_err() { + outcome.skipped_items += 1; + continue; + } + match upsert_feed_episode(&mut tx, user_id, &meta, now).await? { + None => outcome.new_episodes += 1, + Some(true) => outcome.updated_episodes += 1, + Some(false) => outcome.unchanged_episodes += 1, + } + } + + let mut fetch_url = current_url.clone(); + let mut learned = BTreeSet::new(); + if refresh.moved_permanently && *refresh.fetched_from != fetch_url { + learned.insert(fetch_url.clone()); + learned.insert(refresh.fetched_from.clone()); + fetch_url = refresh.fetched_from.clone(); + } + if let Some(new_url) = feed + .new_feed_url + .as_deref() + .and_then(|url| FeedUrl::parse(url).ok()) + && new_url != fetch_url + { + learned.insert(fetch_url.clone()); + learned.insert(new_url.clone()); + fetch_url = new_url; + } + let meta = feed.to_podcast_meta(fetch_url.clone()); + sqlx::query( + "UPDATE furumusic__podcast + SET feed_url = $3, title = $4, author = $5, description = $6, link = $7, + image_url = $8, meta_from_feed = TRUE, updated_at_ms = $9 + WHERE user_id = $1 AND podcast_id = $2", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .bind(meta.feed_url.as_str()) + .bind(&meta.title) + .bind(&meta.author) + .bind(&meta.description) + .bind(&meta.link) + .bind(&meta.image_url) + .bind(now) + .execute(&mut *tx) + .await?; + if !learned.is_empty() { + let learned: Vec = learned.into_iter().collect(); + let current = apply::load_aliases(&mut tx, user_id, podcast_id).await?; + if merge_feed_aliases(podcast_id, ¤t, &learned) != current { + let op = super::actions::emit(&mut tx, user_id, &device_id, |_| { + SyncOpPayload::PodcastFeedAliasesAdded { + podcast_id: podcast_id.clone(), + feed_aliases: learned.clone(), + } + }) + .await?; + outcome.ops.push(op); + } + outcome.learned_aliases = learned; + } + tx.commit().await?; + outcome.feed_url = Some(fetch_url); + Ok(outcome) + } + + /// Local fetch state of a podcast (defaults when never fetched). + pub(crate) async fn fetch_state( + &self, + user_id: i64, + podcast_id: &PodcastId, + ) -> Result { + let mut conn = self.pool.acquire().await?; + fetch_state_in(&mut conn, user_id, podcast_id, false).await + } + + /// Read-modify-write of the fetch state in one transaction with the + /// account lock and the row locked, so concurrent `record_*` / + /// `set_cover_cache` calls never lose each other's columns or failure + /// increments. + async fn update_fetch_state( + &self, + user_id: i64, + podcast_id: &PodcastId, + update: impl FnOnce(&mut FetchState), + ) -> Result { + let mut tx = self.begin_write(user_id).await?; + if podcast_fetch_url(&mut tx, user_id, podcast_id) + .await? + .is_none() + { + return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into()); + } + let mut state = fetch_state_in(&mut tx, user_id, podcast_id, true).await?; + update(&mut state); + sqlx::query( + "INSERT INTO furumusic__podcast_fetch_state + (user_id, podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms, + next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (user_id, podcast_id) DO UPDATE SET + etag = EXCLUDED.etag, last_modified = EXCLUDED.last_modified, + checked_at_ms = EXCLUDED.checked_at_ms, + last_success_at_ms = EXCLUDED.last_success_at_ms, + next_check_at_ms = EXCLUDED.next_check_at_ms, + last_error = EXCLUDED.last_error, + consecutive_failures = EXCLUDED.consecutive_failures, + cover_url = EXCLUDED.cover_url, cover_path = EXCLUDED.cover_path", + ) + .bind(user_id) + .bind(podcast_id.as_str()) + .bind(&state.etag) + .bind(&state.last_modified) + .bind(state.checked_at_ms) + .bind(state.last_success_at_ms) + .bind(state.next_check_at_ms) + .bind(&state.last_error) + .bind(i64::from(state.consecutive_failures)) + .bind(&state.cover_url) + .bind(&state.cover_path) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(state) + } + + /// Records a successful fetch (HTTP 200) with its new validators. + pub(crate) async fn record_fetch_success( + &self, + user_id: i64, + podcast_id: &PodcastId, + now_ms: i64, + etag: Option<&str>, + last_modified: Option<&str>, + next_check_at_ms: Option, + ) -> Result<()> { + self.update_fetch_state(user_id, podcast_id, |state| { + state.etag = etag.map(str::to_string); + state.last_modified = last_modified.map(str::to_string); + state.checked_at_ms = Some(now_ms); + state.last_success_at_ms = Some(now_ms); + state.next_check_at_ms = next_check_at_ms; + state.last_error = None; + state.consecutive_failures = 0; + }) + .await + .map(|_| ()) + } + + /// Records a conditional fetch answered with "304 Not Modified". + pub(crate) async fn record_fetch_not_modified( + &self, + user_id: i64, + podcast_id: &PodcastId, + now_ms: i64, + next_check_at_ms: Option, + ) -> Result<()> { + self.update_fetch_state(user_id, podcast_id, |state| { + state.checked_at_ms = Some(now_ms); + state.last_success_at_ms = Some(now_ms); + state.next_check_at_ms = next_check_at_ms; + state.last_error = None; + state.consecutive_failures = 0; + }) + .await + .map(|_| ()) + } + + /// Records a failed fetch; returns the consecutive failure count. The + /// stored catalog is kept. + pub(crate) async fn record_fetch_failure( + &self, + user_id: i64, + podcast_id: &PodcastId, + now_ms: i64, + error: &str, + next_check_at_ms: Option, + ) -> Result { + self.update_fetch_state(user_id, podcast_id, |state| { + state.checked_at_ms = Some(now_ms); + state.next_check_at_ms = next_check_at_ms; + state.last_error = Some(error.to_string()); + state.consecutive_failures = state.consecutive_failures.saturating_add(1); + }) + .await + .map(|state| state.consecutive_failures) + } + + /// Records the locally cached cover image of a podcast. + pub(crate) async fn set_cover_cache( + &self, + user_id: i64, + podcast_id: &PodcastId, + cover_url: Option<&str>, + cover_path: Option<&str>, + ) -> Result<()> { + self.update_fetch_state(user_id, podcast_id, |state| { + state.cover_url = cover_url.map(str::to_string); + state.cover_path = cover_path.map(str::to_string); + }) + .await + .map(|_| ()) + } + + /// Subscribed podcasts of every account due for a refresh at `now_ms` + /// (never checked or `next_check_at_ms <= now_ms`), most overdue first, + /// with their fetch URL: `(user_id, podcast_id, feed_url)`. + pub(crate) async fn podcasts_due_for_refresh( + &self, + now_ms: i64, + limit: usize, + ) -> Result> { + let rows = sqlx::query( + "SELECT p.user_id, p.podcast_id, p.feed_url + FROM furumusic__podcast p + JOIN furumusic__podcast_subscription s + ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id AND s.subscribed + LEFT JOIN furumusic__podcast_fetch_state f + ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id + WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1 + ORDER BY COALESCE(f.next_check_at_ms, 0), p.user_id, p.podcast_id + LIMIT $2", + ) + .bind(now_ms) + .bind(i64::try_from(limit).unwrap_or(i64::MAX)) + .fetch_all(&self.pool) + .await?; + Ok(rows + .iter() + .map(|row| { + Ok(( + row.try_get("user_id")?, + super::podcast_id_col(row, "podcast_id")?, + super::feed_url_col(row, "feed_url")?, + )) + }) + .collect::>>()?) + } +} diff --git a/src/podcasts/schema.rs b/src/podcasts/schema.rs new file mode 100644 index 0000000..9890041 --- /dev/null +++ b/src/podcasts/schema.rs @@ -0,0 +1,337 @@ +//! PostgreSQL schema of the podcast store and the import of the relay state. +//! +//! The migrations themselves are declared in `crate::music::db_migrations` +//! (`m_0056_create_podcast_store`, `m_0057_import_podcast_relay`), next to +//! the rest of the `furumusic` chain; this module holds their SQL so the +//! store and its tests share one definition. +//! +//! The tables mirror the SQLite reference store of `furumi-library` +//! (`PODCASTS.md`, "SQLite reference store") with a `user_id` column in +//! every key: each web account is one virtual device with its own podcast +//! catalog, registers and fetch state. + +/// Tables, the derived season view and indexes (`m_0056`). +pub(crate) const CREATE_PODCAST_STORE: &[&str] = &[ + // Local channel metadata. `feed_url` is the URL this account fetches; + // `meta_from_feed` is set once the account refreshed the feed itself + // (until then the winning subscription register provides the metadata). + "CREATE TABLE IF NOT EXISTS furumusic__podcast ( + user_id BIGINT NOT NULL, + podcast_id TEXT NOT NULL, + feed_url TEXT NOT NULL, + title TEXT NOT NULL, + author TEXT, + description TEXT, + link TEXT, + image_url TEXT, + meta_from_feed BOOLEAN NOT NULL DEFAULT FALSE, + created_at_ms BIGINT NOT NULL, + updated_at_ms BIGINT NOT NULL, + PRIMARY KEY (user_id, podcast_id) + )", + // Bounded grow-only alias set (`merge_feed_aliases`), synced inside + // subscription records and `podcast_feed_aliases_added`. + "CREATE TABLE IF NOT EXISTS furumusic__podcast_feed_alias ( + user_id BIGINT NOT NULL, + podcast_id TEXT NOT NULL, + feed_url TEXT NOT NULL, + PRIMARY KEY (user_id, podcast_id, feed_url) + )", + "CREATE INDEX IF NOT EXISTS idx_podcast_feed_alias_url + ON furumusic__podcast_feed_alias (user_id, feed_url)", + // Episode catalog (`EpisodeMeta`). `source` is `feed` once this account + // parsed the episode from RSS, `sync` while it is only known from a + // `PodcastEpisodeRef`. `search_norm` caches `normalize_name(title)` and + // `normalize_name(description)` separated by a newline (never produced + // by the normalization), NULL until computed. + "CREATE TABLE IF NOT EXISTS furumusic__podcast_episode ( + user_id BIGINT NOT NULL, + episode_id TEXT NOT NULL, + podcast_id TEXT NOT NULL, + guid TEXT, + title TEXT NOT NULL, + description TEXT, + season BIGINT, + episode_number BIGINT, + episode_type TEXT NOT NULL DEFAULT 'full', + published_at_ms BIGINT, + duration_ms BIGINT, + audio_url TEXT NOT NULL, + mime TEXT, + byte_len BIGINT, + image_url TEXT, + link TEXT, + source TEXT NOT NULL CHECK (source IN ('feed', 'sync')), + in_feed BOOLEAN NOT NULL DEFAULT FALSE, + search_norm TEXT, + first_seen_at_ms BIGINT NOT NULL, + updated_at_ms BIGINT NOT NULL, + PRIMARY KEY (user_id, episode_id), + FOREIGN KEY (user_id, podcast_id) + REFERENCES furumusic__podcast (user_id, podcast_id) ON DELETE CASCADE + )", + "CREATE INDEX IF NOT EXISTS idx_podcast_episode_date + ON furumusic__podcast_episode (user_id, podcast_id, published_at_ms DESC)", + "CREATE INDEX IF NOT EXISTS idx_podcast_episode_season + ON furumusic__podcast_episode (user_id, podcast_id, season, episode_number)", + // Seasons are derived from the episodes; a view cannot go stale. + // `season` NULL groups the episodes without `itunes:season`. + "CREATE OR REPLACE VIEW furumusic__podcast_season AS + SELECT user_id, podcast_id, season, + COUNT(*) AS episode_count, + MIN(published_at_ms) AS first_published_at_ms, + MAX(published_at_ms) AS last_published_at_ms + FROM furumusic__podcast_episode + GROUP BY user_id, podcast_id, season", + // Synced registers. No foreign keys to the catalog: operations for + // podcasts or episodes this account has not fetched are kept as they + // arrive, in any order. + "CREATE TABLE IF NOT EXISTS furumusic__podcast_subscription ( + user_id BIGINT NOT NULL, + podcast_id TEXT NOT NULL, + subscribed BOOLEAN NOT NULL, + podcast_json JSONB NOT NULL, + generation BIGINT NOT NULL CHECK (generation >= 0), + hlc_ms BIGINT NOT NULL, + op_id TEXT NOT NULL, + PRIMARY KEY (user_id, podcast_id) + )", + "CREATE TABLE IF NOT EXISTS furumusic__podcast_share ( + user_id BIGINT NOT NULL, + podcast_id TEXT NOT NULL, + shared BOOLEAN NOT NULL, + generation BIGINT NOT NULL CHECK (generation >= 0), + hlc_ms BIGINT NOT NULL, + op_id TEXT NOT NULL, + PRIMARY KEY (user_id, podcast_id) + )", + "CREATE TABLE IF NOT EXISTS furumusic__podcast_progress ( + user_id BIGINT NOT NULL, + episode_id TEXT NOT NULL, + podcast_id TEXT NOT NULL, + position_ms BIGINT NOT NULL, + status TEXT NOT NULL + CHECK (status IN ('not_started', 'in_progress', 'completed')), + generation BIGINT NOT NULL CHECK (generation >= 0), + session_id TEXT NOT NULL, + hlc_ms BIGINT NOT NULL, + device_id TEXT NOT NULL, + op_id TEXT NOT NULL, + PRIMARY KEY (user_id, episode_id) + )", + "CREATE INDEX IF NOT EXISTS idx_podcast_progress_podcast + ON furumusic__podcast_progress (user_id, podcast_id)", + // Grow-only set of listening sessions. + "CREATE TABLE IF NOT EXISTS furumusic__podcast_listen_session ( + user_id BIGINT NOT NULL, + session_id TEXT NOT NULL, + episode_id TEXT NOT NULL, + podcast_id TEXT NOT NULL, + device_id TEXT NOT NULL, + started_at_ms BIGINT NOT NULL, + ended_at_ms BIGINT NOT NULL, + start_position_ms BIGINT NOT NULL, + end_position_ms BIGINT NOT NULL, + listened_ms BIGINT NOT NULL, + completed BOOLEAN NOT NULL, + PRIMARY KEY (user_id, session_id) + )", + "CREATE INDEX IF NOT EXISTS idx_podcast_listen_session_episode + ON furumusic__podcast_listen_session (user_id, episode_id)", + "CREATE INDEX IF NOT EXISTS idx_podcast_listen_session_end + ON furumusic__podcast_listen_session (user_id, ended_at_ms DESC)", + // Local-only fetch state: never part of sync output. + "CREATE TABLE IF NOT EXISTS furumusic__podcast_fetch_state ( + user_id BIGINT NOT NULL, + podcast_id TEXT NOT NULL, + etag TEXT, + last_modified TEXT, + checked_at_ms BIGINT, + last_success_at_ms BIGINT, + next_check_at_ms BIGINT, + last_error TEXT, + consecutive_failures BIGINT NOT NULL DEFAULT 0, + cover_url TEXT, + cover_path TEXT, + PRIMARY KEY (user_id, podcast_id), + FOREIGN KEY (user_id, podcast_id) + REFERENCES furumusic__podcast (user_id, podcast_id) ON DELETE CASCADE + )", + "CREATE INDEX IF NOT EXISTS idx_podcast_fetch_state_due + ON furumusic__podcast_fetch_state (next_check_at_ms)", +]; + +/// Backwards operation of `m_0056`. +pub(crate) const DROP_PODCAST_STORE: &[&str] = &[ + "DROP VIEW IF EXISTS furumusic__podcast_season", + "DROP TABLE IF EXISTS furumusic__podcast_fetch_state", + "DROP TABLE IF EXISTS furumusic__podcast_listen_session", + "DROP TABLE IF EXISTS furumusic__podcast_progress", + "DROP TABLE IF EXISTS furumusic__podcast_share", + "DROP TABLE IF EXISTS furumusic__podcast_subscription", + "DROP TABLE IF EXISTS furumusic__podcast_episode", + "DROP TABLE IF EXISTS furumusic__podcast_feed_alias", + "DROP TABLE IF EXISTS furumusic__podcast", +]; + +/// Placeholder in [`IMPORT_PODCAST_RELAY`] for the account filter: the +/// migration imports every account (`TRUE`), tests narrow it to one. +pub(crate) const USER_FILTER: &str = "{user_filter}"; + +/// Largest generation the `BIGINT` columns hold; larger records are skipped +/// like the store rejects them. +const GENERATION_FITS: &str = "(record_json->>'generation')::numeric <= 9223372036854775807"; + +/// Import of the opaque relay state kept by 0.13.0 (`m_0057`). +/// +/// `furumusic__fed_podcast_relay` holds the contract registers as JSON, +/// already validated and merged with the `music_dht::podcast` rules when they +/// were received, so they are copied as they are. The catalog rows the store +/// would have created from the episode references of received operations +/// (`PodcastEpisodeRef`, `source = 'sync'`) are rebuilt from the operation +/// log. Every statement is idempotent (`ON CONFLICT DO NOTHING`); the relay +/// table is kept (see `docs/administration.md`). +pub(crate) const IMPORT_PODCAST_RELAY: &[&str] = &[ + // Catalog rows from the subscription registers. + "INSERT INTO furumusic__podcast + (user_id, podcast_id, feed_url, title, author, description, link, image_url, + meta_from_feed, created_at_ms, updated_at_ms) + SELECT user_id, record_key, + record_json->'podcast'->>'feed_url', record_json->'podcast'->>'title', + record_json->'podcast'->>'author', record_json->'podcast'->>'description', + record_json->'podcast'->>'link', record_json->'podcast'->>'image_url', + FALSE, updated_at_ms, updated_at_ms + FROM furumusic__fed_podcast_relay + WHERE section = 'subscription' AND {user_filter} + ON CONFLICT DO NOTHING", + "INSERT INTO furumusic__podcast_subscription + (user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id) + SELECT user_id, record_key, (record_json->>'subscribed')::boolean, + record_json->'podcast', (record_json->>'generation')::bigint, + (record_json->>'hlc_ms')::bigint, record_json->>'op_id' + FROM furumusic__fed_podcast_relay + WHERE section = 'subscription' AND {generation_fits} AND {user_filter} + ON CONFLICT DO NOTHING", + // The relayed subscription already carries the bounded, merged alias set. + "INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url) + SELECT user_id, record_key, jsonb_array_elements_text(record_json->'feed_aliases') + FROM furumusic__fed_podcast_relay + WHERE section = 'subscription' AND {user_filter} + AND jsonb_typeof(record_json->'feed_aliases') = 'array' + ON CONFLICT DO NOTHING", + // Aliases received before any subscription register of their podcast. + "INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url) + SELECT user_id, record_key, jsonb_array_elements_text(record_json) + FROM furumusic__fed_podcast_relay + WHERE section = 'aliases' AND {user_filter} AND jsonb_typeof(record_json) = 'array' + ON CONFLICT DO NOTHING", + "INSERT INTO furumusic__podcast_share + (user_id, podcast_id, shared, generation, hlc_ms, op_id) + SELECT user_id, record_key, (record_json->>'shared')::boolean, + (record_json->>'generation')::bigint, (record_json->>'hlc_ms')::bigint, + record_json->>'op_id' + FROM furumusic__fed_podcast_relay + WHERE section = 'share' AND {generation_fits} AND {user_filter} + ON CONFLICT DO NOTHING", + "INSERT INTO furumusic__podcast_progress + (user_id, episode_id, podcast_id, position_ms, status, generation, session_id, + hlc_ms, device_id, op_id) + SELECT user_id, record_key, record_json->>'podcast_id', + (record_json->>'position_ms')::bigint, record_json->>'status', + (record_json->>'generation')::bigint, record_json->>'session_id', + (record_json->>'hlc_ms')::bigint, record_json->>'device_id', + record_json->>'op_id' + FROM furumusic__fed_podcast_relay + WHERE section = 'progress' AND {generation_fits} AND {user_filter} + ON CONFLICT DO NOTHING", + "INSERT INTO furumusic__podcast_listen_session + (user_id, session_id, episode_id, podcast_id, device_id, started_at_ms, + ended_at_ms, start_position_ms, end_position_ms, listened_ms, completed) + SELECT user_id, record_key, record_json->>'episode_id', record_json->>'podcast_id', + record_json->>'device_id', (record_json->>'started_at_ms')::bigint, + (record_json->>'ended_at_ms')::bigint, + (record_json->>'start_position_ms')::bigint, + (record_json->>'end_position_ms')::bigint, + (record_json->>'listened_ms')::bigint, (record_json->>'completed')::boolean + FROM furumusic__fed_podcast_relay + WHERE section = 'session' AND {user_filter} + ON CONFLICT DO NOTHING", + // Catalog rows for podcasts only known from episode references. + "INSERT INTO furumusic__podcast + (user_id, podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms) + SELECT DISTINCT ON (user_id, payload_json->'episode'->>'podcast_id') + user_id, payload_json->'episode'->>'podcast_id', + payload_json->'episode'->>'feed_url', payload_json->'episode'->>'podcast_title', + FALSE, received_at_ms, received_at_ms + FROM furumusic__fed_sync_ops + WHERE kind IN ('podcast_progress_set', 'podcast_listen_recorded') + AND jsonb_typeof(payload_json->'episode') = 'object' AND {user_filter} + ORDER BY user_id, payload_json->'episode'->>'podcast_id', hlc_ms DESC, op_id DESC + ON CONFLICT DO NOTHING", + // Episodes only known from references (the newest reference wins, like + // the first one would have for the store: feed metadata replaces it on + // the first refresh anyway). + "INSERT INTO furumusic__podcast_episode + (user_id, episode_id, podcast_id, title, season, episode_number, episode_type, + published_at_ms, duration_ms, audio_url, mime, image_url, source, in_feed, + first_seen_at_ms, updated_at_ms) + SELECT DISTINCT ON (user_id, payload_json->'episode'->>'episode_id') + user_id, payload_json->'episode'->>'episode_id', + payload_json->'episode'->>'podcast_id', payload_json->'episode'->>'episode_title', + (payload_json->'episode'->>'season')::bigint, + (payload_json->'episode'->>'episode_number')::bigint, 'full', + (payload_json->'episode'->>'published_at_ms')::bigint, + (payload_json->'episode'->>'duration_ms')::bigint, + payload_json->'episode'->>'audio_url', payload_json->'episode'->>'mime', + payload_json->'episode'->>'image_url', 'sync', FALSE, + received_at_ms, received_at_ms + FROM furumusic__fed_sync_ops + WHERE kind IN ('podcast_progress_set', 'podcast_listen_recorded') + AND jsonb_typeof(payload_json->'episode') = 'object' AND {user_filter} + ORDER BY user_id, payload_json->'episode'->>'episode_id', hlc_ms DESC, op_id DESC + ON CONFLICT DO NOTHING", +]; + +/// The import statements with `filter` as the account condition. +pub(crate) fn import_statements(filter: &str) -> Vec { + IMPORT_PODCAST_RELAY + .iter() + .map(|sql| { + sql.replace("{generation_fits}", GENERATION_FITS) + .replace(USER_FILTER, filter) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn import_statements_fill_every_placeholder() { + for sql in import_statements("user_id = 7") { + assert!(!sql.contains('{'), "{sql}"); + assert!(sql.contains("user_id = 7"), "{sql}"); + assert!(sql.contains("ON CONFLICT DO NOTHING"), "{sql}"); + } + } + + #[test] + fn every_store_table_is_dropped_backwards() { + for sql in CREATE_PODCAST_STORE { + let Some(rest) = sql + .strip_prefix("CREATE TABLE IF NOT EXISTS ") + .or_else(|| sql.strip_prefix("CREATE OR REPLACE VIEW ")) + else { + continue; + }; + let name = rest.split_whitespace().next().unwrap(); + assert!( + DROP_PODCAST_STORE + .iter() + .any(|drop| drop.ends_with(&format!(" {name}"))), + "{name}" + ); + } + } +} diff --git a/src/podcasts/tests.rs b/src/podcasts/tests.rs new file mode 100644 index 0000000..db90aa0 --- /dev/null +++ b/src/podcasts/tests.rs @@ -0,0 +1,81 @@ +//! Pure unit tests of the podcast store helpers (no database). + +use music_dht::podcast::ProgressStatus; + +use super::*; + +#[test] +fn search_text_never_matches_across_title_and_description() { + let norm = episode_search_norm("Hello, World!", Some("WORLD peace")); + assert_eq!(norm, "hello world\nworld peace"); + // A needle is itself normalized, so it never contains the separator. + let needle = music_dht::normalize_name("world world"); + assert!(!norm.contains(&needle)); + assert!(norm.contains(&music_dht::normalize_name("WORLD PEACE"))); + assert_eq!(episode_search_norm("Only", None), "only\n"); +} + +#[test] +fn progress_status_round_trips_through_sql() { + for status in [ + ProgressStatus::NotStarted, + ProgressStatus::InProgress, + ProgressStatus::Completed, + ] { + assert_eq!(status_from_sql(status_to_sql(status)), status); + } + // The CHECK constraint admits exactly these values. + for sql in schema::CREATE_PODCAST_STORE { + if sql.contains("CREATE TABLE IF NOT EXISTS furumusic__podcast_progress (") { + for value in ["not_started", "in_progress", "completed"] { + assert!(sql.contains(&format!("'{value}'"))); + } + } + } +} + +#[test] +fn generations_beyond_bigint_are_rejected() { + assert_eq!(generation_to_sql(0).unwrap(), 0); + assert_eq!(generation_to_sql(i64::MAX as u64).unwrap(), i64::MAX); + assert!(matches!( + generation_to_sql(i64::MAX as u64 + 1), + Err(PodcastStoreError::Invalid(_)) + )); + assert_eq!(generation_from_sql(-1), 0); +} + +#[test] +fn contract_numbers_outside_u32_are_dropped() { + assert_eq!(u32_from_sql(Some(7)), Some(7)); + assert_eq!(u32_from_sql(Some(-1)), None); + assert_eq!(u32_from_sql(Some(i64::from(u32::MAX) + 1)), None); + assert_eq!(u32_from_sql(None), None); +} + +#[test] +fn progress_columns_are_prefixed() { + assert_eq!( + progress_columns("g", "g_"), + "g.episode_id AS g_episode_id, g.podcast_id AS g_podcast_id, \ + g.position_ms AS g_position_ms, g.status AS g_status, \ + g.generation AS g_generation, g.session_id AS g_session_id, \ + g.hlc_ms AS g_hlc_ms, g.device_id AS g_device_id, g.op_id AS g_op_id" + ); +} + +#[test] +fn queries_sort_undated_and_unnumbered_episodes_last() { + for sort in [ + EpisodeSort::NewestFirst, + EpisodeSort::OldestFirst, + EpisodeSort::EpisodeNumber, + EpisodeSort::EpisodeNumberDesc, + ] { + let order = queries::order_by_for_tests(sort); + assert!( + order.starts_with("e.published_at_ms IS NULL") || order.starts_with("e.season IS NULL") + ); + assert!(order.ends_with("e.episode_id"), "{order}"); + } +}