Add a PostgreSQL podcast store with relay import

PodcastStore implements the music_dht::podcast contract per account:
catalog, feed aliases, episodes with a derived season view, the
subscription, share and progress registers, listening sessions and the
local-only fetch state. Received operations, snapshot records and local
actions share one apply path (materializers plus merge functions) under
a per-account advisory lock. Local actions append their operation to the
device-sync log with the next stamp of the web device and apply exactly
that operation in the same transaction.

Migration m_0056 creates the tables; m_0057 imports the registers that
0.13 kept in furumusic__fed_podcast_relay and the catalog rows of the
episode references in the operation log. The relay table is kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent cc1705457e
commit 7f07cb206a
11 files changed
+4914 -31

No files matched your search

+85 -31
View File
@@ -1782,20 +1782,7 @@ async fn record_local_op(
);
}
let now = now_ms();
let row = sqlx::query(
"UPDATE furumusic__fed_device_identity
SET local_seq = local_seq + 1,
last_hlc_ms = GREATEST($2, last_hlc_ms + 1)
WHERE user_id = $1
RETURNING local_seq, last_hlc_ms",
)
.bind(user_id)
.bind(now)
.fetch_one(pool)
.await?;
let seq: i64 = row.get("local_seq");
let hlc_ms: i64 = row.get("last_hlc_ms");
let (seq, hlc_ms) = reserve_local_stamp(pool, user_id).await?;
let op_id = format!("{}:{seq}", identity.device_id);
let op = SyncOpWire {
op_id,
@@ -1805,6 +1792,40 @@ async fn record_local_op(
payload,
};
insert_op(pool, user_id, &op).await?;
advance_vector(pool, user_id, &op.origin_device_id, op.seq).await?;
let _ = apply_op(pool, Arc::new(PlayerDeviceHub::default()), user_id, &op).await?;
Ok(())
}
/// Reserves the next local `(seq, hlc_ms)` of the account's web device.
///
/// `hlc_ms` is the wall clock, but always past the previous local
/// operation. Inside a transaction the identity row stays locked until it
/// ends, so local operations of one account are stamped in commit order.
async fn reserve_local_stamp<'e>(
db: impl sqlx::PgExecutor<'e>,
user_id: i64,
) -> Result<(i64, i64)> {
let row = sqlx::query(
"UPDATE furumusic__fed_device_identity
SET local_seq = local_seq + 1,
last_hlc_ms = GREATEST($2, last_hlc_ms + 1)
WHERE user_id = $1
RETURNING local_seq, last_hlc_ms",
)
.bind(user_id)
.bind(now_ms())
.fetch_one(db)
.await?;
Ok((row.get("local_seq"), row.get("last_hlc_ms")))
}
async fn advance_vector<'e>(
db: impl sqlx::PgExecutor<'e>,
user_id: i64,
device_id: &str,
seq: i64,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__fed_sync_vector (user_id, device_id, max_seq)
VALUES ($1, $2, $3)
@@ -1812,14 +1833,53 @@ async fn record_local_op(
max_seq = GREATEST(furumusic__fed_sync_vector.max_seq, EXCLUDED.max_seq)",
)
.bind(user_id)
.bind(&op.origin_device_id)
.bind(op.seq)
.execute(pool)
.bind(device_id)
.bind(seq)
.execute(db)
.await?;
let _ = apply_op(pool, Arc::new(PlayerDeviceHub::default()), user_id, &op).await?;
Ok(())
}
/// Device id of the account's web device, creating the identity on first
/// use. Local podcast actions stamp their operations with it.
#[cfg_attr(not(test), allow(dead_code))] // podcast UI and RSS job: later commits
pub(crate) async fn local_device_id(pool: &sqlx::PgPool, user_id: i64) -> Result<String> {
Ok(ensure_identity(pool, user_id, "").await?.device_id)
}
/// Appends a local operation of `device_id` (the account's web device, see
/// [`local_device_id`]) to the sync log inside the caller's transaction.
///
/// The stamp (`seq`, `hlc_ms`, `op_id = <device>:<seq>`) is reserved here;
/// `make` receives the `op_id`. The caller applies the returned operation
/// itself in the same transaction, so the log and the materialized state
/// change together (the podcast store does this for its local actions).
#[cfg_attr(not(test), allow(dead_code))] // podcast UI and RSS job: later commits
pub(crate) async fn append_local_op(
conn: &mut sqlx::PgConnection,
user_id: i64,
device_id: &str,
make: impl FnOnce(&str) -> SyncOpPayload,
) -> Result<SyncOpWire> {
let (seq, hlc_ms) = reserve_local_stamp(&mut *conn, user_id).await?;
let op_id = format!("{device_id}:{seq}");
let payload = make(&op_id);
let op = SyncOpWire {
op_id,
origin_device_id: device_id.to_string(),
seq,
hlc_ms,
payload,
};
anyhow::ensure!(
insert_op(&mut *conn, user_id, &op).await?,
"local operation {} already exists",
op.op_id
);
advance_vector(&mut *conn, user_id, device_id, seq).await?;
Ok(op)
}
async fn apply_ops(
pool: &sqlx::PgPool,
hub: Arc<PlayerDeviceHub>,
@@ -1837,23 +1897,17 @@ async fn apply_ops(
} else if insert_op(pool, user_id, &op).await? {
let _ = apply_op(pool, Arc::clone(&hub), user_id, &op).await?;
}
sqlx::query(
"INSERT INTO furumusic__fed_sync_vector (user_id, device_id, max_seq)
VALUES ($1, $2, $3)
ON CONFLICT (user_id, device_id) DO UPDATE SET
max_seq = GREATEST(furumusic__fed_sync_vector.max_seq, EXCLUDED.max_seq)",
)
.bind(user_id)
.bind(&op.origin_device_id)
.bind(op.seq)
.execute(pool)
.await?;
advance_vector(pool, user_id, &op.origin_device_id, op.seq).await?;
}
Ok(())
}
/// Appends an operation to the log; `false` when it was already stored.
async fn insert_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire) -> Result<bool> {
async fn insert_op<'e>(
db: impl sqlx::PgExecutor<'e>,
user_id: i64,
op: &SyncOpWire,
) -> Result<bool> {
let inserted = sqlx::query(
"INSERT INTO furumusic__fed_sync_ops
(user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms,
@@ -1870,7 +1924,7 @@ async fn insert_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire) -> Result
.bind(op.hlc_ms)
.bind(now_ms())
.bind(op.payload.is_tombstone())
.execute(pool)
.execute(db)
.await?
.rows_affected();
Ok(inserted > 0)
+1
View File
@@ -15,6 +15,7 @@ mod metrics;
mod music;
mod oidc;
mod player;
mod podcasts;
mod scheduler;
mod similarity;
mod torrents;
+72
View File
@@ -2942,6 +2942,76 @@ pub mod db_migrations {
.build()];
}
// -- M0056: PostgreSQL podcast store (device-sync protocol v4) ---------
// Per-account podcast catalog, synced registers and local fetch state;
// see `crate::podcasts` and `crate::podcasts::schema`.
#[cot::db::migrations::migration_op]
async fn create_podcast_store(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::CREATE_PODCAST_STORE {
ctx.db.raw(sql).await?;
}
Ok(())
}
#[cot::db::migrations::migration_op]
async fn drop_podcast_store(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::DROP_PODCAST_STORE {
ctx.db.raw(sql).await?;
}
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0056CreatePodcastStore;
impl migrations::Migration for M0056CreatePodcastStore {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0056_create_podcast_store";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0055_create_fed_podcast_relay",
)];
const OPERATIONS: &'static [Operation] = &[Operation::custom(create_podcast_store)
.backwards(drop_podcast_store)
.build()];
}
// -- M0057: import the 0.13.0 podcast relay state into the store --------
// The relay table stays (unused) until a later cleanup release, so a
// rollback to 0.13.0 still finds the state it relayed.
#[cot::db::migrations::migration_op]
async fn import_podcast_relay(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::import_statements("TRUE") {
ctx.db.raw(&sql).await?;
}
Ok(())
}
#[cot::db::migrations::migration_op]
async fn keep_podcast_store_rows(
_ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
// The imported rows go with the tables (`m_0056` backwards).
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0057ImportPodcastRelay;
impl migrations::Migration for M0057ImportPodcastRelay {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0057_import_podcast_relay";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0056_create_podcast_store",
)];
const OPERATIONS: &'static [Operation] = &[Operation::custom(import_podcast_relay)
.backwards(keep_podcast_store_rows)
.build()];
}
pub const MIGRATIONS: &[&SyncDynMigration] = &[
&M0006CreateMediaFile,
&M0007CreateArtist,
@@ -2987,5 +3057,7 @@ pub mod db_migrations {
&M0052AddImportPlaylistDestinations,
&M0054AddTypedPlaybackQueue,
&M0055CreateFedPodcastRelay,
&M0056CreatePodcastStore,
&M0057ImportPodcastRelay,
];
}
+594
View File
@@ -0,0 +1,594 @@
//! Local user actions: they build the device-sync operation, append it to
//! the account's sync log with its stamp and apply it through the shared
//! code path, all in one transaction.
use anyhow::Result;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::{
EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord,
ProgressStatus, effective_share, merge_feed_aliases, merge_listen_session,
next_podcast_generation, resolve_podcast_id,
};
use sqlx::{PgConnection, Row};
use super::apply::{self, ApplyOutcome};
use super::{PodcastStore, PodcastStoreError, invalid};
use crate::federation::devices;
/// Result of [`PodcastStore::subscribe`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct SubscribeOutcome {
/// Podcast the URL resolved to (an existing one when the URL is a known
/// feed URL or alias).
pub(crate) podcast_id: PodcastId,
/// The podcast was already subscribed; no operation was emitted.
pub(crate) already_subscribed: bool,
/// Operations appended to the sync log (and applied).
pub(crate) ops: Vec<SyncOpWire>,
}
/// A listening session of this account as seen by the progress register.
///
/// Returned by explicit actions; periodic checkpoints of the session use its
/// `generation` ([`PodcastStore::checkpoint`]).
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ProgressSession {
/// Episode being played.
pub(crate) episode: PodcastEpisodeRef,
/// Session id (also the `ListenSession::session_id`).
pub(crate) session_id: String,
/// Generation of the session's latest explicit action.
pub(crate) generation: u64,
}
/// A progress register written by a local action.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ProgressWrite {
/// The register as written.
pub(crate) record: ProgressRecord,
/// The session after the action (its generation for later checkpoints).
pub(crate) session: ProgressSession,
/// Operation appended to the sync log.
pub(crate) op: SyncOpWire,
}
/// Result of [`PodcastStore::checkpoint`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum CheckpointOutcome {
/// The checkpoint was written.
Written(Box<ProgressWrite>),
/// The register holds a greater generation (another device or an
/// explicit action took over). Nothing was written: stop writing at the
/// old generation, or continue locally with an explicit action
/// ([`PodcastStore::start_session`] / [`PodcastStore::seek`]).
Superseded {
/// Current register.
current: ProgressRecord,
},
/// This session already completed the episode at its generation; a
/// checkpoint would turn it back into "in progress". Nothing was written.
AlreadyCompleted {
/// Current register.
current: ProgressRecord,
},
}
#[derive(Debug, Clone, Copy)]
enum ExplicitAction {
Start(i64),
Seek(i64),
Restart,
Complete(i64),
MarkPlayed,
MarkUnplayed,
}
/// Validates the payload (with a provisional stamp), appends it to the sync
/// log with the next stamp of `device_id` and applies exactly that operation
/// through [`apply::apply_op`]. `make` receives the operation id, which
/// explicit actions without a session use as their session id.
pub(crate) async fn emit(
conn: &mut PgConnection,
user_id: i64,
device_id: &str,
make: impl Fn(&str) -> SyncOpPayload,
) -> Result<SyncOpWire> {
SyncOpWire {
op_id: "provisional".to_string(),
origin_device_id: device_id.to_string(),
seq: 0,
hlc_ms: 0,
payload: make("provisional"),
}
.validate_podcast()
.map_err(invalid)?;
let op = devices::append_local_op(conn, user_id, device_id, &make).await?;
match apply::apply_op(conn, user_id, &op).await? {
ApplyOutcome::Rejected(reason) => Err(invalid(reason)),
_ => Ok(op),
}
}
/// Resolves a feed URL against every known URL of every podcast of the
/// account: the alias sets, the catalog fetch URLs and the registers'
/// `podcast.feed_url` (`resolve_podcast_id`).
pub(crate) async fn resolve_podcast(
conn: &mut PgConnection,
user_id: i64,
url: &FeedUrl,
) -> Result<PodcastId> {
let rows = sqlx::query(
"SELECT podcast_id FROM furumusic__podcast_feed_alias
WHERE user_id = $1 AND feed_url = $2
UNION SELECT podcast_id FROM furumusic__podcast
WHERE user_id = $1 AND feed_url = $2
UNION SELECT podcast_id FROM furumusic__podcast_subscription
WHERE user_id = $1 AND podcast_json->>'feed_url' = $2",
)
.bind(user_id)
.bind(url.as_str())
.fetch_all(&mut *conn)
.await?;
let ids = rows
.iter()
.map(|row| super::podcast_id_col(row, "podcast_id"))
.collect::<sqlx::Result<Vec<_>>>()?;
Ok(resolve_podcast_id(url, ids.iter().map(|id| (id, url))))
}
async fn catalog_meta(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastMeta>> {
let row = sqlx::query(
"SELECT feed_url, title, author, description, link, image_url
FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(None);
};
Ok(Some(PodcastMeta {
feed_url: super::feed_url_col(&row, "feed_url")?,
title: row.try_get("title")?,
author: row.try_get("author")?,
description: row.try_get("description")?,
link: row.try_get("link")?,
image_url: row.try_get("image_url")?,
}))
}
async fn share_payload(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
shared: bool,
) -> Result<SyncOpPayload> {
let subscription = apply::load_subscription_register(conn, user_id, podcast_id).await?;
let share = apply::load_share(conn, user_id, podcast_id).await?;
Ok(SyncOpPayload::PodcastShareSet {
podcast_id: podcast_id.clone(),
shared,
generation: next_podcast_generation(subscription.as_ref(), share.as_ref()),
})
}
/// The reference carried by operations: identity and display only.
fn op_ref(episode: &PodcastEpisodeRef) -> PodcastEpisodeRef {
let mut episode = episode.clone();
episode.progress_generation = None;
episode
}
impl PodcastStore {
/// Subscribes to the podcast of `podcast.feed_url` (explicit user action).
///
/// The URL is resolved against known feed URLs and aliases first, so
/// subscribing again, through a redirect target or from someone else's
/// card reuses the existing podcast with its progress and history. When
/// the podcast is already subscribed nothing is emitted. Otherwise one
/// `podcast_subscription_set` with generation
/// `next_podcast_generation(..)` and the known aliases is appended and
/// applied. The new subscription is private until
/// [`PodcastStore::set_shared`].
///
/// Pass the channel metadata of the fetched feed
/// (`ParsedFeed::to_podcast_meta(url)`), then ingest the episodes with
/// [`PodcastStore::apply_feed`].
pub(crate) async fn subscribe(
&self,
user_id: i64,
podcast: &PodcastMeta,
) -> Result<SubscribeOutcome> {
let meta = podcast.bounded();
meta.validate().map_err(invalid)?;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let podcast_id = resolve_podcast(&mut tx, user_id, &meta.feed_url).await?;
let current = apply::load_subscription_register(&mut tx, user_id, &podcast_id).await?;
if current.as_ref().is_some_and(|record| record.subscribed) {
return Ok(SubscribeOutcome {
podcast_id,
already_subscribed: true,
ops: Vec::new(),
});
}
let share = apply::load_share(&mut tx, user_id, &podcast_id).await?;
let generation = next_podcast_generation(current.as_ref(), share.as_ref());
let aliases = merge_feed_aliases(
&podcast_id,
&apply::load_aliases(&mut tx, user_id, &podcast_id).await?,
std::slice::from_ref(&meta.feed_url),
);
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast_id.clone(),
subscribed: true,
podcast: meta.clone(),
feed_aliases: aliases.clone(),
generation,
}
})
.await?;
tx.commit().await?;
Ok(SubscribeOutcome {
podcast_id,
already_subscribed: false,
ops: vec![op],
})
}
/// Unsubscribes (explicit user action).
///
/// Emits `podcast_subscription_set { subscribed: false }` followed by
/// `podcast_share_set { shared: false }`, each with the next podcast
/// generation. Progress, sessions and the catalog are kept, so
/// subscribing again continues where the user stopped. Does nothing when
/// the podcast is not subscribed.
pub(crate) async fn unsubscribe(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<SyncOpWire>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let Some(current) = apply::load_subscription_register(&mut tx, user_id, podcast_id).await?
else {
return Ok(Vec::new());
};
if !current.subscribed {
return Ok(Vec::new());
}
let share = apply::load_share(&mut tx, user_id, podcast_id).await?;
let generation = next_podcast_generation(Some(&current), share.as_ref());
let mut meta = catalog_meta(&mut tx, user_id, podcast_id)
.await?
.map(|meta| meta.bounded())
.unwrap_or_else(|| current.podcast.clone());
meta.feed_url = current.podcast.feed_url.clone();
let aliases = current.feed_aliases.clone();
let unsubscribe = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast_id.clone(),
subscribed: false,
podcast: meta.clone(),
feed_aliases: aliases.clone(),
generation,
}
})
.await?;
let revoke_payload = share_payload(&mut tx, user_id, podcast_id, false).await?;
let revoke = emit(&mut tx, user_id, &device_id, |_| revoke_payload.clone()).await?;
tx.commit().await?;
Ok(vec![unsubscribe, revoke])
}
/// Sets the decision to publish the podcast to the federation.
///
/// Sharing needs an active subscription and is a no-op when the podcast
/// is already effectively shared (`effective_share`). Revoking is a no-op
/// when no share decision says `true`. The write uses the next podcast
/// generation, so it orders after the subscription it observed.
pub(crate) async fn set_shared(
&self,
user_id: i64,
podcast_id: &PodcastId,
shared: bool,
) -> Result<Vec<SyncOpWire>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let subscription = apply::load_subscription_register(&mut tx, user_id, podcast_id).await?;
let share = apply::load_share(&mut tx, user_id, podcast_id).await?;
if shared {
if !subscription
.as_ref()
.is_some_and(|record| record.subscribed)
{
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
}
if effective_share(subscription.as_ref(), share.as_ref()) {
return Ok(Vec::new());
}
} else if !share.as_ref().is_some_and(|record| record.shared) {
return Ok(Vec::new());
}
let payload = share_payload(&mut tx, user_id, podcast_id, shared).await?;
let op = emit(&mut tx, user_id, &device_id, |_| payload.clone()).await?;
tx.commit().await?;
Ok(vec![op])
}
async fn explicit_progress(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
session_id: Option<&str>,
carried_generation: Option<u64>,
action: ExplicitAction,
) -> Result<ProgressWrite> {
episode.validate().map_err(invalid)?;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let current = apply::load_progress(&mut tx, user_id, &episode.episode_id).await?;
let max_seen = current
.as_ref()
.map(|record| record.generation)
.into_iter()
.chain(carried_generation)
.max();
let generation = ProgressRecord::explicit_generation(max_seen);
let current_position = current.as_ref().map_or(0, |record| record.position_ms);
let (position_ms, status) = match action {
ExplicitAction::Start(position) | ExplicitAction::Seek(position) => {
(position, ProgressStatus::InProgress)
}
ExplicitAction::Restart => (0, ProgressStatus::InProgress),
ExplicitAction::Complete(position) => (position, ProgressStatus::Completed),
ExplicitAction::MarkPlayed => (current_position, ProgressStatus::Completed),
ExplicitAction::MarkUnplayed => (0, ProgressStatus::NotStarted),
};
let reference = op_ref(episode);
let op = emit(&mut tx, user_id, &device_id, |op_id| {
SyncOpPayload::PodcastProgressSet {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms,
status,
generation,
session_id: session_id.unwrap_or(op_id).to_string(),
episode: Some(reference.clone()),
}
})
.await?;
tx.commit().await?;
let record = op.podcast_progress().expect("progress payload");
Ok(ProgressWrite {
session: ProgressSession {
episode: episode.clone(),
session_id: record.session_id.clone(),
generation,
},
record,
op,
})
}
/// Starts a listening session at `position_ms` (explicit action).
///
/// The generation is `explicit_generation(max(local, carried_generation))`;
/// pass `PodcastEpisodeRef::progress_generation` of a queue item received
/// through a handoff as `carried_generation`. Ordinary Play resumes from
/// [`PodcastStore::resume_position_ms`].
pub(crate) async fn start_session(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
session_id: &str,
position_ms: i64,
carried_generation: Option<u64>,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
episode,
Some(session_id),
carried_generation,
ExplicitAction::Start(position_ms),
)
.await
}
/// Periodic checkpoint (or pause/stop save) of a running session.
///
/// Keeps the session generation. When the register already holds a
/// greater generation nothing is written ([`CheckpointOutcome::Superseded`]).
pub(crate) async fn checkpoint(
&self,
user_id: i64,
session: &ProgressSession,
position_ms: i64,
) -> Result<CheckpointOutcome> {
let episode = &session.episode;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
if let Some(current) = apply::load_progress(&mut tx, user_id, &episode.episode_id).await? {
if current.generation > session.generation {
return Ok(CheckpointOutcome::Superseded { current });
}
if current.generation == session.generation
&& current.session_id == session.session_id
&& current.status == ProgressStatus::Completed
{
return Ok(CheckpointOutcome::AlreadyCompleted { current });
}
}
let reference = op_ref(episode);
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastProgressSet {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms,
status: ProgressStatus::InProgress,
generation: session.generation,
session_id: session.session_id.clone(),
episode: Some(reference.clone()),
}
})
.await?;
tx.commit().await?;
let record = op.podcast_progress().expect("progress payload");
Ok(CheckpointOutcome::Written(Box::new(ProgressWrite {
record,
session: session.clone(),
op,
})))
}
/// User seek during a session (explicit action; the returned session
/// carries the new generation for later checkpoints).
pub(crate) async fn seek(
&self,
user_id: i64,
session: &ProgressSession,
position_ms: i64,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
&session.episode,
Some(&session.session_id),
Some(session.generation),
ExplicitAction::Seek(position_ms),
)
.await
}
/// The session reached the end (`reaches_completion` or end of stream):
/// writes `completed` with an explicit generation. Record the session
/// itself with [`PodcastStore::record_listen_session`].
pub(crate) async fn complete_session(
&self,
user_id: i64,
session: &ProgressSession,
position_ms: i64,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
&session.episode,
Some(&session.session_id),
Some(session.generation),
ExplicitAction::Complete(position_ms),
)
.await
}
/// "Start over": position 0, in progress, explicit generation.
pub(crate) async fn restart(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
session_id: &str,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
episode,
Some(session_id),
None,
ExplicitAction::Restart,
)
.await
}
/// "Mark played": status completed, explicit generation. Creates no
/// listening session and invents no listening time.
pub(crate) async fn mark_played(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<ProgressWrite> {
self.explicit_progress(user_id, episode, None, None, ExplicitAction::MarkPlayed)
.await
}
/// "Mark unplayed": status not started at position 0, explicit generation.
pub(crate) async fn mark_unplayed(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<ProgressWrite> {
self.explicit_progress(user_id, episode, None, None, ExplicitAction::MarkUnplayed)
.await
}
/// Records (or grows) a listening session of this account's device.
///
/// Sessions below the history threshold (`ListenSession::should_record`)
/// and versions that do not grow the stored session emit nothing.
/// `session.device_id` must be the account's web device id. The episode
/// reference defaults to the catalog's.
pub(crate) async fn record_listen_session(
&self,
user_id: i64,
session: &ListenSession,
episode: Option<&PodcastEpisodeRef>,
) -> Result<Option<SyncOpWire>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
if session.device_id != device_id {
return Err(invalid(
"session device_id differs from the local device id",
));
}
session.validate().map_err(invalid)?;
if !session.should_record() {
return Ok(None);
}
let mut tx = self.begin_write(user_id).await?;
if let Some(current) = apply::load_session(&mut tx, user_id, &session.session_id).await? {
let merged = merge_listen_session(&current, session).map_err(invalid)?;
if merged == current {
return Ok(None);
}
}
let reference = match episode {
Some(episode) => Some(op_ref(episode)),
None => super::queries::episode_ref_in(&mut tx, user_id, &session.episode_id)
.await?
.map(|reference| op_ref(&reference)),
};
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastListenRecorded {
session: session.clone(),
episode: reference.clone(),
}
})
.await?;
tx.commit().await?;
Ok(Some(op))
}
/// Resolves a feed URL to a podcast id: the existing podcast when the
/// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise
/// the id a first subscription through this URL would derive.
pub(crate) async fn resolve_podcast_id(
&self,
user_id: i64,
url: &FeedUrl,
) -> Result<PodcastId> {
let mut conn = self.pool.acquire().await?;
resolve_podcast(&mut conn, user_id, url).await
}
/// The episode reference for queueing or progress actions (with
/// `progress_generation` set to the stored register's generation), or
/// `None` when the episode is unknown.
pub(crate) async fn episode_ref(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<PodcastEpisodeRef>> {
let mut conn = self.pool.acquire().await?;
super::queries::episode_ref_in(&mut conn, user_id, episode_id).await
}
}
+868
View File
@@ -0,0 +1,868 @@
//! The single code path that applies podcast records to the store.
//!
//! Remote operations, local actions and snapshot sections all end here.
//! Records come from the `SyncOpWire` materializers of
//! `music_dht::device_sync` and are merged with the functions of
//! `music_dht::podcast`, so applying is idempotent and order-independent.
use anyhow::Result;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot};
use music_dht::podcast::{
EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode,
ListenSession, PodcastEpisodeRef, PodcastError, PodcastId, ProgressRecord, ShareRecord,
SubscriptionRecord, merge_feed_aliases, merge_listen_session,
};
use sqlx::{PgConnection, Row};
use super::{
PodcastStore, PodcastStoreError, SESSION_COLUMNS, episode_id_col, episode_search_norm,
feed_url_col, generation_from_sql, generation_to_sql, invalid, now_ms, podcast_meta_col,
progress_columns, progress_from_row, session_from_row, status_to_sql,
};
/// Result of applying one operation.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ApplyOutcome {
/// Stored state changed.
Changed,
/// The operation was already reflected (duplicate, or older than the
/// stored register).
Unchanged,
/// Not a podcast operation; nothing to do here.
Ignored,
/// The operation violates the contract and was skipped. Acknowledge it
/// anyway so the log keeps moving (`PODCASTS.md`).
Rejected(String),
}
/// Counters returned by [`PodcastStore::apply_snapshot`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) struct SnapshotApplyStats {
/// Records that changed stored state.
pub(crate) changed: usize,
/// Valid records already reflected locally.
pub(crate) unchanged: usize,
/// Records that failed validation and were skipped.
pub(crate) rejected: usize,
/// Whether the podcast sections were ignored for exceeding the counts.
pub(crate) ignored_over_limits: bool,
}
/// Where an episode row came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum EpisodeSource {
/// Parsed from the podcast's feed by this account.
Feed,
/// Learned from a `PodcastEpisodeRef` (sync operation or queue item).
Sync,
}
impl EpisodeSource {
fn as_sql(self) -> &'static str {
match self {
Self::Feed => "feed",
Self::Sync => "sync",
}
}
}
pub(crate) async fn load_subscription_register(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<SubscriptionRecord>> {
let row = sqlx::query(
"SELECT subscribed, podcast_json, generation, hlc_ms, op_id
FROM furumusic__podcast_subscription WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(None);
};
let record = SubscriptionRecord {
podcast_id: podcast_id.clone(),
subscribed: row.try_get("subscribed")?,
podcast: podcast_meta_col(&row, "podcast_json")?,
feed_aliases: load_aliases(conn, user_id, podcast_id).await?,
generation: generation_from_sql(row.try_get("generation")?),
hlc_ms: row.try_get("hlc_ms")?,
op_id: row.try_get("op_id")?,
};
Ok(Some(record))
}
pub(crate) async fn load_aliases(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<FeedUrl>> {
let rows = sqlx::query(
"SELECT feed_url FROM furumusic__podcast_feed_alias
WHERE user_id = $1 AND podcast_id = $2 ORDER BY feed_url",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&mut *conn)
.await?;
Ok(rows
.iter()
.map(|row| feed_url_col(row, "feed_url"))
.collect::<sqlx::Result<Vec<_>>>()?)
}
pub(crate) fn share_from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result<ShareRecord> {
Ok(ShareRecord {
podcast_id: super::podcast_id_col(row, "podcast_id")?,
shared: row.try_get("shared")?,
generation: generation_from_sql(row.try_get("generation")?),
hlc_ms: row.try_get("hlc_ms")?,
op_id: row.try_get("op_id")?,
})
}
pub(crate) async fn load_share(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<ShareRecord>> {
let row = sqlx::query(
"SELECT podcast_id, shared, generation, hlc_ms, op_id
FROM furumusic__podcast_share WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row.as_ref().map(share_from_row).transpose()?)
}
pub(crate) async fn load_progress(
conn: &mut PgConnection,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<ProgressRecord>> {
let row = sqlx::query(&format!(
"SELECT {} FROM furumusic__podcast_progress p
WHERE p.user_id = $1 AND p.episode_id = $2",
progress_columns("p", "")
))
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row
.as_ref()
.map(|row| progress_from_row(row, ""))
.transpose()?)
}
pub(crate) async fn load_session(
conn: &mut PgConnection,
user_id: i64,
session_id: &str,
) -> Result<Option<ListenSession>> {
let row = sqlx::query(&format!(
"SELECT {SESSION_COLUMNS} FROM furumusic__podcast_listen_session s
WHERE s.user_id = $1 AND s.session_id = $2"
))
.bind(user_id)
.bind(session_id)
.fetch_optional(&mut *conn)
.await?;
Ok(row.as_ref().map(session_from_row).transpose()?)
}
/// Replaces the alias set of a podcast with `merged` when it differs.
async fn store_aliases(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
merged: &[FeedUrl],
) -> Result<bool> {
let current = load_aliases(conn, user_id, podcast_id).await?;
if current == merged {
return Ok(false);
}
sqlx::query("DELETE FROM furumusic__podcast_feed_alias WHERE user_id = $1 AND podcast_id = $2")
.bind(user_id)
.bind(podcast_id.as_str())
.execute(&mut *conn)
.await?;
let urls: Vec<&str> = merged.iter().map(FeedUrl::as_str).collect();
sqlx::query(
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
SELECT $1, $2, url FROM UNNEST($3::text[]) AS url",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(&urls)
.execute(&mut *conn)
.await?;
Ok(true)
}
/// Merges feed URLs into the bounded grow-only alias set of a podcast.
///
/// Returns whether the stored set changed.
pub(crate) async fn merge_aliases(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
urls: &[FeedUrl],
) -> Result<bool> {
let current = load_aliases(conn, user_id, podcast_id).await?;
let merged = merge_feed_aliases(podcast_id, &current, urls);
store_aliases(conn, user_id, podcast_id, &merged).await
}
/// Inserts the catalog row of a podcast if it does not exist yet.
pub(crate) async fn ensure_podcast_row(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
feed_url: &FeedUrl,
title: &str,
now: i64,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__podcast
(user_id, podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms)
VALUES ($1, $2, $3, $4, FALSE, $5, $5)
ON CONFLICT (user_id, podcast_id) DO NOTHING",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(feed_url.as_str())
.bind(title)
.bind(now)
.execute(&mut *conn)
.await?;
Ok(())
}
/// Applies a subscription register record (operation or snapshot).
///
/// The register is last-writer-wins ([`SubscriptionRecord::supersedes`]);
/// the alias set is a bounded grow-only set merged independently, so the
/// stored aliases are the bounded union of every URL seen (what
/// `merge_subscription` computes). The catalog row is created from the
/// register metadata and follows the winning register until this account
/// has refreshed the feed itself.
pub(crate) async fn apply_subscription_record(
conn: &mut PgConnection,
user_id: i64,
record: &SubscriptionRecord,
now: i64,
) -> Result<bool> {
let generation = generation_to_sql(record.generation)?;
let podcast_id = &record.podcast_id;
let current = load_subscription_register(conn, user_id, podcast_id).await?;
let wins = current
.as_ref()
.is_none_or(|current| record.supersedes(current));
let mut changed = false;
if wins {
sqlx::query(
"INSERT INTO furumusic__podcast_subscription
(user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
subscribed = EXCLUDED.subscribed, podcast_json = EXCLUDED.podcast_json,
generation = EXCLUDED.generation, hlc_ms = EXCLUDED.hlc_ms,
op_id = EXCLUDED.op_id",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(record.subscribed)
.bind(serde_json::to_value(&record.podcast)?)
.bind(generation)
.bind(record.hlc_ms)
.bind(&record.op_id)
.execute(&mut *conn)
.await?;
changed = true;
}
let mut urls = record.feed_aliases.clone();
urls.push(record.podcast.feed_url.clone());
changed |= merge_aliases(conn, user_id, podcast_id, &urls).await?;
ensure_podcast_row(
conn,
user_id,
podcast_id,
&record.podcast.feed_url,
&record.podcast.title,
now,
)
.await?;
if wins {
let meta = &record.podcast;
sqlx::query(
"UPDATE furumusic__podcast
SET feed_url = $3, title = $4, author = $5,
description = COALESCE($6, description), link = $7, image_url = $8,
updated_at_ms = $9
WHERE user_id = $1 AND podcast_id = $2 AND NOT meta_from_feed",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(meta.feed_url.as_str())
.bind(&meta.title)
.bind(&meta.author)
.bind(&meta.description)
.bind(&meta.link)
.bind(&meta.image_url)
.bind(now)
.execute(&mut *conn)
.await?;
}
Ok(changed)
}
/// Applies a share register record.
pub(crate) async fn apply_share_record(
conn: &mut PgConnection,
user_id: i64,
record: &ShareRecord,
) -> Result<bool> {
let generation = generation_to_sql(record.generation)?;
let current = load_share(conn, user_id, &record.podcast_id).await?;
if current
.as_ref()
.is_some_and(|current| !record.supersedes(current))
{
return Ok(false);
}
sqlx::query(
"INSERT INTO furumusic__podcast_share
(user_id, podcast_id, shared, generation, hlc_ms, op_id)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
shared = EXCLUDED.shared, generation = EXCLUDED.generation,
hlc_ms = EXCLUDED.hlc_ms, op_id = EXCLUDED.op_id",
)
.bind(user_id)
.bind(record.podcast_id.as_str())
.bind(record.shared)
.bind(generation)
.bind(record.hlc_ms)
.bind(&record.op_id)
.execute(&mut *conn)
.await?;
Ok(true)
}
/// Applies a progress register record. Never touches the catalog.
pub(crate) async fn apply_progress_record(
conn: &mut PgConnection,
user_id: i64,
record: &ProgressRecord,
) -> Result<bool> {
let generation = generation_to_sql(record.generation)?;
let current = load_progress(conn, user_id, &record.episode_id).await?;
if current
.as_ref()
.is_some_and(|current| !record.supersedes(current))
{
return Ok(false);
}
sqlx::query(
"INSERT INTO furumusic__podcast_progress
(user_id, episode_id, podcast_id, position_ms, status, generation, session_id,
hlc_ms, device_id, op_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (user_id, episode_id) DO UPDATE SET
podcast_id = EXCLUDED.podcast_id, position_ms = EXCLUDED.position_ms,
status = EXCLUDED.status, generation = EXCLUDED.generation,
session_id = EXCLUDED.session_id, hlc_ms = EXCLUDED.hlc_ms,
device_id = EXCLUDED.device_id, op_id = EXCLUDED.op_id",
)
.bind(user_id)
.bind(record.episode_id.as_str())
.bind(record.podcast_id.as_str())
.bind(record.position_ms)
.bind(status_to_sql(record.status))
.bind(generation)
.bind(&record.session_id)
.bind(record.hlc_ms)
.bind(&record.device_id)
.bind(&record.op_id)
.execute(&mut *conn)
.await?;
Ok(true)
}
/// Adds or grows a listening session (grow-only set keyed by `session_id`).
pub(crate) async fn apply_listen_session(
conn: &mut PgConnection,
user_id: i64,
session: &ListenSession,
) -> Result<bool> {
let current = load_session(conn, user_id, &session.session_id).await?;
let merged = match &current {
Some(current) => merge_listen_session(current, session).map_err(invalid)?,
None => session.clone(),
};
if current.as_ref() == Some(&merged) {
return Ok(false);
}
sqlx::query(
"INSERT INTO furumusic__podcast_listen_session
(user_id, session_id, episode_id, podcast_id, device_id, started_at_ms,
ended_at_ms, start_position_ms, end_position_ms, listened_ms, completed)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (user_id, session_id) DO UPDATE SET
started_at_ms = EXCLUDED.started_at_ms, ended_at_ms = EXCLUDED.ended_at_ms,
start_position_ms = EXCLUDED.start_position_ms,
end_position_ms = EXCLUDED.end_position_ms,
listened_ms = EXCLUDED.listened_ms, completed = EXCLUDED.completed",
)
.bind(user_id)
.bind(&merged.session_id)
.bind(merged.episode_id.as_str())
.bind(merged.podcast_id.as_str())
.bind(&merged.device_id)
.bind(merged.started_at_ms)
.bind(merged.ended_at_ms)
.bind(merged.start_position_ms)
.bind(merged.end_position_ms)
.bind(merged.listened_ms)
.bind(merged.completed)
.execute(&mut *conn)
.await?;
Ok(true)
}
pub(crate) fn episode_type_to_sql(kind: EpisodeType) -> &'static str {
match kind {
EpisodeType::Full => "full",
EpisodeType::Trailer => "trailer",
EpisodeType::Bonus => "bonus",
}
}
/// Inserts an episode row.
pub(crate) async fn insert_episode(
conn: &mut PgConnection,
user_id: i64,
meta: &EpisodeMeta,
source: EpisodeSource,
in_feed: bool,
now: i64,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__podcast_episode
(user_id, episode_id, podcast_id, guid, title, description, season,
episode_number, episode_type, published_at_ms, duration_ms, audio_url, mime,
byte_len, image_url, link, source, in_feed, search_norm, first_seen_at_ms,
updated_at_ms)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17,
$18, $19, $20, $20)",
)
.bind(user_id)
.bind(meta.episode_id.as_str())
.bind(meta.podcast_id.as_str())
.bind(&meta.guid)
.bind(&meta.title)
.bind(&meta.description)
.bind(meta.season.map(i64::from))
.bind(meta.episode_number.map(i64::from))
.bind(episode_type_to_sql(meta.episode_type))
.bind(meta.published_at_ms)
.bind(meta.duration_ms)
.bind(&meta.audio_url)
.bind(&meta.mime)
.bind(meta.byte_len.and_then(|len| i64::try_from(len).ok()))
.bind(&meta.image_url)
.bind(&meta.link)
.bind(source.as_sql())
.bind(in_feed)
.bind(episode_search_norm(
&meta.title,
meta.description.as_deref(),
))
.bind(now)
.execute(&mut *conn)
.await?;
Ok(())
}
/// Episode metadata of a reference learned through sync.
pub(crate) fn meta_from_ref(episode: &PodcastEpisodeRef) -> EpisodeMeta {
EpisodeMeta {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
guid: None,
title: episode.episode_title.clone(),
description: None,
season: episode.season,
episode_number: episode.episode_number,
episode_type: EpisodeType::default(),
published_at_ms: episode.published_at_ms,
duration_ms: episode.duration_ms,
audio_url: episode.audio_url.clone(),
mime: episode.mime.clone(),
byte_len: None,
image_url: episode.image_url.clone(),
link: None,
}
}
/// Records an episode learned through sync when the catalog does not know it.
///
/// Feed metadata always wins: an existing row is left untouched. The
/// podcast catalog row is created from the reference when missing (no
/// register is written).
pub(crate) async fn ensure_episode_from_ref(
conn: &mut PgConnection,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<()> {
let exists: bool = sqlx::query_scalar(
"SELECT EXISTS (SELECT 1 FROM furumusic__podcast_episode
WHERE user_id = $1 AND episode_id = $2)",
)
.bind(user_id)
.bind(episode.episode_id.as_str())
.fetch_one(&mut *conn)
.await?;
if exists {
return Ok(());
}
let now = now_ms();
ensure_podcast_row(
conn,
user_id,
&episode.podcast_id,
&episode.feed_url,
&episode.podcast_title,
now,
)
.await?;
insert_episode(
conn,
user_id,
&meta_from_ref(episode),
EpisodeSource::Sync,
false,
now,
)
.await
}
/// Known episodes of a podcast with the identity hints they were last seen with.
pub(crate) async fn known_episodes(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<KnownEpisode>> {
let rows = sqlx::query(
"SELECT episode_id, guid, audio_url, link, published_at_ms, title
FROM furumusic__podcast_episode
WHERE user_id = $1 AND podcast_id = $2 ORDER BY episode_id",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&mut *conn)
.await?;
let known = rows
.iter()
.map(|row| {
Ok(KnownEpisode {
episode_id: episode_id_col(row, "episode_id")?,
hints: EpisodeIdentityHints {
guid: row.try_get("guid")?,
enclosure_url: Some(row.try_get("audio_url")?),
link: row.try_get("link")?,
published_at_ms: row.try_get("published_at_ms")?,
title: Some(row.try_get("title")?),
},
})
})
.collect::<sqlx::Result<Vec<_>>>()?;
Ok(known)
}
/// Applies one podcast operation inside the caller's transaction (which
/// holds the account lock).
pub(crate) async fn apply_op(
conn: &mut PgConnection,
user_id: i64,
op: &SyncOpWire,
) -> Result<ApplyOutcome> {
if !op.payload.is_podcast() {
return Ok(ApplyOutcome::Ignored);
}
if let Err(error) = op.validate_podcast() {
return Ok(ApplyOutcome::Rejected(error.to_string()));
}
let result = match &op.payload {
SyncOpPayload::PodcastSubscriptionSet { .. } => {
let record = op.podcast_subscription().expect("subscription payload");
apply_subscription_record(conn, user_id, &record, now_ms()).await
}
SyncOpPayload::PodcastShareSet { .. } => {
apply_share_record(conn, user_id, &op.podcast_share().expect("share payload")).await
}
SyncOpPayload::PodcastFeedAliasesAdded { .. } => {
let (podcast_id, urls) = op.podcast_feed_aliases().expect("alias payload");
merge_aliases(conn, user_id, podcast_id, urls).await
}
SyncOpPayload::PodcastProgressSet { episode, .. } => {
let record = op.podcast_progress().expect("progress payload");
if let Err(error) = generation_to_sql(record.generation) {
return Ok(ApplyOutcome::Rejected(error.to_string()));
}
if let Some(episode) = episode {
ensure_episode_from_ref(conn, user_id, episode).await?;
}
apply_progress_record(conn, user_id, &record).await
}
SyncOpPayload::PodcastListenRecorded { session, episode } => {
if let Some(current) = load_session(conn, user_id, &session.session_id).await?
&& let Err(error) = merge_listen_session(&current, session)
{
return Ok(ApplyOutcome::Rejected(error.to_string()));
}
if let Some(episode) = episode {
ensure_episode_from_ref(conn, user_id, episode).await?;
}
apply_listen_session(conn, user_id, session).await
}
_ => return Ok(ApplyOutcome::Ignored),
};
match result {
Ok(true) => Ok(ApplyOutcome::Changed),
Ok(false) => Ok(ApplyOutcome::Unchanged),
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
Some(PodcastStoreError::Invalid(reason)) => Ok(ApplyOutcome::Rejected(reason.clone())),
_ => Err(error),
},
}
}
fn record_outcome(stats: &mut SnapshotApplyStats, result: Result<bool>) -> Result<()> {
match result {
Ok(true) => stats.changed += 1,
Ok(false) => stats.unchanged += 1,
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
Some(PodcastStoreError::Invalid(_)) => stats.rejected += 1,
_ => return Err(error),
},
}
Ok(())
}
fn check(result: Result<(), PodcastError>) -> Result<()> {
result.map_err(invalid)
}
impl PodcastStore {
/// Applies a device-sync operation received from a trusted device.
///
/// Non-podcast operations return [`ApplyOutcome::Ignored`]. Podcast
/// operations are validated (`SyncOpWire::validate_podcast`) and merged
/// with the contract functions in one transaction; applying the same
/// operations in any order, with duplicates, gives the same state.
pub(crate) async fn apply_remote_op(
&self,
user_id: i64,
op: &SyncOpWire,
) -> Result<ApplyOutcome> {
if !op.payload.is_podcast() {
return Ok(ApplyOutcome::Ignored);
}
let mut tx = self.begin_write(user_id).await?;
let outcome = apply_op(&mut tx, user_id, op).await?;
tx.commit().await?;
Ok(outcome)
}
/// Merges the podcast sections of a sync snapshot.
///
/// Sections exceeding the contract counts are ignored
/// (`SyncSnapshot::podcast_state_within_limits`); every record is
/// validated and merged with the same functions as operations, so a
/// snapshot never replaces newer local state. One transaction.
pub(crate) async fn apply_snapshot(
&self,
user_id: i64,
snapshot: &SyncSnapshot,
) -> Result<SnapshotApplyStats> {
let mut stats = SnapshotApplyStats::default();
if !snapshot.has_podcast_state() {
return Ok(stats);
}
if !snapshot.podcast_state_within_limits() {
stats.ignored_over_limits = true;
return Ok(stats);
}
let mut tx = self.begin_write(user_id).await?;
let now = now_ms();
for record in &snapshot.podcast_subscriptions {
let mut record = record.clone();
record.normalize();
let result = match check(record.validate()) {
Ok(()) => apply_subscription_record(&mut tx, user_id, &record, now).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
for record in &snapshot.podcast_shares {
let result = match check(record.validate()) {
Ok(()) => apply_share_record(&mut tx, user_id, record).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
for record in &snapshot.podcast_progress {
let result = match check(record.validate()) {
Ok(()) => apply_progress_record(&mut tx, user_id, record).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
for session in &snapshot.podcast_listen_sessions {
let result = match check(session.validate()) {
Ok(()) => apply_listen_session(&mut tx, user_id, session).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
tx.commit().await?;
Ok(stats)
}
/// Fills the podcast sections of `snapshot` with this account's
/// registers and sessions, bounded by
/// `SyncSnapshot::bound_podcast_state_to(budget_bytes)`.
///
/// Records are read newest first and capped by the per-section counts
/// before the byte budget is applied. Fetch state and the catalog are
/// never part of the snapshot. Send it only to v4 peers
/// (`SyncSnapshot::for_peer`).
pub(crate) async fn fill_snapshot(
&self,
user_id: i64,
snapshot: &mut SyncSnapshot,
budget_bytes: usize,
) -> Result<()> {
use music_dht::podcast::{
MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES,
MAX_SNAPSHOT_SUBSCRIPTIONS,
};
// One consistent view of every section.
let mut tx = self.pool.begin().await?;
sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
.execute(&mut *tx)
.await?;
let rows = sqlx::query(
"SELECT s.podcast_id, s.subscribed, s.podcast_json, s.generation, s.hlc_ms, s.op_id,
COALESCE((SELECT array_agg(a.feed_url ORDER BY a.feed_url)
FROM furumusic__podcast_feed_alias a
WHERE a.user_id = s.user_id AND a.podcast_id = s.podcast_id),
'{}') AS aliases
FROM furumusic__podcast_subscription s
WHERE s.user_id = $1
ORDER BY s.generation DESC, s.hlc_ms DESC, s.op_id DESC, s.podcast_id DESC
LIMIT $2",
)
.bind(user_id)
.bind(MAX_SNAPSHOT_SUBSCRIPTIONS as i64)
.fetch_all(&mut *tx)
.await?;
let subscriptions = rows
.iter()
.map(|row| {
let aliases: Vec<String> = row.try_get("aliases")?;
Ok(SubscriptionRecord {
podcast_id: super::podcast_id_col(row, "podcast_id")?,
subscribed: row.try_get("subscribed")?,
podcast: podcast_meta_col(row, "podcast_json")?,
feed_aliases: aliases
.into_iter()
.map(FeedUrl::try_from)
.collect::<Result<_, _>>()
.map_err(|error| sqlx::Error::Decode(Box::new(error)))?,
generation: generation_from_sql(row.try_get("generation")?),
hlc_ms: row.try_get("hlc_ms")?,
op_id: row.try_get("op_id")?,
})
})
.collect::<sqlx::Result<Vec<_>>>()?;
let shares = sqlx::query(
"SELECT podcast_id, shared, generation, hlc_ms, op_id FROM furumusic__podcast_share
WHERE user_id = $1
ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC
LIMIT $2",
)
.bind(user_id)
.bind(MAX_SNAPSHOT_SHARES as i64)
.fetch_all(&mut *tx)
.await?
.iter()
.map(share_from_row)
.collect::<sqlx::Result<Vec<_>>>()?;
let progress = sqlx::query(&format!(
"SELECT {} FROM furumusic__podcast_progress p
WHERE p.user_id = $1
ORDER BY p.generation DESC, p.hlc_ms DESC, p.op_id DESC, p.episode_id DESC
LIMIT $2",
progress_columns("p", "")
))
.bind(user_id)
.bind(MAX_SNAPSHOT_PROGRESS as i64)
.fetch_all(&mut *tx)
.await?
.iter()
.map(|row| progress_from_row(row, ""))
.collect::<sqlx::Result<Vec<_>>>()?;
let sessions = sqlx::query(&format!(
"SELECT {SESSION_COLUMNS} FROM furumusic__podcast_listen_session s
WHERE s.user_id = $1
ORDER BY s.ended_at_ms DESC, s.session_id DESC
LIMIT $2"
))
.bind(user_id)
.bind(MAX_SNAPSHOT_LISTEN_SESSIONS as i64)
.fetch_all(&mut *tx)
.await?
.iter()
.map(session_from_row)
.collect::<sqlx::Result<Vec<_>>>()?;
tx.commit().await?;
snapshot.podcast_subscriptions = subscriptions;
snapshot.podcast_shares = shares;
snapshot.podcast_progress = progress;
snapshot.podcast_listen_sessions = sessions;
snapshot.bound_podcast_state_to(budget_bytes);
Ok(())
}
/// A snapshot holding only this account's podcast sections, bounded
/// with the default budget (`MAX_PODCAST_SNAPSHOT_BYTES`).
pub(crate) async fn export_snapshot(&self, user_id: i64) -> Result<SyncSnapshot> {
let mut snapshot = SyncSnapshot::default();
self.fill_snapshot(
user_id,
&mut snapshot,
music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES,
)
.await?;
Ok(snapshot)
}
/// Progress register of an episode (also for episodes the catalog does
/// not know yet).
pub(crate) async fn progress(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<ProgressRecord>> {
let mut conn = self.pool.acquire().await?;
load_progress(&mut conn, user_id, episode_id).await
}
}
+273
View File
@@ -0,0 +1,273 @@
//! PostgreSQL podcast store of the web server.
//!
//! [`PodcastStore`] implements the podcast contract of `music_dht::podcast`
//! (normative description: `PODCASTS.md` of frid) on PostgreSQL, with the
//! same invariants as the SQLite reference store of `furumi-library`. Each
//! web account is one virtual trusted device, so every table is keyed by
//! `user_id`. The store owns no HTTP, scheduling or UI: RSS fetching and the
//! player use it.
//!
//! # Tables
//!
//! | Table | Key | Content | Synced |
//! | --- | --- | --- | --- |
//! | `furumusic__podcast` | `podcast_id` | local channel metadata, `feed_url` = URL this account fetches | no (catalog) |
//! | `furumusic__podcast_feed_alias` | `(podcast_id, feed_url)` | bounded grow-only alias set | yes, inside subscription records and `podcast_feed_aliases_added` |
//! | `furumusic__podcast_episode` | `episode_id` | `EpisodeMeta`, `source` (`feed`/`sync`), `in_feed` | no (catalog) |
//! | `furumusic__podcast_season` (view) | `(podcast_id, season)` | derived from the episodes | no |
//! | `furumusic__podcast_subscription` | `podcast_id` | subscription register | yes |
//! | `furumusic__podcast_share` | `podcast_id` | share register | yes |
//! | `furumusic__podcast_progress` | `episode_id` | progress register | yes |
//! | `furumusic__podcast_listen_session` | `session_id` | grow-only set of sessions | yes |
//! | `furumusic__podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, check times, errors, cover cache | **never** |
//!
//! # One apply path
//!
//! Remote operations ([`PodcastStore::apply_remote_op`]), snapshot sections
//! ([`PodcastStore::apply_snapshot`]) and local actions all end in the same
//! functions: the `SyncOpWire` materializers of `music_dht::device_sync` and
//! the `merge_*` rules of `music_dht::podcast`. Applying is therefore
//! idempotent and independent of delivery order, and a snapshot never
//! replaces newer state.
//!
//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...)
//! run in one transaction: they read the registers, append the operation to
//! the account's device-sync log with its stamp
//! ([`crate::federation::devices::append_local_op`]: next `seq`, `hlc_ms`,
//! `op_id = <device>:<seq>`) and apply exactly that operation. Either both
//! the log and the registers change or neither does. Every write
//! transaction takes a per-account advisory lock, so concurrent writers of
//! one account serialize.
//!
//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the
//! alias set only; they never write a register, progress or a session.
// The RSS refresh job and the podcast UI/HTTP API that call the local
// actions, queries and fetch state land in the following commits of this
// series; until then only sync and the tests use them. Remove this
// allowance with them.
#![allow(dead_code, unused_imports)]
mod actions;
mod apply;
mod queries;
mod refresh;
pub(crate) mod schema;
#[cfg(test)]
pub(crate) mod postgres_tests;
#[cfg(test)]
mod tests;
use std::fmt;
use anyhow::Result;
use music_dht::podcast::{
EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord,
ProgressStatus,
};
use sqlx::Row;
use sqlx::postgres::PgRow;
pub(crate) use actions::{CheckpointOutcome, ProgressSession, ProgressWrite, SubscribeOutcome};
pub(crate) use apply::{ApplyOutcome, SnapshotApplyStats};
pub(crate) use queries::{
EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter,
PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary,
};
pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome};
/// Typed errors of [`PodcastStore`]; other failures are database errors.
///
/// They are returned inside `anyhow::Error`; use `downcast_ref` to match.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum PodcastStoreError {
/// The podcast is unknown to this account.
UnknownPodcast(PodcastId),
/// The action needs an active subscription.
NotSubscribed(PodcastId),
/// Input violates the podcast contract.
Invalid(String),
}
impl fmt::Display for PodcastStoreError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"),
Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"),
Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"),
}
}
}
impl std::error::Error for PodcastStoreError {}
impl From<PodcastError> for PodcastStoreError {
fn from(error: PodcastError) -> Self {
Self::Invalid(error.to_string())
}
}
pub(crate) fn invalid(reason: impl ToString) -> anyhow::Error {
PodcastStoreError::Invalid(reason.to_string()).into()
}
/// PostgreSQL store of podcast catalog, personal state and fetch state.
///
/// Cheap to clone (the pool is shared). See the module documentation.
#[derive(Debug, Clone)]
pub(crate) struct PodcastStore {
pool: sqlx::PgPool,
}
impl PodcastStore {
pub(crate) fn new(pool: sqlx::PgPool) -> Self {
Self { pool }
}
/// Starts a write transaction holding the account's podcast lock.
async fn begin_write(
&self,
user_id: i64,
) -> Result<sqlx::Transaction<'static, sqlx::Postgres>> {
let mut tx = self.pool.begin().await?;
lock_account(&mut tx, user_id).await?;
Ok(tx)
}
}
/// Serializes podcast writers of one account until the transaction ends.
pub(crate) async fn lock_account(conn: &mut sqlx::PgConnection, user_id: i64) -> Result<()> {
sqlx::query(
"SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcasts:' || $1::text, 0))",
)
.bind(user_id)
.execute(conn)
.await?;
Ok(())
}
pub(crate) fn now_ms() -> i64 {
chrono::Utc::now().timestamp_millis()
}
/// Converts a contract generation to the `BIGINT` column value.
///
/// Generations above `i64::MAX` cannot be stored; records carrying one are
/// rejected (no conforming client produces them).
pub(crate) fn generation_to_sql(generation: u64) -> Result<i64, PodcastStoreError> {
i64::try_from(generation)
.map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string()))
}
pub(crate) fn generation_from_sql(value: i64) -> u64 {
u64::try_from(value).unwrap_or(0)
}
pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str {
match status {
ProgressStatus::NotStarted => "not_started",
ProgressStatus::InProgress => "in_progress",
ProgressStatus::Completed => "completed",
}
}
pub(crate) fn status_from_sql(value: &str) -> ProgressStatus {
match value {
"in_progress" => ProgressStatus::InProgress,
"completed" => ProgressStatus::Completed,
_ => ProgressStatus::NotStarted,
}
}
/// `u32` contract numbers (season, episode number) from a `BIGINT` column.
pub(crate) fn u32_from_sql(value: Option<i64>) -> Option<u32> {
value.and_then(|value| u32::try_from(value).ok())
}
fn decode_error(message: impl ToString) -> sqlx::Error {
sqlx::Error::Decode(Box::new(PodcastStoreError::Invalid(message.to_string())))
}
/// Parses a stored id; stored ids are validated on write.
pub(crate) fn podcast_id_col(row: &PgRow, column: &str) -> sqlx::Result<PodcastId> {
PodcastId::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
}
pub(crate) fn episode_id_col(row: &PgRow, column: &str) -> sqlx::Result<EpisodeId> {
EpisodeId::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
}
pub(crate) fn feed_url_col(row: &PgRow, column: &str) -> sqlx::Result<FeedUrl> {
FeedUrl::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
}
pub(crate) fn podcast_meta_col(row: &PgRow, column: &str) -> sqlx::Result<PodcastMeta> {
serde_json::from_value(row.try_get::<serde_json::Value, _>(column)?).map_err(decode_error)
}
/// Progress register columns of table alias `alias`, named `{prefix}...`.
pub(crate) fn progress_columns(alias: &str, prefix: &str) -> String {
[
"episode_id",
"podcast_id",
"position_ms",
"status",
"generation",
"session_id",
"hlc_ms",
"device_id",
"op_id",
]
.iter()
.map(|column| format!("{alias}.{column} AS {prefix}{column}"))
.collect::<Vec<_>>()
.join(", ")
}
/// Progress register selected with [`progress_columns`].
pub(crate) fn progress_from_row(row: &PgRow, prefix: &str) -> sqlx::Result<ProgressRecord> {
let column = |name: &str| format!("{prefix}{name}");
Ok(ProgressRecord {
episode_id: episode_id_col(row, &column("episode_id"))?,
podcast_id: podcast_id_col(row, &column("podcast_id"))?,
position_ms: row.try_get(column("position_ms").as_str())?,
status: status_from_sql(&row.try_get::<String, _>(column("status").as_str())?),
generation: generation_from_sql(row.try_get(column("generation").as_str())?),
session_id: row.try_get(column("session_id").as_str())?,
hlc_ms: row.try_get(column("hlc_ms").as_str())?,
device_id: row.try_get(column("device_id").as_str())?,
op_id: row.try_get(column("op_id").as_str())?,
})
}
/// Listening session columns of table alias `s` (plain names).
pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \
s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \
s.completed";
pub(crate) fn session_from_row(row: &PgRow) -> sqlx::Result<ListenSession> {
Ok(ListenSession {
session_id: row.try_get("session_id")?,
episode_id: episode_id_col(row, "episode_id")?,
podcast_id: podcast_id_col(row, "podcast_id")?,
device_id: row.try_get("device_id")?,
started_at_ms: row.try_get("started_at_ms")?,
ended_at_ms: row.try_get("ended_at_ms")?,
start_position_ms: row.try_get("start_position_ms")?,
end_position_ms: row.try_get("end_position_ms")?,
listened_ms: row.try_get("listened_ms")?,
completed: row.try_get("completed")?,
})
}
/// Normalized search text of an episode: `normalize_name` of the title and
/// of the description, separated by a newline (the normalization never
/// produces one, so a needle cannot match across the two).
pub(crate) fn episode_search_norm(title: &str, description: Option<&str>) -> String {
format!(
"{}\n{}",
music_dht::normalize_name(title),
music_dht::normalize_name(description.unwrap_or_default())
)
}
File diff suppressed because it is too large. Load diff
+664
View File
@@ -0,0 +1,664 @@
//! Read-only queries for the podcast UI and the player.
use anyhow::Result;
use music_dht::podcast::{
EpisodeId, EpisodeMeta, EpisodeType, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId,
PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, effective_share,
};
use sqlx::postgres::PgRow;
use sqlx::{PgConnection, Postgres, QueryBuilder, Row};
use super::apply;
use super::{
PodcastStore, SESSION_COLUMNS, episode_id_col, episode_search_norm, feed_url_col,
podcast_id_col, progress_columns, progress_from_row, session_from_row, u32_from_sql,
};
/// Which podcasts [`PodcastStore::list_podcasts`] returns.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum PodcastListFilter {
/// Podcasts with an active subscription.
#[default]
Subscribed,
/// Every podcast in the catalog, including unsubscribed ones and
/// podcasts only known from synced progress or queue items.
All,
}
/// A podcast with its personal state and episode counters.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct PodcastSummary {
/// Podcast id.
pub(crate) podcast_id: PodcastId,
/// Local channel metadata; `feed_url` is the URL this account fetches.
pub(crate) meta: PodcastMeta,
/// Whether the subscription register says subscribed.
pub(crate) subscribed: bool,
/// Whether the podcast must currently be published (`effective_share`).
pub(crate) shared: bool,
/// Episodes in the catalog.
pub(crate) episode_count: u64,
/// Episodes not completed (no progress, not started or in progress).
pub(crate) unplayed_count: u64,
/// Episodes in progress.
pub(crate) in_progress_count: u64,
/// Newest publication time.
pub(crate) latest_published_at_ms: Option<i64>,
/// Numbered seasons (episodes without a season are not counted).
pub(crate) season_count: u64,
}
/// One season group of a podcast. `season == None` groups the episodes
/// without `itunes:season`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct SeasonSummary {
/// Season number; `None` for episodes without a season.
pub(crate) season: Option<u32>,
/// Episodes in the group.
pub(crate) episode_count: u64,
/// Episodes not completed.
pub(crate) unplayed_count: u64,
/// Oldest publication time.
pub(crate) first_published_at_ms: Option<i64>,
/// Newest publication time.
pub(crate) last_published_at_ms: Option<i64>,
}
/// Season filter of an [`EpisodeQuery`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum SeasonFilter {
/// All episodes.
#[default]
Any,
/// Episodes of one season.
Season(u32),
/// Episodes without a season.
NoSeason,
}
/// Listening-state filter of an [`EpisodeQuery`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum PlayedFilter {
/// All episodes.
#[default]
Any,
/// Not completed (not started or in progress).
Unplayed,
/// Never started or marked unplayed.
NotStarted,
/// In progress.
InProgress,
/// Completed.
Played,
}
/// Episode ordering of an [`EpisodeQuery`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum EpisodeSort {
/// Newest publication first; undated episodes last.
#[default]
NewestFirst,
/// Oldest publication first; undated episodes last.
OldestFirst,
/// By season, then episode number, ascending; unnumbered last.
EpisodeNumber,
/// By season, then episode number, descending; unnumbered last.
EpisodeNumberDesc,
}
impl EpisodeSort {
fn order_by(self) -> &'static str {
match self {
Self::NewestFirst => "e.published_at_ms IS NULL, e.published_at_ms DESC, e.episode_id",
Self::OldestFirst => "e.published_at_ms IS NULL, e.published_at_ms, e.episode_id",
Self::EpisodeNumber => {
"e.season IS NULL, e.season, e.episode_number IS NULL, e.episode_number, \
e.published_at_ms IS NULL, e.published_at_ms, e.episode_id"
}
Self::EpisodeNumberDesc => {
"e.season IS NULL, e.season DESC, e.episode_number IS NULL, \
e.episode_number DESC, e.published_at_ms IS NULL, e.published_at_ms DESC, \
e.episode_id"
}
}
}
}
#[cfg(test)]
pub(crate) fn order_by_for_tests(sort: EpisodeSort) -> &'static str {
sort.order_by()
}
/// Parameters of [`PodcastStore::episodes`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct EpisodeQuery {
/// Podcast.
pub(crate) podcast_id: PodcastId,
/// Season filter.
pub(crate) season: SeasonFilter,
/// Listening-state filter.
pub(crate) played: PlayedFilter,
/// Text searched in titles and descriptions (`normalize_name`: Unicode
/// NFKC, case-insensitive substring match). Blank means no filter.
pub(crate) search: Option<String>,
/// Ordering.
pub(crate) sort: EpisodeSort,
/// Maximum rows (`None`: all).
pub(crate) limit: Option<usize>,
/// Rows to skip.
pub(crate) offset: usize,
}
impl EpisodeQuery {
/// All episodes of a podcast, newest first.
pub(crate) fn new(podcast_id: PodcastId) -> Self {
Self {
podcast_id,
season: SeasonFilter::Any,
played: PlayedFilter::Any,
search: None,
sort: EpisodeSort::NewestFirst,
limit: None,
offset: 0,
}
}
}
/// An episode with its personal state.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct EpisodeListItem {
/// Episode metadata (feed or synced reference).
pub(crate) episode: EpisodeMeta,
/// Progress register, if any.
pub(crate) progress: Option<ProgressRecord>,
/// Distinct completed listening sessions (`completed_listen_count`).
pub(crate) completed_count: u64,
/// Whether the latest refresh of the feed listed the episode.
pub(crate) in_feed: bool,
}
impl EpisodeListItem {
/// Listening status (`NotStarted` without a register).
pub(crate) fn status(&self) -> ProgressStatus {
self.progress
.as_ref()
.map_or(ProgressStatus::NotStarted, |progress| progress.status)
}
/// Position ordinary Play resumes from.
pub(crate) fn resume_position_ms(&self) -> i64 {
self.progress
.as_ref()
.map_or(0, ProgressRecord::resume_position_ms)
}
}
/// A listening session in history, with display titles when known.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct PodcastListenEntry {
/// The session.
pub(crate) session: ListenSession,
/// Episode title from the catalog.
pub(crate) episode_title: Option<String>,
/// Podcast title from the catalog.
pub(crate) podcast_title: Option<String>,
}
const EPISODE_COLUMNS: &str = "e.episode_id, e.podcast_id, e.guid, e.title, e.description, \
e.season, e.episode_number, e.episode_type, e.published_at_ms, e.duration_ms, \
e.audio_url, e.mime, e.byte_len, e.image_url, e.link";
const NOT_COMPLETED: &str = "COALESCE(g.status, 'not_started') <> 'completed'";
fn episode_from_row(row: &PgRow) -> sqlx::Result<EpisodeMeta> {
Ok(EpisodeMeta {
episode_id: episode_id_col(row, "episode_id")?,
podcast_id: podcast_id_col(row, "podcast_id")?,
guid: row.try_get("guid")?,
title: row.try_get("title")?,
description: row.try_get("description")?,
season: u32_from_sql(row.try_get("season")?),
episode_number: u32_from_sql(row.try_get("episode_number")?),
episode_type: EpisodeType::parse(&row.try_get::<String, _>("episode_type")?),
published_at_ms: row.try_get("published_at_ms")?,
duration_ms: row.try_get("duration_ms")?,
audio_url: row.try_get("audio_url")?,
mime: row.try_get("mime")?,
byte_len: row
.try_get::<Option<i64>, _>("byte_len")?
.and_then(|len| u64::try_from(len).ok()),
image_url: row.try_get("image_url")?,
link: row.try_get("link")?,
})
}
fn list_item_from_row(row: &PgRow) -> sqlx::Result<EpisodeListItem> {
let progress = if row.try_get::<Option<String>, _>("g_episode_id")?.is_some() {
Some(progress_from_row(row, "g_")?)
} else {
None
};
Ok(EpisodeListItem {
episode: episode_from_row(row)?,
in_feed: row.try_get("in_feed")?,
progress,
completed_count: count(row.try_get("completed_count")?),
})
}
fn count(value: i64) -> u64 {
u64::try_from(value).unwrap_or(0)
}
pub(crate) async fn episode_meta_in(
conn: &mut PgConnection,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<EpisodeMeta>> {
let row = sqlx::query(&format!(
"SELECT {EPISODE_COLUMNS} FROM furumusic__podcast_episode e
WHERE e.user_id = $1 AND e.episode_id = $2"
))
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row.as_ref().map(episode_from_row).transpose()?)
}
pub(crate) async fn podcast_meta_in(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastMeta>> {
let row = sqlx::query(
"SELECT feed_url, title, author, description, link, image_url
FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(None);
};
Ok(Some(PodcastMeta {
feed_url: feed_url_col(&row, "feed_url")?,
title: row.try_get("title")?,
author: row.try_get("author")?,
description: row.try_get("description")?,
link: row.try_get("link")?,
image_url: row.try_get("image_url")?,
}))
}
/// Queue reference of a catalog episode, with `progress_generation` set to
/// the generation of its progress register (handoff hint).
pub(crate) async fn episode_ref_in(
conn: &mut PgConnection,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<PodcastEpisodeRef>> {
let Some(episode) = episode_meta_in(conn, user_id, episode_id).await? else {
return Ok(None);
};
let Some(podcast) = podcast_meta_in(conn, user_id, &episode.podcast_id).await? else {
return Ok(None);
};
let mut reference = episode.to_ref(&podcast);
reference.progress_generation = apply::load_progress(conn, user_id, episode_id)
.await?
.map(|progress| progress.generation);
Ok(Some(reference))
}
async fn summary_in(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastSummary>> {
let Some(meta) = podcast_meta_in(conn, user_id, podcast_id).await? else {
return Ok(None);
};
let row = sqlx::query(&format!(
"SELECT COUNT(*) AS episode_count,
COUNT(*) FILTER (WHERE {NOT_COMPLETED}) AS unplayed_count,
COUNT(*) FILTER (WHERE g.status = 'in_progress') AS in_progress_count,
MAX(e.published_at_ms) AS latest,
(SELECT COUNT(DISTINCT e2.season) FROM furumusic__podcast_episode e2
WHERE e2.user_id = $1 AND e2.podcast_id = $2) AS season_count
FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = $1 AND e.podcast_id = $2"
))
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_one(&mut *conn)
.await?;
let subscription = apply::load_subscription_register(conn, user_id, podcast_id).await?;
let share = apply::load_share(conn, user_id, podcast_id).await?;
Ok(Some(PodcastSummary {
podcast_id: podcast_id.clone(),
meta,
subscribed: subscription
.as_ref()
.is_some_and(|record| record.subscribed),
shared: effective_share(subscription.as_ref(), share.as_ref()),
episode_count: count(row.try_get("episode_count")?),
unplayed_count: count(row.try_get("unplayed_count")?),
in_progress_count: count(row.try_get("in_progress_count")?),
latest_published_at_ms: row.try_get("latest")?,
season_count: count(row.try_get("season_count")?),
}))
}
/// Computes the cached search text of episodes that do not have it yet
/// (rows imported by the `m_0057` migration).
async fn backfill_search_norm(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<()> {
let rows = sqlx::query(
"SELECT episode_id, title, description FROM furumusic__podcast_episode
WHERE user_id = $1 AND podcast_id = $2 AND search_norm IS NULL",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&mut *conn)
.await?;
for row in rows {
let title: String = row.try_get("title")?;
let description: Option<String> = row.try_get("description")?;
sqlx::query(
"UPDATE furumusic__podcast_episode SET search_norm = $3
WHERE user_id = $1 AND episode_id = $2 AND search_norm IS NULL",
)
.bind(user_id)
.bind(row.try_get::<String, _>("episode_id")?)
.bind(episode_search_norm(&title, description.as_deref()))
.execute(&mut *conn)
.await?;
}
Ok(())
}
impl PodcastStore {
/// Podcasts with subscription state and counters, sorted by title.
pub(crate) async fn list_podcasts(
&self,
user_id: i64,
filter: PodcastListFilter,
) -> Result<Vec<PodcastSummary>> {
let mut conn = self.pool.acquire().await?;
let join = match filter {
PodcastListFilter::Subscribed => {
"JOIN furumusic__podcast_subscription r
ON r.user_id = p.user_id AND r.podcast_id = p.podcast_id AND r.subscribed"
}
PodcastListFilter::All => "",
};
let rows = sqlx::query(&format!(
"SELECT p.podcast_id FROM furumusic__podcast p {join}
WHERE p.user_id = $1
ORDER BY lower(p.title), p.podcast_id"
))
.bind(user_id)
.fetch_all(&mut *conn)
.await?;
let ids = rows
.iter()
.map(|row| podcast_id_col(row, "podcast_id"))
.collect::<sqlx::Result<Vec<_>>>()?;
let mut out = Vec::with_capacity(ids.len());
for id in ids {
if let Some(summary) = summary_in(&mut conn, user_id, &id).await? {
out.push(summary);
}
}
Ok(out)
}
/// One podcast with subscription state and counters.
pub(crate) async fn podcast(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastSummary>> {
let mut conn = self.pool.acquire().await?;
summary_in(&mut conn, user_id, podcast_id).await
}
/// The subscription register (with the stored alias set), if any.
pub(crate) async fn subscription(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<SubscriptionRecord>> {
let mut conn = self.pool.acquire().await?;
apply::load_subscription_register(&mut conn, user_id, podcast_id).await
}
/// The share register, if any.
pub(crate) async fn share(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<ShareRecord>> {
let mut conn = self.pool.acquire().await?;
apply::load_share(&mut conn, user_id, podcast_id).await
}
/// The bounded, synced feed alias set of a podcast.
pub(crate) async fn feed_aliases(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<FeedUrl>> {
let mut conn = self.pool.acquire().await?;
apply::load_aliases(&mut conn, user_id, podcast_id).await
}
/// Season groups of a podcast: numbered seasons ascending, then the
/// group of episodes without a season.
pub(crate) async fn seasons(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<SeasonSummary>> {
let rows = sqlx::query(&format!(
"SELECT s.season, s.episode_count, s.first_published_at_ms, s.last_published_at_ms,
(SELECT COUNT(*) FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = s.user_id AND e.podcast_id = s.podcast_id
AND e.season IS NOT DISTINCT FROM s.season
AND {NOT_COMPLETED}) AS unplayed_count
FROM furumusic__podcast_season s
WHERE s.user_id = $1 AND s.podcast_id = $2
ORDER BY s.season IS NULL, s.season"
))
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&self.pool)
.await?;
let seasons = rows
.iter()
.map(|row| {
Ok(SeasonSummary {
season: u32_from_sql(row.try_get("season")?),
episode_count: count(row.try_get("episode_count")?),
first_published_at_ms: row.try_get("first_published_at_ms")?,
last_published_at_ms: row.try_get("last_published_at_ms")?,
unplayed_count: count(row.try_get("unplayed_count")?),
})
})
.collect::<sqlx::Result<Vec<_>>>()?;
Ok(seasons)
}
/// Episodes of a podcast or season with their progress.
pub(crate) async fn episodes(
&self,
user_id: i64,
query: &EpisodeQuery,
) -> Result<Vec<EpisodeListItem>> {
let mut conn = self.pool.acquire().await?;
let needle = query
.search
.as_deref()
.map(music_dht::normalize_name)
.filter(|needle| !needle.is_empty());
if needle.is_some() {
backfill_search_norm(&mut conn, user_id, &query.podcast_id).await?;
}
let mut sql: QueryBuilder<'_, Postgres> = QueryBuilder::new(format!(
"SELECT {EPISODE_COLUMNS}, e.in_feed, {},
(SELECT COUNT(*) FROM furumusic__podcast_listen_session s
WHERE s.user_id = e.user_id AND s.episode_id = e.episode_id
AND s.completed) AS completed_count
FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = ",
progress_columns("g", "g_")
));
sql.push_bind(user_id);
sql.push(" AND e.podcast_id = ");
sql.push_bind(query.podcast_id.as_str().to_string());
match query.season {
SeasonFilter::Any => {}
SeasonFilter::Season(season) => {
sql.push(" AND e.season = ");
sql.push_bind(i64::from(season));
}
SeasonFilter::NoSeason => {
sql.push(" AND e.season IS NULL");
}
}
match query.played {
PlayedFilter::Any => {}
PlayedFilter::Unplayed => {
sql.push(format!(" AND {NOT_COMPLETED}"));
}
PlayedFilter::NotStarted => {
sql.push(" AND COALESCE(g.status, 'not_started') = 'not_started'");
}
PlayedFilter::InProgress => {
sql.push(" AND g.status = 'in_progress'");
}
PlayedFilter::Played => {
sql.push(" AND g.status = 'completed'");
}
}
if let Some(needle) = needle {
sql.push(" AND strpos(e.search_norm, ");
sql.push_bind(needle);
sql.push(") > 0");
}
sql.push(format!(" ORDER BY {}", query.sort.order_by()));
if let Some(limit) = query.limit {
sql.push(" LIMIT ");
sql.push_bind(i64::try_from(limit).unwrap_or(i64::MAX));
}
sql.push(" OFFSET ");
sql.push_bind(i64::try_from(query.offset).unwrap_or(i64::MAX));
let rows = sql.build().fetch_all(&mut *conn).await?;
Ok(rows
.iter()
.map(list_item_from_row)
.collect::<sqlx::Result<Vec<_>>>()?)
}
/// One episode with its progress.
pub(crate) async fn episode(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<EpisodeListItem>> {
let row = sqlx::query(&format!(
"SELECT {EPISODE_COLUMNS}, e.in_feed, {},
(SELECT COUNT(*) FROM furumusic__podcast_listen_session s
WHERE s.user_id = e.user_id AND s.episode_id = e.episode_id
AND s.completed) AS completed_count
FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = $1 AND e.episode_id = $2",
progress_columns("g", "g_")
))
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&self.pool)
.await?;
Ok(row.as_ref().map(list_item_from_row).transpose()?)
}
/// Position ordinary Play resumes from
/// (`ProgressRecord::resume_position_ms`, 0 without progress).
pub(crate) async fn resume_position_ms(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<i64> {
Ok(self
.progress(user_id, episode_id)
.await?
.map_or(0, |progress| progress.resume_position_ms()))
}
/// Listening history (sessions that `should_record`), newest first.
pub(crate) async fn listen_history(
&self,
user_id: i64,
podcast_id: Option<&PodcastId>,
limit: usize,
) -> Result<Vec<PodcastListenEntry>> {
let rows = sqlx::query(&format!(
"SELECT {SESSION_COLUMNS}, e.title AS episode_title, p.title AS podcast_title
FROM furumusic__podcast_listen_session s
LEFT JOIN furumusic__podcast_episode e
ON e.user_id = s.user_id AND e.episode_id = s.episode_id
LEFT JOIN furumusic__podcast p
ON p.user_id = s.user_id AND p.podcast_id = s.podcast_id
WHERE s.user_id = $1 AND (s.completed OR s.listened_ms >= $2)
AND ($3::text IS NULL OR s.podcast_id = $3)
ORDER BY s.ended_at_ms DESC, s.session_id DESC
LIMIT $4"
))
.bind(user_id)
.bind(music_dht::podcast::MIN_RECORDED_EPISODE_LISTEN_MS)
.bind(podcast_id.map(PodcastId::as_str))
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
.iter()
.map(|row| {
Ok(PodcastListenEntry {
session: session_from_row(row)?,
episode_title: row.try_get("episode_title")?,
podcast_title: row.try_get("podcast_title")?,
})
})
.collect::<sqlx::Result<Vec<_>>>()?)
}
/// Number of distinct completed sessions of an episode.
pub(crate) async fn completed_listen_count(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<u64> {
let total: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_listen_session
WHERE user_id = $1 AND episode_id = $2 AND completed",
)
.bind(user_id)
.bind(episode_id.as_str())
.fetch_one(&self.pool)
.await?;
Ok(count(total))
}
}
+472
View File
@@ -0,0 +1,472 @@
//! Feed refresh ingestion and the local-only fetch state.
use std::collections::BTreeSet;
use anyhow::Result;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::feed::ParsedFeed;
use music_dht::podcast::{EpisodeMeta, FeedUrl, PodcastId, merge_feed_aliases};
use sqlx::{PgConnection, Row};
use super::apply::{self, EpisodeSource};
use super::{PodcastStore, PodcastStoreError, episode_search_norm, now_ms};
use crate::federation::devices;
/// A fetched and parsed feed document for [`PodcastStore::apply_feed`].
#[derive(Debug, Clone, Copy)]
pub(crate) struct FeedRefresh<'a> {
/// Podcast the feed belongs to.
pub(crate) podcast_id: &'a PodcastId,
/// The parsed document (`music_dht::podcast::feed::parse_feed_with_base`
/// with `fetched_from` as base).
pub(crate) feed: &'a ParsedFeed,
/// URL the document was finally served from, after redirects.
pub(crate) fetched_from: &'a FeedUrl,
/// Whether every redirect leading to `fetched_from` was permanent
/// (301/308). Temporary redirects do not move the feed.
pub(crate) moved_permanently: bool,
}
/// Result of [`PodcastStore::apply_feed`].
#[derive(Debug, Clone, Default, PartialEq)]
pub(crate) struct RefreshOutcome {
/// Episodes seen for the first time.
pub(crate) new_episodes: usize,
/// Known episodes whose metadata changed (title, audio URL, ...).
pub(crate) updated_episodes: usize,
/// Known episodes without metadata changes.
pub(crate) unchanged_episodes: usize,
/// Items without identity or with invalid metadata.
pub(crate) skipped_items: usize,
/// Feed URLs learned from a permanent redirect or `itunes:new-feed-url`.
pub(crate) learned_aliases: Vec<FeedUrl>,
/// URL the podcast is fetched from after this refresh.
pub(crate) feed_url: Option<FeedUrl>,
/// `podcast_feed_aliases_added`, only when the learned URLs changed the
/// stored alias set (already appended to the sync log).
pub(crate) ops: Vec<SyncOpWire>,
}
/// Local-only fetch state of a podcast. Never part of sync output.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct FetchState {
/// HTTP `ETag` validator of the last successful fetch.
pub(crate) etag: Option<String>,
/// HTTP `Last-Modified` validator of the last successful fetch.
pub(crate) last_modified: Option<String>,
/// Last fetch attempt, unix milliseconds.
pub(crate) checked_at_ms: Option<i64>,
/// Last successful fetch (including "not modified").
pub(crate) last_success_at_ms: Option<i64>,
/// When the scheduler should check again (`None`: due now).
pub(crate) next_check_at_ms: Option<i64>,
/// Error of the last attempt; `None` after a success.
pub(crate) last_error: Option<String>,
/// Failed attempts since the last success.
pub(crate) consecutive_failures: u32,
/// Image URL the cached cover was downloaded from.
pub(crate) cover_url: Option<String>,
/// Local path of the cached cover.
pub(crate) cover_path: Option<String>,
}
async fn podcast_fetch_url(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<FeedUrl>> {
let row = sqlx::query(
"SELECT feed_url FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row
.as_ref()
.map(|row| super::feed_url_col(row, "feed_url"))
.transpose()?)
}
async fn fetch_state_in(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
lock: bool,
) -> Result<FetchState> {
let row = sqlx::query(&format!(
"SELECT etag, last_modified, checked_at_ms, last_success_at_ms, next_check_at_ms,
last_error, consecutive_failures, cover_url, cover_path
FROM furumusic__podcast_fetch_state WHERE user_id = $1 AND podcast_id = $2{}",
if lock { " FOR UPDATE" } else { "" }
))
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(FetchState::default());
};
Ok(FetchState {
etag: row.try_get("etag")?,
last_modified: row.try_get("last_modified")?,
checked_at_ms: row.try_get("checked_at_ms")?,
last_success_at_ms: row.try_get("last_success_at_ms")?,
next_check_at_ms: row.try_get("next_check_at_ms")?,
last_error: row.try_get("last_error")?,
consecutive_failures: u32::try_from(row.try_get::<i64, _>("consecutive_failures")?)
.unwrap_or(u32::MAX),
cover_url: row.try_get("cover_url")?,
cover_path: row.try_get("cover_path")?,
})
}
/// Inserts or updates a feed episode; `None` when it is new, otherwise
/// whether its metadata changed.
async fn upsert_feed_episode(
conn: &mut PgConnection,
user_id: i64,
meta: &EpisodeMeta,
now: i64,
) -> Result<Option<bool>> {
let Some(stored) = super::queries::episode_meta_in(conn, user_id, &meta.episode_id).await?
else {
apply::insert_episode(conn, user_id, meta, EpisodeSource::Feed, true, now).await?;
return Ok(None);
};
let changed = stored != *meta;
sqlx::query(
"UPDATE furumusic__podcast_episode
SET podcast_id = $3, guid = $4, title = $5, description = $6, season = $7,
episode_number = $8, episode_type = $9, published_at_ms = $10,
duration_ms = $11, audio_url = $12, mime = $13, byte_len = $14,
image_url = $15, link = $16, source = 'feed', in_feed = TRUE,
search_norm = $17,
updated_at_ms = CASE WHEN $18 THEN $19 ELSE updated_at_ms END
WHERE user_id = $1 AND episode_id = $2",
)
.bind(user_id)
.bind(meta.episode_id.as_str())
.bind(meta.podcast_id.as_str())
.bind(&meta.guid)
.bind(&meta.title)
.bind(&meta.description)
.bind(meta.season.map(i64::from))
.bind(meta.episode_number.map(i64::from))
.bind(apply::episode_type_to_sql(meta.episode_type))
.bind(meta.published_at_ms)
.bind(meta.duration_ms)
.bind(&meta.audio_url)
.bind(&meta.mime)
.bind(meta.byte_len.and_then(|len| i64::try_from(len).ok()))
.bind(&meta.image_url)
.bind(&meta.link)
.bind(episode_search_norm(
&meta.title,
meta.description.as_deref(),
))
.bind(changed)
.bind(now)
.execute(&mut *conn)
.await?;
Ok(Some(changed))
}
impl PodcastStore {
/// Ingests a fetched feed document for a podcast of the account.
///
/// * Channel metadata of the catalog row is replaced by the feed's.
/// * Episode ids are resolved with the contract identity rule
/// (`ParsedFeed::resolve_episode_ids`) against every stored episode of
/// the podcast, including episodes learned through sync, so a changed
/// title, audio URL or a GUID-less item keeps its id. Known episodes
/// get the new metadata (including the audio URL); new ones are added;
/// episodes missing from the document are kept (`in_feed = false`).
/// * A permanent redirect target and `itunes:new-feed-url` become feed
/// aliases and the new fetch URL. When they change the stored alias
/// set, one `podcast_feed_aliases_added` operation is appended.
/// * Subscription, share and progress registers and listening sessions
/// are never touched, so a refresh cannot resurrect an unsubscribed
/// podcast or move anyone's position.
///
/// Record the HTTP outcome separately with
/// [`PodcastStore::record_fetch_success`] and friends.
pub(crate) async fn apply_feed(
&self,
user_id: i64,
refresh: &FeedRefresh<'_>,
) -> Result<RefreshOutcome> {
let podcast_id = refresh.podcast_id;
let feed = refresh.feed;
let now = now_ms();
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let current_url = match podcast_fetch_url(&mut tx, user_id, podcast_id).await? {
Some(url) => url,
None => {
let Some(register) =
apply::load_subscription_register(&mut tx, user_id, podcast_id).await?
else {
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
};
apply::apply_subscription_record(&mut tx, user_id, &register, now).await?;
register.podcast.feed_url
}
};
let known = apply::known_episodes(&mut tx, user_id, podcast_id).await?;
let ids = feed.resolve_episode_ids(podcast_id, &known);
sqlx::query(
"UPDATE furumusic__podcast_episode SET in_feed = FALSE
WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.execute(&mut *tx)
.await?;
let mut outcome = RefreshOutcome::default();
for (item, id) in feed.episodes.iter().zip(ids) {
let Some(id) = id else {
outcome.skipped_items += 1;
continue;
};
let meta = item.to_meta(podcast_id, id);
if meta.validate().is_err() {
outcome.skipped_items += 1;
continue;
}
match upsert_feed_episode(&mut tx, user_id, &meta, now).await? {
None => outcome.new_episodes += 1,
Some(true) => outcome.updated_episodes += 1,
Some(false) => outcome.unchanged_episodes += 1,
}
}
let mut fetch_url = current_url.clone();
let mut learned = BTreeSet::new();
if refresh.moved_permanently && *refresh.fetched_from != fetch_url {
learned.insert(fetch_url.clone());
learned.insert(refresh.fetched_from.clone());
fetch_url = refresh.fetched_from.clone();
}
if let Some(new_url) = feed
.new_feed_url
.as_deref()
.and_then(|url| FeedUrl::parse(url).ok())
&& new_url != fetch_url
{
learned.insert(fetch_url.clone());
learned.insert(new_url.clone());
fetch_url = new_url;
}
let meta = feed.to_podcast_meta(fetch_url.clone());
sqlx::query(
"UPDATE furumusic__podcast
SET feed_url = $3, title = $4, author = $5, description = $6, link = $7,
image_url = $8, meta_from_feed = TRUE, updated_at_ms = $9
WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(meta.feed_url.as_str())
.bind(&meta.title)
.bind(&meta.author)
.bind(&meta.description)
.bind(&meta.link)
.bind(&meta.image_url)
.bind(now)
.execute(&mut *tx)
.await?;
if !learned.is_empty() {
let learned: Vec<FeedUrl> = learned.into_iter().collect();
let current = apply::load_aliases(&mut tx, user_id, podcast_id).await?;
if merge_feed_aliases(podcast_id, &current, &learned) != current {
let op = super::actions::emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastFeedAliasesAdded {
podcast_id: podcast_id.clone(),
feed_aliases: learned.clone(),
}
})
.await?;
outcome.ops.push(op);
}
outcome.learned_aliases = learned;
}
tx.commit().await?;
outcome.feed_url = Some(fetch_url);
Ok(outcome)
}
/// Local fetch state of a podcast (defaults when never fetched).
pub(crate) async fn fetch_state(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<FetchState> {
let mut conn = self.pool.acquire().await?;
fetch_state_in(&mut conn, user_id, podcast_id, false).await
}
/// Read-modify-write of the fetch state in one transaction with the
/// account lock and the row locked, so concurrent `record_*` /
/// `set_cover_cache` calls never lose each other's columns or failure
/// increments.
async fn update_fetch_state(
&self,
user_id: i64,
podcast_id: &PodcastId,
update: impl FnOnce(&mut FetchState),
) -> Result<FetchState> {
let mut tx = self.begin_write(user_id).await?;
if podcast_fetch_url(&mut tx, user_id, podcast_id)
.await?
.is_none()
{
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
}
let mut state = fetch_state_in(&mut tx, user_id, podcast_id, true).await?;
update(&mut state);
sqlx::query(
"INSERT INTO furumusic__podcast_fetch_state
(user_id, podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms,
next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
etag = EXCLUDED.etag, last_modified = EXCLUDED.last_modified,
checked_at_ms = EXCLUDED.checked_at_ms,
last_success_at_ms = EXCLUDED.last_success_at_ms,
next_check_at_ms = EXCLUDED.next_check_at_ms,
last_error = EXCLUDED.last_error,
consecutive_failures = EXCLUDED.consecutive_failures,
cover_url = EXCLUDED.cover_url, cover_path = EXCLUDED.cover_path",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(&state.etag)
.bind(&state.last_modified)
.bind(state.checked_at_ms)
.bind(state.last_success_at_ms)
.bind(state.next_check_at_ms)
.bind(&state.last_error)
.bind(i64::from(state.consecutive_failures))
.bind(&state.cover_url)
.bind(&state.cover_path)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(state)
}
/// Records a successful fetch (HTTP 200) with its new validators.
pub(crate) async fn record_fetch_success(
&self,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
etag: Option<&str>,
last_modified: Option<&str>,
next_check_at_ms: Option<i64>,
) -> Result<()> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.etag = etag.map(str::to_string);
state.last_modified = last_modified.map(str::to_string);
state.checked_at_ms = Some(now_ms);
state.last_success_at_ms = Some(now_ms);
state.next_check_at_ms = next_check_at_ms;
state.last_error = None;
state.consecutive_failures = 0;
})
.await
.map(|_| ())
}
/// Records a conditional fetch answered with "304 Not Modified".
pub(crate) async fn record_fetch_not_modified(
&self,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
next_check_at_ms: Option<i64>,
) -> Result<()> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.checked_at_ms = Some(now_ms);
state.last_success_at_ms = Some(now_ms);
state.next_check_at_ms = next_check_at_ms;
state.last_error = None;
state.consecutive_failures = 0;
})
.await
.map(|_| ())
}
/// Records a failed fetch; returns the consecutive failure count. The
/// stored catalog is kept.
pub(crate) async fn record_fetch_failure(
&self,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
error: &str,
next_check_at_ms: Option<i64>,
) -> Result<u32> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.checked_at_ms = Some(now_ms);
state.next_check_at_ms = next_check_at_ms;
state.last_error = Some(error.to_string());
state.consecutive_failures = state.consecutive_failures.saturating_add(1);
})
.await
.map(|state| state.consecutive_failures)
}
/// Records the locally cached cover image of a podcast.
pub(crate) async fn set_cover_cache(
&self,
user_id: i64,
podcast_id: &PodcastId,
cover_url: Option<&str>,
cover_path: Option<&str>,
) -> Result<()> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.cover_url = cover_url.map(str::to_string);
state.cover_path = cover_path.map(str::to_string);
})
.await
.map(|_| ())
}
/// Subscribed podcasts of every account due for a refresh at `now_ms`
/// (never checked or `next_check_at_ms <= now_ms`), most overdue first,
/// with their fetch URL: `(user_id, podcast_id, feed_url)`.
pub(crate) async fn podcasts_due_for_refresh(
&self,
now_ms: i64,
limit: usize,
) -> Result<Vec<(i64, PodcastId, FeedUrl)>> {
let rows = sqlx::query(
"SELECT p.user_id, p.podcast_id, p.feed_url
FROM furumusic__podcast p
JOIN furumusic__podcast_subscription s
ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id AND s.subscribed
LEFT JOIN furumusic__podcast_fetch_state f
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1
ORDER BY COALESCE(f.next_check_at_ms, 0), p.user_id, p.podcast_id
LIMIT $2",
)
.bind(now_ms)
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
.iter()
.map(|row| {
Ok((
row.try_get("user_id")?,
super::podcast_id_col(row, "podcast_id")?,
super::feed_url_col(row, "feed_url")?,
))
})
.collect::<sqlx::Result<Vec<_>>>()?)
}
}
+337
View File
@@ -0,0 +1,337 @@
//! PostgreSQL schema of the podcast store and the import of the relay state.
//!
//! The migrations themselves are declared in `crate::music::db_migrations`
//! (`m_0056_create_podcast_store`, `m_0057_import_podcast_relay`), next to
//! the rest of the `furumusic` chain; this module holds their SQL so the
//! store and its tests share one definition.
//!
//! The tables mirror the SQLite reference store of `furumi-library`
//! (`PODCASTS.md`, "SQLite reference store") with a `user_id` column in
//! every key: each web account is one virtual device with its own podcast
//! catalog, registers and fetch state.
/// Tables, the derived season view and indexes (`m_0056`).
pub(crate) const CREATE_PODCAST_STORE: &[&str] = &[
// Local channel metadata. `feed_url` is the URL this account fetches;
// `meta_from_feed` is set once the account refreshed the feed itself
// (until then the winning subscription register provides the metadata).
"CREATE TABLE IF NOT EXISTS furumusic__podcast (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
feed_url TEXT NOT NULL,
title TEXT NOT NULL,
author TEXT,
description TEXT,
link TEXT,
image_url TEXT,
meta_from_feed BOOLEAN NOT NULL DEFAULT FALSE,
created_at_ms BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
// Bounded grow-only alias set (`merge_feed_aliases`), synced inside
// subscription records and `podcast_feed_aliases_added`.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_feed_alias (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
feed_url TEXT NOT NULL,
PRIMARY KEY (user_id, podcast_id, feed_url)
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_feed_alias_url
ON furumusic__podcast_feed_alias (user_id, feed_url)",
// Episode catalog (`EpisodeMeta`). `source` is `feed` once this account
// parsed the episode from RSS, `sync` while it is only known from a
// `PodcastEpisodeRef`. `search_norm` caches `normalize_name(title)` and
// `normalize_name(description)` separated by a newline (never produced
// by the normalization), NULL until computed.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_episode (
user_id BIGINT NOT NULL,
episode_id TEXT NOT NULL,
podcast_id TEXT NOT NULL,
guid TEXT,
title TEXT NOT NULL,
description TEXT,
season BIGINT,
episode_number BIGINT,
episode_type TEXT NOT NULL DEFAULT 'full',
published_at_ms BIGINT,
duration_ms BIGINT,
audio_url TEXT NOT NULL,
mime TEXT,
byte_len BIGINT,
image_url TEXT,
link TEXT,
source TEXT NOT NULL CHECK (source IN ('feed', 'sync')),
in_feed BOOLEAN NOT NULL DEFAULT FALSE,
search_norm TEXT,
first_seen_at_ms BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL,
PRIMARY KEY (user_id, episode_id),
FOREIGN KEY (user_id, podcast_id)
REFERENCES furumusic__podcast (user_id, podcast_id) ON DELETE CASCADE
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_episode_date
ON furumusic__podcast_episode (user_id, podcast_id, published_at_ms DESC)",
"CREATE INDEX IF NOT EXISTS idx_podcast_episode_season
ON furumusic__podcast_episode (user_id, podcast_id, season, episode_number)",
// Seasons are derived from the episodes; a view cannot go stale.
// `season` NULL groups the episodes without `itunes:season`.
"CREATE OR REPLACE VIEW furumusic__podcast_season AS
SELECT user_id, podcast_id, season,
COUNT(*) AS episode_count,
MIN(published_at_ms) AS first_published_at_ms,
MAX(published_at_ms) AS last_published_at_ms
FROM furumusic__podcast_episode
GROUP BY user_id, podcast_id, season",
// Synced registers. No foreign keys to the catalog: operations for
// podcasts or episodes this account has not fetched are kept as they
// arrive, in any order.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_subscription (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
subscribed BOOLEAN NOT NULL,
podcast_json JSONB NOT NULL,
generation BIGINT NOT NULL CHECK (generation >= 0),
hlc_ms BIGINT NOT NULL,
op_id TEXT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
"CREATE TABLE IF NOT EXISTS furumusic__podcast_share (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
shared BOOLEAN NOT NULL,
generation BIGINT NOT NULL CHECK (generation >= 0),
hlc_ms BIGINT NOT NULL,
op_id TEXT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
"CREATE TABLE IF NOT EXISTS furumusic__podcast_progress (
user_id BIGINT NOT NULL,
episode_id TEXT NOT NULL,
podcast_id TEXT NOT NULL,
position_ms BIGINT NOT NULL,
status TEXT NOT NULL
CHECK (status IN ('not_started', 'in_progress', 'completed')),
generation BIGINT NOT NULL CHECK (generation >= 0),
session_id TEXT NOT NULL,
hlc_ms BIGINT NOT NULL,
device_id TEXT NOT NULL,
op_id TEXT NOT NULL,
PRIMARY KEY (user_id, episode_id)
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_progress_podcast
ON furumusic__podcast_progress (user_id, podcast_id)",
// Grow-only set of listening sessions.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_listen_session (
user_id BIGINT NOT NULL,
session_id TEXT NOT NULL,
episode_id TEXT NOT NULL,
podcast_id TEXT NOT NULL,
device_id TEXT NOT NULL,
started_at_ms BIGINT NOT NULL,
ended_at_ms BIGINT NOT NULL,
start_position_ms BIGINT NOT NULL,
end_position_ms BIGINT NOT NULL,
listened_ms BIGINT NOT NULL,
completed BOOLEAN NOT NULL,
PRIMARY KEY (user_id, session_id)
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_listen_session_episode
ON furumusic__podcast_listen_session (user_id, episode_id)",
"CREATE INDEX IF NOT EXISTS idx_podcast_listen_session_end
ON furumusic__podcast_listen_session (user_id, ended_at_ms DESC)",
// Local-only fetch state: never part of sync output.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_fetch_state (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
etag TEXT,
last_modified TEXT,
checked_at_ms BIGINT,
last_success_at_ms BIGINT,
next_check_at_ms BIGINT,
last_error TEXT,
consecutive_failures BIGINT NOT NULL DEFAULT 0,
cover_url TEXT,
cover_path TEXT,
PRIMARY KEY (user_id, podcast_id),
FOREIGN KEY (user_id, podcast_id)
REFERENCES furumusic__podcast (user_id, podcast_id) ON DELETE CASCADE
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_fetch_state_due
ON furumusic__podcast_fetch_state (next_check_at_ms)",
];
/// Backwards operation of `m_0056`.
pub(crate) const DROP_PODCAST_STORE: &[&str] = &[
"DROP VIEW IF EXISTS furumusic__podcast_season",
"DROP TABLE IF EXISTS furumusic__podcast_fetch_state",
"DROP TABLE IF EXISTS furumusic__podcast_listen_session",
"DROP TABLE IF EXISTS furumusic__podcast_progress",
"DROP TABLE IF EXISTS furumusic__podcast_share",
"DROP TABLE IF EXISTS furumusic__podcast_subscription",
"DROP TABLE IF EXISTS furumusic__podcast_episode",
"DROP TABLE IF EXISTS furumusic__podcast_feed_alias",
"DROP TABLE IF EXISTS furumusic__podcast",
];
/// Placeholder in [`IMPORT_PODCAST_RELAY`] for the account filter: the
/// migration imports every account (`TRUE`), tests narrow it to one.
pub(crate) const USER_FILTER: &str = "{user_filter}";
/// Largest generation the `BIGINT` columns hold; larger records are skipped
/// like the store rejects them.
const GENERATION_FITS: &str = "(record_json->>'generation')::numeric <= 9223372036854775807";
/// Import of the opaque relay state kept by 0.13.0 (`m_0057`).
///
/// `furumusic__fed_podcast_relay` holds the contract registers as JSON,
/// already validated and merged with the `music_dht::podcast` rules when they
/// were received, so they are copied as they are. The catalog rows the store
/// would have created from the episode references of received operations
/// (`PodcastEpisodeRef`, `source = 'sync'`) are rebuilt from the operation
/// log. Every statement is idempotent (`ON CONFLICT DO NOTHING`); the relay
/// table is kept (see `docs/administration.md`).
pub(crate) const IMPORT_PODCAST_RELAY: &[&str] = &[
// Catalog rows from the subscription registers.
"INSERT INTO furumusic__podcast
(user_id, podcast_id, feed_url, title, author, description, link, image_url,
meta_from_feed, created_at_ms, updated_at_ms)
SELECT user_id, record_key,
record_json->'podcast'->>'feed_url', record_json->'podcast'->>'title',
record_json->'podcast'->>'author', record_json->'podcast'->>'description',
record_json->'podcast'->>'link', record_json->'podcast'->>'image_url',
FALSE, updated_at_ms, updated_at_ms
FROM furumusic__fed_podcast_relay
WHERE section = 'subscription' AND {user_filter}
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_subscription
(user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
SELECT user_id, record_key, (record_json->>'subscribed')::boolean,
record_json->'podcast', (record_json->>'generation')::bigint,
(record_json->>'hlc_ms')::bigint, record_json->>'op_id'
FROM furumusic__fed_podcast_relay
WHERE section = 'subscription' AND {generation_fits} AND {user_filter}
ON CONFLICT DO NOTHING",
// The relayed subscription already carries the bounded, merged alias set.
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
SELECT user_id, record_key, jsonb_array_elements_text(record_json->'feed_aliases')
FROM furumusic__fed_podcast_relay
WHERE section = 'subscription' AND {user_filter}
AND jsonb_typeof(record_json->'feed_aliases') = 'array'
ON CONFLICT DO NOTHING",
// Aliases received before any subscription register of their podcast.
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
SELECT user_id, record_key, jsonb_array_elements_text(record_json)
FROM furumusic__fed_podcast_relay
WHERE section = 'aliases' AND {user_filter} AND jsonb_typeof(record_json) = 'array'
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_share
(user_id, podcast_id, shared, generation, hlc_ms, op_id)
SELECT user_id, record_key, (record_json->>'shared')::boolean,
(record_json->>'generation')::bigint, (record_json->>'hlc_ms')::bigint,
record_json->>'op_id'
FROM furumusic__fed_podcast_relay
WHERE section = 'share' AND {generation_fits} AND {user_filter}
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_progress
(user_id, episode_id, podcast_id, position_ms, status, generation, session_id,
hlc_ms, device_id, op_id)
SELECT user_id, record_key, record_json->>'podcast_id',
(record_json->>'position_ms')::bigint, record_json->>'status',
(record_json->>'generation')::bigint, record_json->>'session_id',
(record_json->>'hlc_ms')::bigint, record_json->>'device_id',
record_json->>'op_id'
FROM furumusic__fed_podcast_relay
WHERE section = 'progress' AND {generation_fits} AND {user_filter}
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_listen_session
(user_id, session_id, episode_id, podcast_id, device_id, started_at_ms,
ended_at_ms, start_position_ms, end_position_ms, listened_ms, completed)
SELECT user_id, record_key, record_json->>'episode_id', record_json->>'podcast_id',
record_json->>'device_id', (record_json->>'started_at_ms')::bigint,
(record_json->>'ended_at_ms')::bigint,
(record_json->>'start_position_ms')::bigint,
(record_json->>'end_position_ms')::bigint,
(record_json->>'listened_ms')::bigint, (record_json->>'completed')::boolean
FROM furumusic__fed_podcast_relay
WHERE section = 'session' AND {user_filter}
ON CONFLICT DO NOTHING",
// Catalog rows for podcasts only known from episode references.
"INSERT INTO furumusic__podcast
(user_id, podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms)
SELECT DISTINCT ON (user_id, payload_json->'episode'->>'podcast_id')
user_id, payload_json->'episode'->>'podcast_id',
payload_json->'episode'->>'feed_url', payload_json->'episode'->>'podcast_title',
FALSE, received_at_ms, received_at_ms
FROM furumusic__fed_sync_ops
WHERE kind IN ('podcast_progress_set', 'podcast_listen_recorded')
AND jsonb_typeof(payload_json->'episode') = 'object' AND {user_filter}
ORDER BY user_id, payload_json->'episode'->>'podcast_id', hlc_ms DESC, op_id DESC
ON CONFLICT DO NOTHING",
// Episodes only known from references (the newest reference wins, like
// the first one would have for the store: feed metadata replaces it on
// the first refresh anyway).
"INSERT INTO furumusic__podcast_episode
(user_id, episode_id, podcast_id, title, season, episode_number, episode_type,
published_at_ms, duration_ms, audio_url, mime, image_url, source, in_feed,
first_seen_at_ms, updated_at_ms)
SELECT DISTINCT ON (user_id, payload_json->'episode'->>'episode_id')
user_id, payload_json->'episode'->>'episode_id',
payload_json->'episode'->>'podcast_id', payload_json->'episode'->>'episode_title',
(payload_json->'episode'->>'season')::bigint,
(payload_json->'episode'->>'episode_number')::bigint, 'full',
(payload_json->'episode'->>'published_at_ms')::bigint,
(payload_json->'episode'->>'duration_ms')::bigint,
payload_json->'episode'->>'audio_url', payload_json->'episode'->>'mime',
payload_json->'episode'->>'image_url', 'sync', FALSE,
received_at_ms, received_at_ms
FROM furumusic__fed_sync_ops
WHERE kind IN ('podcast_progress_set', 'podcast_listen_recorded')
AND jsonb_typeof(payload_json->'episode') = 'object' AND {user_filter}
ORDER BY user_id, payload_json->'episode'->>'episode_id', hlc_ms DESC, op_id DESC
ON CONFLICT DO NOTHING",
];
/// The import statements with `filter` as the account condition.
pub(crate) fn import_statements(filter: &str) -> Vec<String> {
IMPORT_PODCAST_RELAY
.iter()
.map(|sql| {
sql.replace("{generation_fits}", GENERATION_FITS)
.replace(USER_FILTER, filter)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn import_statements_fill_every_placeholder() {
for sql in import_statements("user_id = 7") {
assert!(!sql.contains('{'), "{sql}");
assert!(sql.contains("user_id = 7"), "{sql}");
assert!(sql.contains("ON CONFLICT DO NOTHING"), "{sql}");
}
}
#[test]
fn every_store_table_is_dropped_backwards() {
for sql in CREATE_PODCAST_STORE {
let Some(rest) = sql
.strip_prefix("CREATE TABLE IF NOT EXISTS ")
.or_else(|| sql.strip_prefix("CREATE OR REPLACE VIEW "))
else {
continue;
};
let name = rest.split_whitespace().next().unwrap();
assert!(
DROP_PODCAST_STORE
.iter()
.any(|drop| drop.ends_with(&format!(" {name}"))),
"{name}"
);
}
}
}
+81
View File
@@ -0,0 +1,81 @@
//! Pure unit tests of the podcast store helpers (no database).
use music_dht::podcast::ProgressStatus;
use super::*;
#[test]
fn search_text_never_matches_across_title_and_description() {
let norm = episode_search_norm("Hello, World!", Some("WORLD peace"));
assert_eq!(norm, "hello world\nworld peace");
// A needle is itself normalized, so it never contains the separator.
let needle = music_dht::normalize_name("world world");
assert!(!norm.contains(&needle));
assert!(norm.contains(&music_dht::normalize_name("WORLD PEACE")));
assert_eq!(episode_search_norm("Only", None), "only\n");
}
#[test]
fn progress_status_round_trips_through_sql() {
for status in [
ProgressStatus::NotStarted,
ProgressStatus::InProgress,
ProgressStatus::Completed,
] {
assert_eq!(status_from_sql(status_to_sql(status)), status);
}
// The CHECK constraint admits exactly these values.
for sql in schema::CREATE_PODCAST_STORE {
if sql.contains("CREATE TABLE IF NOT EXISTS furumusic__podcast_progress (") {
for value in ["not_started", "in_progress", "completed"] {
assert!(sql.contains(&format!("'{value}'")));
}
}
}
}
#[test]
fn generations_beyond_bigint_are_rejected() {
assert_eq!(generation_to_sql(0).unwrap(), 0);
assert_eq!(generation_to_sql(i64::MAX as u64).unwrap(), i64::MAX);
assert!(matches!(
generation_to_sql(i64::MAX as u64 + 1),
Err(PodcastStoreError::Invalid(_))
));
assert_eq!(generation_from_sql(-1), 0);
}
#[test]
fn contract_numbers_outside_u32_are_dropped() {
assert_eq!(u32_from_sql(Some(7)), Some(7));
assert_eq!(u32_from_sql(Some(-1)), None);
assert_eq!(u32_from_sql(Some(i64::from(u32::MAX) + 1)), None);
assert_eq!(u32_from_sql(None), None);
}
#[test]
fn progress_columns_are_prefixed() {
assert_eq!(
progress_columns("g", "g_"),
"g.episode_id AS g_episode_id, g.podcast_id AS g_podcast_id, \
g.position_ms AS g_position_ms, g.status AS g_status, \
g.generation AS g_generation, g.session_id AS g_session_id, \
g.hlc_ms AS g_hlc_ms, g.device_id AS g_device_id, g.op_id AS g_op_id"
);
}
#[test]
fn queries_sort_undated_and_unnumbered_episodes_last() {
for sort in [
EpisodeSort::NewestFirst,
EpisodeSort::OldestFirst,
EpisodeSort::EpisodeNumber,
EpisodeSort::EpisodeNumberDesc,
] {
let order = queries::order_by_for_tests(sort);
assert!(
order.starts_with("e.published_at_ms IS NULL") || order.starts_with("e.season IS NULL")
);
assert!(order.ends_with("e.episode_id"), "{order}");
}
}