//! PostgreSQL podcast store of the web server. //! //! [`PodcastStore`] implements the podcast contract of `music_dht::podcast` //! (normative description: `PODCASTS.md` of frid) on PostgreSQL, with the //! same invariants as the SQLite reference store of `furumi-library`. Each //! web account is one virtual trusted device, so every table is keyed by //! `user_id`. The store owns no HTTP, scheduling or UI: RSS fetching and the //! player use it. //! //! # Tables //! //! | Table | Key | Content | Synced | //! | --- | --- | --- | --- | //! | `furumusic__podcast` | `podcast_id` | local channel metadata, `feed_url` = URL this account fetches | no (catalog) | //! | `furumusic__podcast_feed_alias` | `(podcast_id, feed_url)` | bounded grow-only alias set | yes, inside subscription records and `podcast_feed_aliases_added` | //! | `furumusic__podcast_episode` | `episode_id` | `EpisodeMeta`, `source` (`feed`/`sync`), `in_feed` | no (catalog) | //! | `furumusic__podcast_season` (view) | `(podcast_id, season)` | derived from the episodes | no | //! | `furumusic__podcast_subscription` | `podcast_id` | subscription register | yes | //! | `furumusic__podcast_share` | `podcast_id` | share register | yes | //! | `furumusic__podcast_progress` | `episode_id` | progress register | yes | //! | `furumusic__podcast_listen_session` | `session_id` | grow-only set of sessions | yes | //! | `furumusic__podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, check times, errors, cover cache | **never** | //! | `furumusic__podcast_share_confirmation` | `podcast_id` | privacy confirmation of a shared card (share decision, card URLs) | **never** | //! //! # One apply path //! //! Remote operations ([`PodcastStore::apply_remote_op`]), snapshot sections //! ([`PodcastStore::apply_snapshot`]) and local actions all end in the same //! functions: the `SyncOpWire` materializers of `music_dht::device_sync` and //! the `merge_*` rules of `music_dht::podcast`. Applying is therefore //! idempotent and independent of delivery order, and a snapshot never //! replaces newer state. //! //! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...) //! run in one transaction: they read the registers, append the operation to //! the account's device-sync log with its stamp //! ([`crate::federation::devices::append_local_op`]: next `seq`, `hlc_ms`, //! `op_id = :`) and apply exactly that operation. Either both //! the log and the registers change or neither does. Every write //! transaction takes a per-account advisory lock, so concurrent writers of //! one account serialize. //! //! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the //! alias set only; they never write a register, progress or a session. //! //! # Runtime //! //! * `net` — the SSRF-guarded HTTP client used for every feed and audio //! request (public addresses only, checked per connection and per //! redirect hop). //! * `fetch` and [`runner`] — conditional feed downloads and their schedule //! (hourly checks, exponential backoff, one request per feed URL), run by //! the `podcast_refresh` job and the manual refresh endpoint. //! * `proxy` and [`http`] — the optional audio proxy and the //! `/api/podcasts` endpoints. Audio is streamed from the publisher, never //! stored. //! * [`api`] — what the endpoints of the podcast UI and the web player do //! (subscribe by URL, lists, explicit progress actions, sharing, playback //! sessions and progress reports), independent of HTTP. //! * [`discovery`] — public discovery: which podcasts the server node //! publishes to `music_dht::podcast_dht` for the accounts that explicitly //! shared them, the local privacy confirmations and the reconciler that //! calls `PodcastDht::sync_shares`. //! * [`search`] — public podcasts found by the federated search, as seen by //! the searching account ("already subscribed"), and the preview of a //! found podcast's feed before subscribing. mod actions; pub(crate) mod api; mod apply; pub(crate) mod discovery; mod fetch; pub(crate) mod http; mod limits; mod net; mod proxy; mod queries; mod refresh; pub(crate) mod runner; pub(crate) mod schema; pub(crate) mod search; #[cfg(test)] mod discovery_tests; #[cfg(test)] mod fetch_tests; #[cfg(test)] mod net_tests; #[cfg(test)] pub(crate) mod postgres_tests; #[cfg(test)] mod proxy_tests; #[cfg(test)] mod search_tests; #[cfg(test)] pub(crate) mod test_server; #[cfg(test)] mod tests; use std::fmt; use anyhow::Result; use music_dht::podcast::{ EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord, ProgressStatus, }; use sqlx::Row; use sqlx::postgres::PgRow; pub(crate) use actions::{CheckpointOutcome, ProgressSession}; pub(crate) use apply::ApplyOutcome; pub(crate) use fetch::FeedFetcher; pub(crate) use queries::{ EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter, PodcastSummary, SeasonFilter, SeasonSummary, }; pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome}; /// Typed errors of [`PodcastStore`]; other failures are database errors. /// /// They are returned inside `anyhow::Error`; use `downcast_ref` to match. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum PodcastStoreError { /// The podcast is unknown to this account. UnknownPodcast(PodcastId), /// The action needs an active subscription. NotSubscribed(PodcastId), /// Input violates the podcast contract. Invalid(String), /// A new subscription would exceed [`MAX_SUBSCRIPTIONS_PER_ACCOUNT`]. SubscriptionLimit(usize), /// A new episode reference from the browser would exceed /// [`MAX_REFERENCE_EPISODES_PER_ACCOUNT`]. ReferenceLimit(usize), } /// Active subscriptions an account may add locally (subscriptions synced /// from the account's own devices are always applied). Every subscription /// is a feed the server downloads about once an hour. pub(crate) const MAX_SUBSCRIPTIONS_PER_ACCOUNT: usize = 1000; /// Episodes known only from references (`source = 'sync'`: not from one of /// the account's feeds) beyond which the browser cannot add another one /// through `POST /api/podcasts/episodes/{id}/playback`. References in /// operations of the account's own devices are always applied; this bounds /// what a signed-in user can insert directly, one request at a time. pub(crate) const MAX_REFERENCE_EPISODES_PER_ACCOUNT: usize = 10_000; impl fmt::Display for PodcastStoreError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"), Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"), Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"), Self::SubscriptionLimit(limit) => { write!(f, "subscription limit reached ({limit} podcasts)") } Self::ReferenceLimit(limit) => write!( f, "too many episodes from outside your feeds ({limit}); subscribe to the podcast to play it" ), } } } impl std::error::Error for PodcastStoreError {} impl From for PodcastStoreError { fn from(error: PodcastError) -> Self { Self::Invalid(error.to_string()) } } pub(crate) fn invalid(reason: impl ToString) -> anyhow::Error { PodcastStoreError::Invalid(reason.to_string()).into() } /// PostgreSQL store of podcast catalog, personal state and fetch state. /// /// Cheap to clone (the pool is shared). See the module documentation. #[derive(Debug, Clone)] pub(crate) struct PodcastStore { pool: sqlx::PgPool, } impl PodcastStore { pub(crate) fn new(pool: sqlx::PgPool) -> Self { Self { pool } } /// The pool the store runs on. pub(crate) fn pool(&self) -> &sqlx::PgPool { &self.pool } /// Starts a write transaction holding the account's podcast lock. async fn begin_write( &self, user_id: i64, ) -> Result> { let mut tx = self.pool.begin().await?; lock_account(&mut tx, user_id).await?; Ok(tx) } } /// Serializes podcast writers of one account until the transaction ends. pub(crate) async fn lock_account(conn: &mut sqlx::PgConnection, user_id: i64) -> Result<()> { sqlx::query( "SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcasts:' || $1::text, 0))", ) .bind(user_id) .execute(conn) .await?; Ok(()) } pub(crate) fn now_ms() -> i64 { chrono::Utc::now().timestamp_millis() } /// Converts a contract generation to the `BIGINT` column value. /// /// Generations above `i64::MAX` cannot be stored; records carrying one are /// rejected (no conforming client produces them). pub(crate) fn generation_to_sql(generation: u64) -> Result { i64::try_from(generation) .map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string())) } pub(crate) fn generation_from_sql(value: i64) -> u64 { u64::try_from(value).unwrap_or(0) } pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str { match status { ProgressStatus::NotStarted => "not_started", ProgressStatus::InProgress => "in_progress", ProgressStatus::Completed => "completed", } } pub(crate) fn status_from_sql(value: &str) -> ProgressStatus { match value { "in_progress" => ProgressStatus::InProgress, "completed" => ProgressStatus::Completed, _ => ProgressStatus::NotStarted, } } /// `u32` contract numbers (season, episode number) from a `BIGINT` column. pub(crate) fn u32_from_sql(value: Option) -> Option { value.and_then(|value| u32::try_from(value).ok()) } fn decode_error(message: impl ToString) -> sqlx::Error { sqlx::Error::Decode(Box::new(PodcastStoreError::Invalid(message.to_string()))) } /// Parses a stored id; stored ids are validated on write. pub(crate) fn podcast_id_col(row: &PgRow, column: &str) -> sqlx::Result { PodcastId::try_from(row.try_get::(column)?).map_err(decode_error) } pub(crate) fn episode_id_col(row: &PgRow, column: &str) -> sqlx::Result { EpisodeId::try_from(row.try_get::(column)?).map_err(decode_error) } pub(crate) fn feed_url_col(row: &PgRow, column: &str) -> sqlx::Result { FeedUrl::try_from(row.try_get::(column)?).map_err(decode_error) } pub(crate) fn podcast_meta_col(row: &PgRow, column: &str) -> sqlx::Result { serde_json::from_value(row.try_get::(column)?).map_err(decode_error) } /// Progress register columns of table alias `alias`, named `{prefix}...`. pub(crate) fn progress_columns(alias: &str, prefix: &str) -> String { [ "episode_id", "podcast_id", "position_ms", "status", "generation", "session_id", "hlc_ms", "device_id", "op_id", ] .iter() .map(|column| format!("{alias}.{column} AS {prefix}{column}")) .collect::>() .join(", ") } /// Progress register selected with [`progress_columns`]. pub(crate) fn progress_from_row(row: &PgRow, prefix: &str) -> sqlx::Result { let column = |name: &str| format!("{prefix}{name}"); Ok(ProgressRecord { episode_id: episode_id_col(row, &column("episode_id"))?, podcast_id: podcast_id_col(row, &column("podcast_id"))?, position_ms: row.try_get(column("position_ms").as_str())?, status: status_from_sql(&row.try_get::(column("status").as_str())?), generation: generation_from_sql(row.try_get(column("generation").as_str())?), session_id: row.try_get(column("session_id").as_str())?, hlc_ms: row.try_get(column("hlc_ms").as_str())?, device_id: row.try_get(column("device_id").as_str())?, op_id: row.try_get(column("op_id").as_str())?, }) } /// Listening session columns of table alias `s` (plain names). pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \ s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \ s.completed"; pub(crate) fn session_from_row(row: &PgRow) -> sqlx::Result { Ok(ListenSession { session_id: row.try_get("session_id")?, episode_id: episode_id_col(row, "episode_id")?, podcast_id: podcast_id_col(row, "podcast_id")?, device_id: row.try_get("device_id")?, started_at_ms: row.try_get("started_at_ms")?, ended_at_ms: row.try_get("ended_at_ms")?, start_position_ms: row.try_get("start_position_ms")?, end_position_ms: row.try_get("end_position_ms")?, listened_ms: row.try_get("listened_ms")?, completed: row.try_get("completed")?, }) } /// Normalized search text of an episode: `normalize_name` of the title and /// of the description, separated by a newline (the normalization never /// produces one, so a needle cannot match across the two). pub(crate) fn episode_search_norm(title: &str, description: Option<&str>) -> String { format!( "{}\n{}", music_dht::normalize_name(title), music_dht::normalize_name(description.unwrap_or_default()) ) }