forked from Furumi/furumi-web
342 lines
13 KiB
Rust
342 lines
13 KiB
Rust
//! PostgreSQL podcast store of the web server.
|
|
//!
|
|
//! [`PodcastStore`] implements the podcast contract of `music_dht::podcast`
|
|
//! (normative description: `PODCASTS.md` of frid) on PostgreSQL, with the
|
|
//! same invariants as the SQLite reference store of `furumi-library`. Each
|
|
//! web account is one virtual trusted device, so every table is keyed by
|
|
//! `user_id`. The store owns no HTTP, scheduling or UI: RSS fetching and the
|
|
//! player use it.
|
|
//!
|
|
//! # Tables
|
|
//!
|
|
//! | Table | Key | Content | Synced |
|
|
//! | --- | --- | --- | --- |
|
|
//! | `furumusic__podcast` | `podcast_id` | local channel metadata, `feed_url` = URL this account fetches | no (catalog) |
|
|
//! | `furumusic__podcast_feed_alias` | `(podcast_id, feed_url)` | bounded grow-only alias set | yes, inside subscription records and `podcast_feed_aliases_added` |
|
|
//! | `furumusic__podcast_episode` | `episode_id` | `EpisodeMeta`, `source` (`feed`/`sync`), `in_feed` | no (catalog) |
|
|
//! | `furumusic__podcast_season` (view) | `(podcast_id, season)` | derived from the episodes | no |
|
|
//! | `furumusic__podcast_subscription` | `podcast_id` | subscription register | yes |
|
|
//! | `furumusic__podcast_share` | `podcast_id` | share register | yes |
|
|
//! | `furumusic__podcast_progress` | `episode_id` | progress register | yes |
|
|
//! | `furumusic__podcast_listen_session` | `session_id` | grow-only set of sessions | yes |
|
|
//! | `furumusic__podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, check times, errors, cover cache | **never** |
|
|
//! | `furumusic__podcast_share_confirmation` | `podcast_id` | privacy confirmation of a shared card (share decision, card URLs) | **never** |
|
|
//!
|
|
//! # One apply path
|
|
//!
|
|
//! Remote operations ([`PodcastStore::apply_remote_op`]), snapshot sections
|
|
//! ([`PodcastStore::apply_snapshot`]) and local actions all end in the same
|
|
//! functions: the `SyncOpWire` materializers of `music_dht::device_sync` and
|
|
//! the `merge_*` rules of `music_dht::podcast`. Applying is therefore
|
|
//! idempotent and independent of delivery order, and a snapshot never
|
|
//! replaces newer state.
|
|
//!
|
|
//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...)
|
|
//! run in one transaction: they read the registers, append the operation to
|
|
//! the account's device-sync log with its stamp
|
|
//! ([`crate::federation::devices::append_local_op`]: next `seq`, `hlc_ms`,
|
|
//! `op_id = <device>:<seq>`) and apply exactly that operation. Either both
|
|
//! the log and the registers change or neither does. Every write
|
|
//! transaction takes a per-account advisory lock, so concurrent writers of
|
|
//! one account serialize.
|
|
//!
|
|
//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the
|
|
//! alias set only; they never write a register, progress or a session.
|
|
//!
|
|
//! # Runtime
|
|
//!
|
|
//! * `net` — the SSRF-guarded HTTP client used for every feed and audio
|
|
//! request (public addresses only, checked per connection and per
|
|
//! redirect hop).
|
|
//! * `fetch` and [`runner`] — conditional feed downloads and their schedule
|
|
//! (hourly checks, exponential backoff, one request per feed URL), run by
|
|
//! the `podcast_refresh` job and the manual refresh endpoint.
|
|
//! * `proxy` and [`http`] — the optional audio proxy and the
|
|
//! `/api/podcasts` endpoints. Audio is streamed from the publisher, never
|
|
//! stored.
|
|
//! * [`api`] — what the endpoints of the podcast UI and the web player do
|
|
//! (subscribe by URL, lists, explicit progress actions, sharing, playback
|
|
//! sessions and progress reports), independent of HTTP.
|
|
//! * [`discovery`] — public discovery: which podcasts the server node
|
|
//! publishes to `music_dht::podcast_dht` for the accounts that explicitly
|
|
//! shared them, the local privacy confirmations and the reconciler that
|
|
//! calls `PodcastDht::sync_shares`.
|
|
//! * [`search`] — public podcasts found by the federated search, as seen by
|
|
//! the searching account ("already subscribed"), and the preview of a
|
|
//! found podcast's feed before subscribing.
|
|
|
|
mod actions;
|
|
pub(crate) mod api;
|
|
mod apply;
|
|
pub(crate) mod discovery;
|
|
mod fetch;
|
|
pub(crate) mod http;
|
|
mod limits;
|
|
mod net;
|
|
mod proxy;
|
|
mod queries;
|
|
mod refresh;
|
|
pub(crate) mod runner;
|
|
pub(crate) mod schema;
|
|
pub(crate) mod search;
|
|
|
|
#[cfg(test)]
|
|
mod discovery_tests;
|
|
#[cfg(test)]
|
|
mod fetch_tests;
|
|
#[cfg(test)]
|
|
mod net_tests;
|
|
#[cfg(test)]
|
|
pub(crate) mod postgres_tests;
|
|
#[cfg(test)]
|
|
mod proxy_tests;
|
|
#[cfg(test)]
|
|
mod search_tests;
|
|
#[cfg(test)]
|
|
pub(crate) mod test_server;
|
|
#[cfg(test)]
|
|
mod tests;
|
|
|
|
use std::fmt;
|
|
|
|
use anyhow::Result;
|
|
use music_dht::podcast::{
|
|
EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord,
|
|
ProgressStatus,
|
|
};
|
|
use sqlx::Row;
|
|
use sqlx::postgres::PgRow;
|
|
|
|
pub(crate) use actions::{CheckpointOutcome, ProgressSession};
|
|
pub(crate) use apply::ApplyOutcome;
|
|
pub(crate) use fetch::FeedFetcher;
|
|
pub(crate) use queries::{
|
|
EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter, PodcastSummary,
|
|
SeasonFilter, SeasonSummary,
|
|
};
|
|
pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome};
|
|
|
|
/// Typed errors of [`PodcastStore`]; other failures are database errors.
|
|
///
|
|
/// They are returned inside `anyhow::Error`; use `downcast_ref` to match.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) enum PodcastStoreError {
|
|
/// The podcast is unknown to this account.
|
|
UnknownPodcast(PodcastId),
|
|
/// The action needs an active subscription.
|
|
NotSubscribed(PodcastId),
|
|
/// Input violates the podcast contract.
|
|
Invalid(String),
|
|
/// A new subscription would exceed [`MAX_SUBSCRIPTIONS_PER_ACCOUNT`].
|
|
SubscriptionLimit(usize),
|
|
/// A new episode reference from the browser would exceed
|
|
/// [`MAX_REFERENCE_EPISODES_PER_ACCOUNT`].
|
|
ReferenceLimit(usize),
|
|
}
|
|
|
|
/// Active subscriptions an account may add locally (subscriptions synced
|
|
/// from the account's own devices are always applied). Every subscription
|
|
/// is a feed the server downloads about once an hour.
|
|
pub(crate) const MAX_SUBSCRIPTIONS_PER_ACCOUNT: usize = 1000;
|
|
|
|
/// Episodes known only from references (`source = 'sync'`: not from one of
|
|
/// the account's feeds) beyond which the browser cannot add another one
|
|
/// through `POST /api/podcasts/episodes/{id}/playback`. References in
|
|
/// operations of the account's own devices are always applied; this bounds
|
|
/// what a signed-in user can insert directly, one request at a time.
|
|
pub(crate) const MAX_REFERENCE_EPISODES_PER_ACCOUNT: usize = 10_000;
|
|
|
|
impl fmt::Display for PodcastStoreError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"),
|
|
Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"),
|
|
Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"),
|
|
Self::SubscriptionLimit(limit) => {
|
|
write!(f, "subscription limit reached ({limit} podcasts)")
|
|
}
|
|
Self::ReferenceLimit(limit) => write!(
|
|
f,
|
|
"too many episodes from outside your feeds ({limit}); subscribe to the podcast to play it"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for PodcastStoreError {}
|
|
|
|
impl From<PodcastError> for PodcastStoreError {
|
|
fn from(error: PodcastError) -> Self {
|
|
Self::Invalid(error.to_string())
|
|
}
|
|
}
|
|
|
|
pub(crate) fn invalid(reason: impl ToString) -> anyhow::Error {
|
|
PodcastStoreError::Invalid(reason.to_string()).into()
|
|
}
|
|
|
|
/// PostgreSQL store of podcast catalog, personal state and fetch state.
|
|
///
|
|
/// Cheap to clone (the pool is shared). See the module documentation.
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) struct PodcastStore {
|
|
pool: sqlx::PgPool,
|
|
}
|
|
|
|
impl PodcastStore {
|
|
pub(crate) fn new(pool: sqlx::PgPool) -> Self {
|
|
Self { pool }
|
|
}
|
|
|
|
/// The pool the store runs on.
|
|
pub(crate) fn pool(&self) -> &sqlx::PgPool {
|
|
&self.pool
|
|
}
|
|
|
|
/// Starts a write transaction holding the account's podcast lock.
|
|
async fn begin_write(
|
|
&self,
|
|
user_id: i64,
|
|
) -> Result<sqlx::Transaction<'static, sqlx::Postgres>> {
|
|
let mut tx = self.pool.begin().await?;
|
|
lock_account(&mut tx, user_id).await?;
|
|
Ok(tx)
|
|
}
|
|
}
|
|
|
|
/// Serializes podcast writers of one account until the transaction ends.
|
|
pub(crate) async fn lock_account(conn: &mut sqlx::PgConnection, user_id: i64) -> Result<()> {
|
|
sqlx::query(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcasts:' || $1::text, 0))",
|
|
)
|
|
.bind(user_id)
|
|
.execute(conn)
|
|
.await?;
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn now_ms() -> i64 {
|
|
chrono::Utc::now().timestamp_millis()
|
|
}
|
|
|
|
/// Converts a contract generation to the `BIGINT` column value.
|
|
///
|
|
/// Generations above `i64::MAX` cannot be stored; records carrying one are
|
|
/// rejected (no conforming client produces them).
|
|
pub(crate) fn generation_to_sql(generation: u64) -> Result<i64, PodcastStoreError> {
|
|
i64::try_from(generation)
|
|
.map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string()))
|
|
}
|
|
|
|
pub(crate) fn generation_from_sql(value: i64) -> u64 {
|
|
u64::try_from(value).unwrap_or(0)
|
|
}
|
|
|
|
pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str {
|
|
match status {
|
|
ProgressStatus::NotStarted => "not_started",
|
|
ProgressStatus::InProgress => "in_progress",
|
|
ProgressStatus::Completed => "completed",
|
|
}
|
|
}
|
|
|
|
pub(crate) fn status_from_sql(value: &str) -> ProgressStatus {
|
|
match value {
|
|
"in_progress" => ProgressStatus::InProgress,
|
|
"completed" => ProgressStatus::Completed,
|
|
_ => ProgressStatus::NotStarted,
|
|
}
|
|
}
|
|
|
|
/// `u32` contract numbers (season, episode number) from a `BIGINT` column.
|
|
pub(crate) fn u32_from_sql(value: Option<i64>) -> Option<u32> {
|
|
value.and_then(|value| u32::try_from(value).ok())
|
|
}
|
|
|
|
fn decode_error(message: impl ToString) -> sqlx::Error {
|
|
sqlx::Error::Decode(Box::new(PodcastStoreError::Invalid(message.to_string())))
|
|
}
|
|
|
|
/// Parses a stored id; stored ids are validated on write.
|
|
pub(crate) fn podcast_id_col(row: &PgRow, column: &str) -> sqlx::Result<PodcastId> {
|
|
PodcastId::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
|
|
}
|
|
|
|
pub(crate) fn episode_id_col(row: &PgRow, column: &str) -> sqlx::Result<EpisodeId> {
|
|
EpisodeId::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
|
|
}
|
|
|
|
pub(crate) fn feed_url_col(row: &PgRow, column: &str) -> sqlx::Result<FeedUrl> {
|
|
FeedUrl::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
|
|
}
|
|
|
|
pub(crate) fn podcast_meta_col(row: &PgRow, column: &str) -> sqlx::Result<PodcastMeta> {
|
|
serde_json::from_value(row.try_get::<serde_json::Value, _>(column)?).map_err(decode_error)
|
|
}
|
|
|
|
/// Progress register columns of table alias `alias`, named `{prefix}...`.
|
|
pub(crate) fn progress_columns(alias: &str, prefix: &str) -> String {
|
|
[
|
|
"episode_id",
|
|
"podcast_id",
|
|
"position_ms",
|
|
"status",
|
|
"generation",
|
|
"session_id",
|
|
"hlc_ms",
|
|
"device_id",
|
|
"op_id",
|
|
]
|
|
.iter()
|
|
.map(|column| format!("{alias}.{column} AS {prefix}{column}"))
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
}
|
|
|
|
/// Progress register selected with [`progress_columns`].
|
|
pub(crate) fn progress_from_row(row: &PgRow, prefix: &str) -> sqlx::Result<ProgressRecord> {
|
|
let column = |name: &str| format!("{prefix}{name}");
|
|
Ok(ProgressRecord {
|
|
episode_id: episode_id_col(row, &column("episode_id"))?,
|
|
podcast_id: podcast_id_col(row, &column("podcast_id"))?,
|
|
position_ms: row.try_get(column("position_ms").as_str())?,
|
|
status: status_from_sql(&row.try_get::<String, _>(column("status").as_str())?),
|
|
generation: generation_from_sql(row.try_get(column("generation").as_str())?),
|
|
session_id: row.try_get(column("session_id").as_str())?,
|
|
hlc_ms: row.try_get(column("hlc_ms").as_str())?,
|
|
device_id: row.try_get(column("device_id").as_str())?,
|
|
op_id: row.try_get(column("op_id").as_str())?,
|
|
})
|
|
}
|
|
|
|
/// Listening session columns of table alias `s` (plain names).
|
|
pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \
|
|
s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \
|
|
s.completed";
|
|
|
|
pub(crate) fn session_from_row(row: &PgRow) -> sqlx::Result<ListenSession> {
|
|
Ok(ListenSession {
|
|
session_id: row.try_get("session_id")?,
|
|
episode_id: episode_id_col(row, "episode_id")?,
|
|
podcast_id: podcast_id_col(row, "podcast_id")?,
|
|
device_id: row.try_get("device_id")?,
|
|
started_at_ms: row.try_get("started_at_ms")?,
|
|
ended_at_ms: row.try_get("ended_at_ms")?,
|
|
start_position_ms: row.try_get("start_position_ms")?,
|
|
end_position_ms: row.try_get("end_position_ms")?,
|
|
listened_ms: row.try_get("listened_ms")?,
|
|
completed: row.try_get("completed")?,
|
|
})
|
|
}
|
|
|
|
/// Normalized search text of an episode: `normalize_name` of the title and
|
|
/// of the description, separated by a newline (the normalization never
|
|
/// produces one, so a needle cannot match across the two).
|
|
pub(crate) fn episode_search_norm(title: &str, description: Option<&str>) -> String {
|
|
format!(
|
|
"{}\n{}",
|
|
music_dht::normalize_name(title),
|
|
music_dht::normalize_name(description.unwrap_or_default())
|
|
)
|
|
}
|