diff --git a/src/devices.rs b/src/devices.rs index b6d04b7..034841e 100644 --- a/src/devices.rs +++ b/src/devices.rs @@ -13,6 +13,9 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{Context as _, Result}; +use furumi_library::podcasts::{ + OpStamp as PodcastOpStamp, PodcastLocalOp, StampSource as PodcastStampSource, +}; use music_dht::device_sync::{ListenEvent, ListenTrackMetadata}; use music_dht::{ByteStream, MusicDhtService, NetworkId, PeerTicket, SecretKey, StreamAcceptor}; use rusqlite::{Connection, OptionalExtension, params}; @@ -1325,6 +1328,30 @@ impl DeviceSync { Ok(()) } + /// Runs a `PodcastStore` local action with stamps reserved in this + /// device's sync log and appends the operations it returns with exactly + /// those stamps (`op_id`, `seq`, `hlc_ms`). + /// + /// The log connection stays locked from the first stamp until the + /// operations are appended, so no sync batch can read past a reserved + /// `seq` before its operation exists. Lock order is podcast store, then + /// this log: never call the podcast store while holding `self.conn`. + pub fn run_podcast_action( + &self, + action: impl FnOnce(&mut dyn PodcastStampSource) -> Result<(T, Vec)>, + ) -> Result { + let identity = self.ensure_identity()?; + let mut log = PodcastLogStamps { + conn: &self.conn, + guard: None, + device_id: identity.device_id, + reserved: BTreeMap::new(), + }; + let (value, ops) = action(&mut log)?; + log.append(ops)?; + Ok(value) + } + pub async fn sync_once( &self, service: Arc, @@ -2303,6 +2330,9 @@ impl DeviceSync { } fn ops_for_peer(&self, peer_device_id: &str) -> Result> { + // Podcast operations (`podcast_*`, protocol v4) stay in the log but + // are not sent yet: this client still announces protocol v3 and its + // wire types cannot carry them. Gated sending comes with sync v4. let conn = lock(&self.conn); let mut stmt = conn.prepare( "SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json @@ -2310,6 +2340,7 @@ impl DeviceSync { LEFT JOIN sync_peer_acks a ON a.peer_device_id = ?1 AND a.origin_device_id = o.origin_device_id WHERE o.seq > COALESCE(a.max_seq, 0) + AND substr(o.kind, 1, 8) != 'podcast_' AND ( o.kind != 'playback_command' OR ( @@ -2800,6 +2831,104 @@ impl DeviceSync { } } +/// Stamp source over the sync log for [`DeviceSync::run_podcast_action`]. +struct PodcastLogStamps<'a> { + conn: &'a std::sync::Mutex, + /// Taken at the first stamp and held until the operations are appended. + guard: Option>, + device_id: String, + /// `op_id` -> reserved `seq`. + reserved: BTreeMap, +} + +impl PodcastStampSource for PodcastLogStamps<'_> { + fn device_id(&self) -> &str { + &self.device_id + } + + fn next_stamp(&mut self) -> Result { + let conn = self.guard.get_or_insert_with(|| lock(self.conn)); + let seq = get_meta(conn, "local_seq")? + .and_then(|v| v.parse::().ok()) + .unwrap_or(0) + + 1; + let last_hlc = get_meta(conn, "last_hlc_ms")? + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); + let hlc_ms = now_ms().max(last_hlc + 1); + set_meta(conn, "local_seq", &seq.to_string())?; + set_meta(conn, "last_hlc_ms", &hlc_ms.to_string())?; + let op_id = format!("{}:{seq}", self.device_id); + self.reserved.insert(op_id.clone(), seq); + Ok(PodcastOpStamp::new(op_id, self.device_id.clone(), hlc_ms)) + } +} + +impl PodcastLogStamps<'_> { + fn append(mut self, ops: Vec) -> Result<()> { + if ops.len() != self.reserved.len() { + tracing::warn!( + reserved = self.reserved.len(), + returned = ops.len(), + "podcast action reserved stamps it did not return operations for" + ); + } + if ops.is_empty() { + return Ok(()); + } + let conn = self.guard.get_or_insert_with(|| lock(self.conn)); + for op in ops { + let Some(&seq) = self.reserved.get(&op.stamp.op_id) else { + anyhow::bail!( + "podcast operation {} was not stamped by this sync log", + op.stamp.op_id + ); + }; + let op = op.into_wire(seq); + let payload = serde_json::to_value(&op.payload)?; + let kind = payload + .get("kind") + .and_then(serde_json::Value::as_str) + .unwrap_or("podcast_unknown") + .to_string(); + conn.execute( + "INSERT OR IGNORE INTO sync_ops + (op_id, origin_device_id, seq, kind, payload_json, hlc_ms, + received_at_ms, tombstone) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + op.op_id, + op.origin_device_id, + op.seq, + kind, + payload.to_string(), + op.hlc_ms, + now_ms(), + i64::from(op.payload.is_tombstone()), + ], + )?; + conn.execute( + "INSERT INTO sync_vectors (device_id, max_seq) + VALUES (?1, ?2) + ON CONFLICT(device_id) DO UPDATE SET + max_seq = MAX(max_seq, excluded.max_seq)", + params![op.origin_device_id, op.seq], + )?; + tracing::debug!(op_id = %op.op_id, kind, "recorded podcast sync op"); + } + Ok(()) + } +} + +impl crate::podcasts::OpLog for DeviceSync { + fn record( + &self, + action: &mut dyn FnMut(&mut dyn PodcastStampSource) -> Result>, + ) -> Result<()> { + self.run_podcast_action(|stamps| Ok(((), action(stamps)?))) + } +} + #[derive(Debug)] struct StoredDevice { device_id: String, diff --git a/src/devices/tests.rs b/src/devices/tests.rs index 21f1d3f..7e07869 100644 --- a/src/devices/tests.rs +++ b/src/devices/tests.rs @@ -937,3 +937,136 @@ fn stale_synced_playlist_item_metadata_repairs_pending_fed_track() { assert!(detail.tracks[0].is_fed_pending()); assert_eq!(detail.tracks[0].title, fed.title); } + +fn podcast_meta() -> music_dht::podcast::PodcastMeta { + music_dht::podcast::PodcastMeta { + feed_url: music_dht::podcast::FeedUrl::parse("https://example.test/feed.xml").unwrap(), + title: "Show".into(), + author: None, + description: None, + link: None, + image_url: None, + } +} + +/// Rows of the local sync log: (op_id, seq, kind, payload_json, hlc_ms). +fn logged_ops(sync: &DeviceSync) -> Vec<(String, i64, String, String, i64)> { + let conn = lock(&sync.conn); + let mut stmt = conn + .prepare("SELECT op_id, seq, kind, payload_json, hlc_ms FROM sync_ops ORDER BY seq") + .unwrap(); + stmt.query_map([], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap() +} + +#[test] +fn podcast_actions_are_appended_to_the_sync_log_with_their_stamps() { + let sync = test_sync(); + let device_id = sync.ensure_identity().unwrap().device_id; + let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap(); + sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true) + .unwrap(); + let outcome = sync + .run_podcast_action(|stamps| { + let outcome = store.subscribe(stamps, &podcast_meta())?; + let ops = outcome.ops.clone(); + Ok((outcome, ops)) + }) + .unwrap(); + let ops = logged_ops(&sync); + assert_eq!(ops.len(), 2, "{ops:?}"); + let (op_id, seq, kind, payload_json, hlc_ms) = &ops[1]; + assert_eq!(kind, "podcast_subscription_set"); + assert_eq!(*seq, 2); + assert_eq!(*op_id, format!("{device_id}:2")); + assert!(*hlc_ms > ops[0].4, "hlc keeps increasing"); + // The log carries the shared FRID payload, and the store's register + // was written with exactly this stamp. + let payload: music_dht::device_sync::SyncOpPayload = + serde_json::from_str(payload_json).unwrap(); + assert!(payload.is_podcast()); + let register = store.subscription(&outcome.podcast_id).unwrap().unwrap(); + assert_eq!(register.op_id, *op_id); + assert_eq!(register.hlc_ms, *hlc_ms); + let vector = sync.vector().unwrap(); + assert_eq!(vector.get(&device_id), Some(&2)); + + // Unsubscribing emits two operations with consecutive stamps; a music + // operation afterwards continues the same sequence. + let unsubscribe = sync + .run_podcast_action(|stamps| { + let ops = store.unsubscribe(stamps, &outcome.podcast_id)?; + Ok((ops.len(), ops)) + }) + .unwrap(); + assert_eq!(unsubscribe, 2); + sync.record_content_like(&format!("b3:{}", "b".repeat(64)), true) + .unwrap(); + let ops = logged_ops(&sync); + let kinds: Vec<(i64, &str)> = ops + .iter() + .map(|(_, seq, kind, ..)| (*seq, kind.as_str())) + .collect(); + assert_eq!( + kinds, + [ + (1, "track_like_set"), + (2, "podcast_subscription_set"), + (3, "podcast_subscription_set"), + (4, "podcast_share_set"), + (5, "track_like_set"), + ] + ); + + // An action that emits nothing leaves the log alone. + sync.run_podcast_action(|stamps| { + let ops = store.unsubscribe(stamps, &outcome.podcast_id)?; + Ok(((), ops)) + }) + .unwrap(); + assert_eq!(logged_ops(&sync).len(), 5); +} + +#[test] +fn podcast_ops_are_not_sent_to_peers_before_sync_v4() { + let sync = test_sync(); + let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap(); + sync.run_podcast_action(|stamps| { + let outcome = store.subscribe(stamps, &podcast_meta())?; + let ops = outcome.ops.clone(); + Ok(((), ops)) + }) + .unwrap(); + sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true) + .unwrap(); + // This client still speaks protocol v3; its wire types cannot carry the + // podcast payload, so the batch keeps only the music operation. + let ops = sync.ops_for_peer("dev_peer").unwrap(); + assert_eq!(ops.len(), 1); + assert!(matches!(ops[0].payload, SyncOpPayload::TrackLikeSet { .. })); +} + +#[test] +fn failed_podcast_actions_append_nothing() { + let sync = test_sync(); + let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap(); + let unknown = music_dht::podcast::PodcastId::from_feed_url( + &music_dht::podcast::FeedUrl::parse("https://example.test/other.xml").unwrap(), + ); + let result = sync.run_podcast_action(|stamps| { + let ops = store.set_shared(stamps, &unknown, true)?; + Ok(((), ops)) + }); + assert!(result.is_err()); + assert!(logged_ops(&sync).is_empty()); +} diff --git a/src/main.rs b/src/main.rs index c906a76..9cbff22 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ mod jam; mod library; mod media; mod player; +mod podcasts; mod share; mod similarity; mod status; diff --git a/src/podcasts/fetch.rs b/src/podcasts/fetch.rs new file mode 100644 index 0000000..cddcf50 --- /dev/null +++ b/src/podcasts/fetch.rs @@ -0,0 +1,347 @@ +//! HTTP for podcast feeds and cover images. +//! +//! Redirects are followed by hand so the caller learns where the feed +//! really lives and whether every hop was permanent (301/308): only then +//! does the store record the new URL as a feed alias. Conditional requests +//! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and +//! the user agent are fixed here; parsing and storage happen elsewhere. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use music_dht::podcast::{FeedUrl, PodcastId}; +use reqwest::header::{ + ACCEPT, ETAG, HeaderMap, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED, LOCATION, RETRY_AFTER, +}; +use reqwest::{StatusCode, Url}; + +/// Hard cap on a feed document (large catalogs are a few MiB). +pub const MAX_FEED_BYTES: usize = 32 * 1024 * 1024; +/// Hard cap on a cover image. +pub const MAX_COVER_BYTES: usize = 10 * 1024 * 1024; +/// Redirect hops followed before giving up. +pub const MAX_REDIRECTS: usize = 10; +/// TCP + TLS connection setup. +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +/// Whole request, including the body, per hop. +pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60); +/// Sent with every podcast request. +pub const USER_AGENT: &str = concat!("furumi/", env!("CARGO_PKG_VERSION"), " (podcast reader)"); + +const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \ + text/xml;q=0.9, */*;q=0.5"; + +/// HTTP validators of the last successful fetch. +#[derive(Debug, Clone, Copy, Default)] +pub struct Validators<'a> { + pub etag: Option<&'a str>, + pub last_modified: Option<&'a str>, +} + +/// What a feed request returned. +#[derive(Debug)] +pub enum FeedBody { + /// `304 Not Modified` to a conditional request. + NotModified, + /// A new document with its validators. + Modified { + body: Vec, + etag: Option, + last_modified: Option, + }, +} + +/// A feed response and the redirect chain that led to it. +#[derive(Debug)] +pub struct FeedResponse { + pub body: FeedBody, + /// URL that finally answered, canonicalized. + pub final_url: FeedUrl, + /// At least one redirect was followed and every one of them was + /// permanent (301/308). Temporary redirects (tracking prefixes, CDNs) + /// never move a feed. + pub moved_permanently: bool, + /// Redirect hops followed. + pub redirects: usize, +} + +/// A non-success HTTP status, with the server's `Retry-After` (seconds form) +/// so the scheduler can back off as asked. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HttpStatusError { + pub status: u16, + pub retry_after: Option, +} + +impl std::fmt::Display for HttpStatusError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let reason = StatusCode::from_u16(self.status) + .ok() + .and_then(|status| status.canonical_reason()) + .unwrap_or("error"); + write!(f, "HTTP {} {reason}", self.status) + } +} + +impl std::error::Error for HttpStatusError {} + +/// Shared HTTP client for feeds and covers. +pub struct Fetcher { + client: reqwest::Client, + covers_dir: PathBuf, +} + +impl Fetcher { + pub fn new(covers_dir: PathBuf) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let client = reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(USER_AGENT) + .build() + .context("building the podcast HTTP client")?; + Ok(Fetcher { client, covers_dir }) + } + + /// GET of a feed. `validators` make it conditional; they are dropped + /// after a permanent redirect because they belong to the old URL. + pub async fn fetch_feed( + &self, + url: &FeedUrl, + validators: Validators<'_>, + ) -> Result { + let Followed { + response, + url: final_url, + moved_permanently, + redirects, + conditional, + } = self + .get_following(url.as_str(), Some(FEED_ACCEPT), validators) + .await?; + let final_url = FeedUrl::parse(final_url.as_str()).map_err(|err| { + anyhow!( + "the feed was redirected to an unsupported URL {}: {err}", + redact(&final_url) + ) + })?; + let status = response.status(); + if status == StatusCode::NOT_MODIFIED { + if !conditional { + bail!("the server answered 304 Not Modified to an unconditional request"); + } + return Ok(FeedResponse { + body: FeedBody::NotModified, + final_url, + moved_permanently, + redirects, + }); + } + if !status.is_success() { + return Err(status_error(status, response.headers()).into()); + } + let etag = header_string(response.headers(), ETAG); + let last_modified = header_string(response.headers(), LAST_MODIFIED); + let body = read_capped(response, MAX_FEED_BYTES, "feed").await?; + Ok(FeedResponse { + body: FeedBody::Modified { + body, + etag, + last_modified, + }, + final_url, + moved_permanently, + redirects, + }) + } + + /// Downloads a cover image into the covers directory; returns its path. + pub async fn fetch_cover(&self, podcast_id: &PodcastId, image_url: &str) -> Result { + let response = self + .get_following(image_url, None, Validators::default()) + .await? + .response; + let status = response.status(); + if !status.is_success() { + return Err(status_error(status, response.headers()).into()); + } + let body = read_capped(response, MAX_COVER_BYTES, "cover image").await?; + let format = image::guess_format(&body).context("the cover is not a supported image")?; + let extension = format.extensions_str().first().copied().unwrap_or("img"); + let name = cover_file_name(podcast_id, image_url, extension); + let dir = self.covers_dir.clone(); + tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body)) + .await + .map_err(|err| anyhow!("cover writer task failed: {err}"))? + } + + /// GET following up to [`MAX_REDIRECTS`] http(s) redirects by hand. + async fn get_following( + &self, + url: &str, + accept: Option<&str>, + validators: Validators<'_>, + ) -> Result { + let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?; + check_scheme(¤t)?; + let mut all_permanent = true; + let mut send_validators = true; + let mut hops = 0; + loop { + let mut request = self.client.get(current.clone()); + if let Some(accept) = accept { + request = request.header(ACCEPT, accept); + } + if send_validators { + if let Some(etag) = validators.etag { + request = request.header(IF_NONE_MATCH, etag); + } + if let Some(last_modified) = validators.last_modified { + request = request.header(IF_MODIFIED_SINCE, last_modified); + } + } + let response = request + .send() + .await + // reqwest errors print the full URL, query tokens included. + .map_err(reqwest::Error::without_url) + .with_context(|| format!("requesting {}", redact(¤t)))?; + let status = response.status(); + if !status.is_redirection() || status == StatusCode::NOT_MODIFIED { + let conditional = send_validators + && (validators.etag.is_some() || validators.last_modified.is_some()); + return Ok(Followed { + response, + url: current, + moved_permanently: hops > 0 && all_permanent, + redirects: hops, + conditional, + }); + } + let location = response + .headers() + .get(LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| anyhow!("HTTP {status} redirect without a Location header"))?; + let next = current + .join(location.trim()) + .with_context(|| format!("invalid redirect location {location:?}"))?; + check_scheme(&next)?; + hops += 1; + if hops > MAX_REDIRECTS { + bail!("too many redirects (more than {MAX_REDIRECTS})"); + } + let permanent = matches!( + status, + StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT + ); + all_permanent &= permanent; + if permanent { + send_validators = false; + } + current = next; + } + } +} + +struct Followed { + response: reqwest::Response, + /// URL that answered. + url: Url, + /// At least one hop and all of them permanent. + moved_permanently: bool, + redirects: usize, + /// The final request carried validators. + conditional: bool, +} + +fn check_scheme(url: &Url) -> Result<()> { + if !matches!(url.scheme(), "http" | "https") { + bail!("unsupported URL scheme {:?}", url.scheme()); + } + if url.host_str().is_none_or(str::is_empty) { + bail!("URL has no host"); + } + Ok(()) +} + +/// The URL without query and fragment, for messages and logs: feed URLs +/// may carry private tokens in their query. +pub fn redact(url: &Url) -> String { + let mut shown = url.clone(); + if shown.query().is_some() { + shown.set_query(Some("…")); + } + shown.set_fragment(None); + shown.to_string() +} + +fn header_string(headers: &HeaderMap, name: reqwest::header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +fn status_error(status: StatusCode, headers: &HeaderMap) -> HttpStatusError { + let retry_after = header_string(headers, RETRY_AFTER) + .and_then(|value| value.parse::().ok()) + .map(Duration::from_secs); + HttpStatusError { + status: status.as_u16(), + retry_after, + } +} + +async fn read_capped(mut response: reqwest::Response, cap: usize, what: &str) -> Result> { + if let Some(len) = response.content_length() + && len > cap as u64 + { + bail!("the {what} is too large ({len} bytes, limit {cap})"); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(reqwest::Error::without_url) + .with_context(|| format!("reading the {what}"))? + { + if body.len() + chunk.len() > cap { + bail!("the {what} is too large (over {cap} bytes)"); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +/// Stable per podcast and image URL, so a changed image gets a new file and +/// the previous one can be removed. +fn cover_file_name(podcast_id: &PodcastId, image_url: &str, extension: &str) -> String { + let podcast = blake3::hash(podcast_id.as_str().as_bytes()).to_hex(); + let image = blake3::hash(image_url.as_bytes()).to_hex(); + format!("{}-{}.{extension}", &podcast[..16], &image[..16]) +} + +fn write_atomically(dir: &Path, name: &str, bytes: &[u8]) -> Result { + static COUNTER: AtomicU64 = AtomicU64::new(0); + std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?; + let target = dir.join(name); + let temp = dir.join(format!( + ".{name}.{}-{}.tmp", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + let result = std::fs::write(&temp, bytes) + .and_then(|()| std::fs::rename(&temp, &target)) + .with_context(|| format!("writing {}", target.display())); + if result.is_err() { + let _ = std::fs::remove_file(&temp); + } + result?; + Ok(target) +} diff --git a/src/podcasts/mod.rs b/src/podcasts/mod.rs new file mode 100644 index 0000000..4f5fa04 --- /dev/null +++ b/src/podcasts/mod.rs @@ -0,0 +1,107 @@ +//! Podcast runtime of the TUI: feed HTTP, refresh scheduling and the glue +//! between the shared podcast store and the device-sync log. +//! +//! The data model, identity rules, merge rules and the SQLite store are not +//! defined here: they come from FRID (`music_dht::podcast` and +//! `furumi_library::podcasts::PodcastStore`, file `podcasts.sqlite3`). This +//! module only does what the contract leaves to clients: +//! +//! * [`fetch`] downloads feeds (conditional GET, redirects reported so +//! permanent moves become feed aliases, size and time limits) and covers; +//! * [`refresh`] subscribes and refreshes feeds: it parses them with the +//! shared `music_dht::podcast::feed` parser, ingests them with +//! `PodcastStore::apply_feed` and keeps the local fetch state (validators, +//! errors, back-off); +//! * [`OpLog`] hands store actions the stamps of the device-sync log and +//! appends the operations they return. +//! +//! Only RSS metadata is stored. Audio stays on the publisher's server. + +pub mod fetch; +pub mod refresh; + +#[cfg(test)] +mod test_server; +#[cfg(test)] +mod tests; + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Result, anyhow}; +use furumi_library::podcasts::{ + PODCAST_DB_FILE, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource, +}; + +pub use fetch::Fetcher; +use refresh::Services; + +/// Appends podcast operations to the device-sync log. +/// +/// [`OpLog::record`] gives `action` a [`StampSource`] that reserves stamps +/// in the log. The action runs a `PodcastStore` local action and returns the +/// operations it emitted; they are appended with exactly those stamps. +pub trait OpLog: Send + Sync { + fn record( + &self, + action: &mut dyn FnMut(&mut dyn StampSource) -> Result>, + ) -> Result<()>; +} + +/// The podcast store file. It is the file the prototype podcast build +/// used, so an existing prototype database is migrated in place. +pub fn db_path() -> Result { + let dirs = crate::config::project_dirs() + .ok_or_else(|| anyhow!("no home directory for podcast data"))?; + Ok(dirs.data_dir().join(PODCAST_DB_FILE)) +} + +/// Replaceable cache of downloaded podcast covers. +pub fn covers_dir() -> Result { + let dirs = crate::config::project_dirs() + .ok_or_else(|| anyhow!("no cache directory for podcast covers"))?; + Ok(dirs.cache_dir().join("podcasts").join("covers")) +} + +/// Opens the podcast store at `db_path` and builds the feed client. +/// +/// `PodcastStore::open` migrates a prototype database in place and refuses, +/// without writing anything, a database from a newer Furumi or a file that +/// is not a podcast database. Blocking: call it on the blocking pool. +pub fn open_services(db_path: &Path, covers_dir: PathBuf, log: Arc) -> Result { + let store = PodcastStore::open(db_path)?; + tracing::info!(path = %db_path.display(), "podcast store opened"); + match store.legacy_quarantine() { + Ok(entries) if !entries.is_empty() => tracing::warn!( + count = entries.len(), + "some prototype podcast rows could not be imported; they are kept in the legacy_* tables" + ), + Ok(_) => {} + Err(err) => tracing::debug!("reading the podcast import quarantine failed: {err:#}"), + } + Ok(Services { + store: Arc::new(store), + fetcher: Arc::new(Fetcher::new(covers_dir)?), + log, + }) +} + +/// A readable reason why the store did not open; the typed store errors +/// already say that the file was left unchanged. +pub fn describe_open_error(err: &anyhow::Error) -> String { + match err.downcast_ref::() { + Some(store_error) => store_error.to_string(), + None => format!("{err:#}"), + } +} + +/// Runs blocking store or filesystem work off the Tokio workers. +pub(crate) async fn blocking(work: F) -> Result +where + T: Send + 'static, + F: FnOnce() -> Result + Send + 'static, +{ + tokio::task::spawn_blocking(work) + .await + .map_err(|err| anyhow!("background task failed: {err}"))? +} diff --git a/src/podcasts/refresh.rs b/src/podcasts/refresh.rs new file mode 100644 index 0000000..a3e172b --- /dev/null +++ b/src/podcasts/refresh.rs @@ -0,0 +1,380 @@ +//! Subscribing and refreshing feeds. +//! +//! Every function records its HTTP outcome in the store's local fetch state +//! (validators, last error, consecutive failures, next check) so the +//! scheduler never needs its own bookkeeping, and an error never clears the +//! stored catalog. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context as _, Result, anyhow, bail}; +use furumi_library::podcasts::{FeedRefresh, PodcastStore}; +use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; +use music_dht::podcast::{FeedUrl, PodcastId}; + +use super::fetch::{FeedBody, Fetcher, HttpStatusError, Validators}; +use super::{OpLog, blocking}; + +/// A successfully checked feed is checked again after this long. +pub const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60); +/// First retry after a failed check; doubles per consecutive failure. +pub const FAILURE_BACKOFF_BASE: Duration = Duration::from_secs(5 * 60); +/// Longest wait between checks of a failing feed (also caps `Retry-After`). +pub const FAILURE_BACKOFF_MAX: Duration = Duration::from_secs(24 * 60 * 60); + +/// Store, HTTP client and sync log shared by podcast background tasks. +#[derive(Clone)] +pub struct Services { + pub store: Arc, + pub fetcher: Arc, + pub log: Arc, +} + +impl std::fmt::Debug for Services { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Services") + .field("store", &self.store) + .finish_non_exhaustive() + } +} + +/// Result of [`subscribe`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubscribeReport { + pub podcast_id: PodcastId, + pub title: String, + /// The podcast was already subscribed; nothing was fetched or written. + pub already_subscribed: bool, + /// Episodes the first refresh added to the catalog. + pub new_episodes: usize, +} + +/// Result of [`refresh`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RefreshReport { + /// The server answered 304 Not Modified. + pub not_modified: bool, + pub new_episodes: usize, + pub updated_episodes: usize, + /// Feed items without identity or with invalid metadata. + pub skipped_items: usize, + /// New fetch URL after a permanent redirect or `itunes:new-feed-url`. + pub moved_to: Option, +} + +/// When to check a feed again after a success. +pub fn next_check_after_success(now_ms: i64) -> i64 { + now_ms.saturating_add(duration_ms(REFRESH_INTERVAL)) +} + +/// When to check a feed again after its `failures`-th consecutive failure: +/// exponential back-off, at least the server's `Retry-After`, at most +/// [`FAILURE_BACKOFF_MAX`]. +pub fn next_check_after_failure(now_ms: i64, failures: u32, retry_after: Option) -> i64 { + let exponent = failures.saturating_sub(1).min(16); + let backoff = FAILURE_BACKOFF_BASE + .saturating_mul(1 << exponent) + .max(retry_after.unwrap_or_default()) + .min(FAILURE_BACKOFF_MAX); + now_ms.saturating_add(duration_ms(backoff)) +} + +fn duration_ms(duration: Duration) -> i64 { + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) +} + +/// Subscribes to the feed the user typed (`FeedUrl::from_user_input`). +/// +/// Nothing is stored unless the URL serves a parsable feed. A URL that +/// resolves (directly or through a known alias or redirect) to a podcast +/// this device already knows reuses its id, so resubscribing continues the +/// saved progress and history. The subscription is private. +pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result { + let url = FeedUrl::from_user_input(input).map_err(|err| anyhow!("{err}"))?; + let store = Arc::clone(&services.store); + let lookup = url.clone(); + let existing = blocking(move || { + let id = store.resolve_podcast_id(&lookup)?; + let subscribed = store + .subscription(&id)? + .is_some_and(|record| record.subscribed); + let title = store.podcast(&id)?.map(|summary| summary.meta.title); + Ok(subscribed.then(|| (id, title))) + }) + .await?; + if let Some((podcast_id, title)) = existing { + return Ok(SubscribeReport { + podcast_id, + title: title.unwrap_or_default(), + already_subscribed: true, + new_episodes: 0, + }); + } + + let response = services + .fetcher + .fetch_feed(&url, Validators::default()) + .await?; + let FeedBody::Modified { + body, + etag, + last_modified, + } = response.body + else { + bail!("the server answered 304 Not Modified to an unconditional request"); + }; + let final_url = response.final_url; + let feed = parse(body, &final_url).await?; + + let store = Arc::clone(&services.store); + let log = Arc::clone(&services.log); + let moved_permanently = response.moved_permanently; + let image_url = feed.image_url.clone(); + let report = blocking(move || { + // Identity comes from the URL of the first subscription. Reuse a + // podcast this device already knows under the typed or the final + // URL; otherwise the typed URL defines the id and a permanent + // redirect becomes an alias below. + let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? { + url.clone() + } else { + final_url.clone() + }; + let meta = feed.to_podcast_meta(identity_url); + let mut subscribed = None; + log.record(&mut |stamps| { + let outcome = store.subscribe(stamps, &meta)?; + let ops = outcome.ops.clone(); + subscribed = Some(outcome); + Ok(ops) + })?; + let outcome = subscribed.ok_or_else(|| anyhow!("subscription was not recorded"))?; + let podcast_id = outcome.podcast_id; + let mut refreshed = None; + log.record(&mut |stamps| { + let outcome = store.apply_feed( + stamps, + &FeedRefresh { + podcast_id: &podcast_id, + feed: &feed, + fetched_from: &final_url, + moved_permanently, + }, + )?; + let ops = outcome.ops.clone(); + refreshed = Some(outcome); + Ok(ops) + })?; + let refreshed = refreshed.ok_or_else(|| anyhow!("feed refresh was not recorded"))?; + store.record_fetch_success( + &podcast_id, + now_ms, + etag.as_deref(), + last_modified.as_deref(), + Some(next_check_after_success(now_ms)), + )?; + Ok(SubscribeReport { + podcast_id, + title: meta.title, + already_subscribed: outcome.already_subscribed, + new_episodes: refreshed.new_episodes, + }) + }) + .await?; + if let Some(image_url) = image_url { + update_cover(services, &report.podcast_id, &image_url).await; + } + Ok(report) +} + +fn knows(store: &PodcastStore, url: &FeedUrl) -> Result { + let id = store.resolve_podcast_id(url)?; + Ok(store.podcast(&id)?.is_some() || store.subscription(&id)?.is_some()) +} + +/// Checks one feed (conditional GET from `url`, the store's fetch URL) and +/// ingests a changed document. Failures are recorded in the fetch state +/// with back-off and returned; the stored catalog is kept. +pub async fn refresh( + services: &Services, + podcast_id: &PodcastId, + url: &FeedUrl, + now_ms: i64, +) -> Result { + let store = Arc::clone(&services.store); + let id = podcast_id.clone(); + let fetch_state = blocking(move || store.fetch_state(&id)).await?; + let result = fetch_and_apply(services, podcast_id, url, &fetch_state, now_ms).await; + if let Err(err) = &result { + let message = format!("{err:#}"); + let retry_after = err + .downcast_ref::() + .and_then(|status| status.retry_after); + let next = next_check_after_failure( + now_ms, + fetch_state.consecutive_failures.saturating_add(1), + retry_after, + ); + let store = Arc::clone(&services.store); + let id = podcast_id.clone(); + if let Err(record_err) = blocking(move || { + store + .record_fetch_failure(&id, now_ms, &message, Some(next)) + .map(|_| ()) + }) + .await + { + tracing::warn!("recording a podcast refresh error failed: {record_err:#}"); + } + } + result +} + +async fn fetch_and_apply( + services: &Services, + podcast_id: &PodcastId, + url: &FeedUrl, + fetch_state: &furumi_library::podcasts::FetchState, + now_ms: i64, +) -> Result { + let response = services + .fetcher + .fetch_feed( + url, + Validators { + etag: fetch_state.etag.as_deref(), + last_modified: fetch_state.last_modified.as_deref(), + }, + ) + .await?; + tracing::debug!( + podcast = %podcast_id, + redirects = response.redirects, + moved_permanently = response.moved_permanently, + not_modified = matches!(response.body, FeedBody::NotModified), + "podcast feed checked" + ); + let next = Some(next_check_after_success(now_ms)); + let (body, etag, last_modified) = match response.body { + FeedBody::NotModified => { + let store = Arc::clone(&services.store); + let id = podcast_id.clone(); + blocking(move || store.record_fetch_not_modified(&id, now_ms, next)).await?; + return Ok(RefreshReport { + not_modified: true, + ..RefreshReport::default() + }); + } + FeedBody::Modified { + body, + etag, + last_modified, + } => (body, etag, last_modified), + }; + let final_url = response.final_url; + let feed = parse(body, &final_url).await?; + let image_url = feed.image_url.clone(); + let store = Arc::clone(&services.store); + let log = Arc::clone(&services.log); + let id = podcast_id.clone(); + let previous_url = url.clone(); + let moved_permanently = response.moved_permanently; + let report = blocking(move || { + let mut refreshed = None; + log.record(&mut |stamps| { + let outcome = store.apply_feed( + stamps, + &FeedRefresh { + podcast_id: &id, + feed: &feed, + fetched_from: &final_url, + moved_permanently, + }, + )?; + let ops = outcome.ops.clone(); + refreshed = Some(outcome); + Ok(ops) + })?; + let outcome = refreshed.ok_or_else(|| anyhow!("feed refresh was not recorded"))?; + store.record_fetch_success(&id, now_ms, etag.as_deref(), last_modified.as_deref(), next)?; + Ok(RefreshReport { + not_modified: false, + new_episodes: outcome.new_episodes, + updated_episodes: outcome.updated_episodes, + skipped_items: outcome.skipped_items, + moved_to: outcome + .feed_url + .filter(|feed_url| *feed_url != previous_url), + }) + }) + .await?; + if let Some(image_url) = image_url { + update_cover(services, podcast_id, &image_url).await; + } + Ok(report) +} + +/// Parses a feed document on the blocking pool; relative URLs resolve +/// against where the document was served from. +async fn parse(body: Vec, base: &FeedUrl) -> Result { + let base = base.as_str().to_string(); + blocking(move || { + parse_feed_with_base(&body, Some(&base)).map_err(|err| anyhow!("unusable feed: {err}")) + }) + .await +} + +/// Downloads the cover when the feed image changed or the cached file is +/// gone, records it and deletes the replaced file. Best effort: failures are +/// only logged, covers are a replaceable cache. +async fn update_cover(services: &Services, podcast_id: &PodcastId, image_url: &str) { + let store = Arc::clone(&services.store); + let id = podcast_id.clone(); + let current = blocking(move || { + let state = store.fetch_state(&id)?; + Ok((state.cover_url, state.cover_path)) + }) + .await; + let (old_url, old_path) = match current { + Ok(current) => current, + Err(err) => { + tracing::warn!("reading podcast cover state failed: {err:#}"); + return; + } + }; + let cached = old_path + .as_deref() + .is_some_and(|path| Path::new(path).is_file()); + if old_url.as_deref() == Some(image_url) && cached { + return; + } + let path = match services.fetcher.fetch_cover(podcast_id, image_url).await { + Ok(path) => path, + Err(err) => { + tracing::debug!(podcast = %podcast_id, "podcast cover download failed: {err:#}"); + return; + } + }; + let store = Arc::clone(&services.store); + let id = podcast_id.clone(); + let image_url = image_url.to_string(); + let result = blocking(move || { + let new_path = std::path::absolute(&path) + .unwrap_or(path) + .to_string_lossy() + .into_owned(); + store + .set_cover_cache(&id, Some(&image_url), Some(&new_path)) + .context("recording the cover")?; + if let Some(old_path) = old_path.filter(|old| *old != new_path) { + let _ = std::fs::remove_file(old_path); + } + Ok(()) + }) + .await; + if let Err(err) = result { + tracing::warn!("recording a podcast cover failed: {err:#}"); + } +} diff --git a/src/podcasts/test_server.rs b/src/podcasts/test_server.rs new file mode 100644 index 0000000..2fcd38f --- /dev/null +++ b/src/podcasts/test_server.rs @@ -0,0 +1,207 @@ +//! Minimal HTTP/1.1 server on 127.0.0.1 for podcast tests (no internet). +//! One request per connection (`Connection: close`). + +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +#[derive(Clone)] +pub(super) struct Route { + pub status: u16, + pub body: Arc>, + pub content_type: String, + pub etag: Option, + pub last_modified: Option, + /// `Location` of a redirect status. + pub location: Option, + pub retry_after: Option, +} + +impl Route { + pub fn new(body: impl Into>, content_type: &str) -> Route { + Route { + status: 200, + body: Arc::new(body.into()), + content_type: content_type.to_string(), + etag: None, + last_modified: None, + location: None, + retry_after: None, + } + } + + pub fn redirect(status: u16, location: &str) -> Route { + Route { + status, + location: Some(location.to_string()), + ..Route::new(Vec::new(), "text/plain") + } + } + + pub fn status(mut self, status: u16) -> Route { + self.status = status; + self + } + + pub fn etag(mut self, etag: &str) -> Route { + self.etag = Some(etag.to_string()); + self + } + + pub fn last_modified(mut self, value: &str) -> Route { + self.last_modified = Some(value.to_string()); + self + } + + pub fn retry_after(mut self, value: &str) -> Route { + self.retry_after = Some(value.to_string()); + self + } +} + +/// Headers of one received request (names lowercased). +pub(super) type RequestHeaders = HashMap; + +#[derive(Default)] +struct ServerState { + routes: HashMap, + requests: HashMap>, +} + +pub(super) struct TestServer { + base: String, + state: Arc>, +} + +impl TestServer { + pub fn start() -> TestServer { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server"); + let base = format!("http://{}", listener.local_addr().expect("local addr")); + let state = Arc::new(Mutex::new(ServerState::default())); + let accept_state = Arc::clone(&state); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let state = Arc::clone(&accept_state); + std::thread::spawn(move || handle(stream, &state)); + } + }); + TestServer { base, state } + } + + pub fn url(&self, path: &str) -> String { + format!("{}{path}", self.base) + } + + pub fn set(&self, path: &str, route: Route) { + self.lock().routes.insert(path.to_string(), route); + } + + /// Headers of every request received for `path`, in order. + pub fn requests(&self, path: &str) -> Vec { + self.lock().requests.get(path).cloned().unwrap_or_default() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, ServerState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +fn handle(stream: TcpStream, state: &Mutex) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let mut reader = BufReader::new(match stream.try_clone() { + Ok(stream) => stream, + Err(_) => return, + }); + let mut request_line = String::new(); + if reader.read_line(&mut request_line).is_err() { + return; + } + let path = request_line + .split_whitespace() + .nth(1) + .unwrap_or("/") + .to_string(); + let mut headers = RequestHeaders::new(); + loop { + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) | Err(_) => return, + Ok(_) => {} + } + let line = line.trim_end(); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string()); + } + } + let route = { + let mut state = state.lock().unwrap_or_else(PoisonError::into_inner); + state + .requests + .entry(path.clone()) + .or_default() + .push(headers.clone()); + state.routes.get(&path).cloned() + }; + let mut out = stream; + let Some(route) = route else { + let _ = out + .write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + }; + let etag_matches = route + .etag + .as_ref() + .is_some_and(|etag| headers.get("if-none-match") == Some(etag)); + let date_matches = route + .last_modified + .as_ref() + .is_some_and(|date| headers.get("if-modified-since") == Some(date)); + if route.status == 200 && (etag_matches || date_matches) { + let _ = write!( + out, + "HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n" + ); + return; + } + let mut extra = String::new(); + for (name, value) in [ + ("ETag", &route.etag), + ("Last-Modified", &route.last_modified), + ("Location", &route.location), + ("Retry-After", &route.retry_after), + ] { + if let Some(value) = value { + extra.push_str(&format!("{name}: {value}\r\n")); + } + } + let reason = match route.status { + 200 => "OK", + 301 => "Moved Permanently", + 302 => "Found", + 307 => "Temporary Redirect", + 308 => "Permanent Redirect", + 404 => "Not Found", + 503 => "Service Unavailable", + _ => "Status", + }; + if write!( + out, + "HTTP/1.1 {} {reason}\r\nContent-Type: {}\r\nContent-Length: {}\r\n{extra}Connection: close\r\n\r\n", + route.status, + route.content_type, + route.body.len() + ) + .is_err() + { + return; + } + let _ = out.write_all(&route.body); + let _ = out.flush(); + let _ = out.shutdown(std::net::Shutdown::Both); +} diff --git a/src/podcasts/tests.rs b/src/podcasts/tests.rs new file mode 100644 index 0000000..c73b0e4 --- /dev/null +++ b/src/podcasts/tests.rs @@ -0,0 +1,588 @@ +//! Store opening, subscribing and refreshing against a local HTTP server. +//! No internet, no real user directories: every test uses a temp dir. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use anyhow::Result; +use furumi_library::podcasts::{ + EpisodeQuery, OpStamp, PodcastListFilter, PodcastLocalOp, PodcastStore, StampSource, +}; +use music_dht::device_sync::{SyncOpPayload, SyncOpWire}; +use music_dht::podcast::{FeedUrl, PodcastId}; + +use super::OpLog; +use super::refresh::{ + FAILURE_BACKOFF_BASE, FAILURE_BACKOFF_MAX, REFRESH_INTERVAL, Services, + next_check_after_failure, next_check_after_success, refresh, subscribe, +}; +use super::test_server::{Route, TestServer}; + +const NOW: i64 = 1_700_000_000_000; + +/// In-memory sync log: hands out stamps and keeps the appended operations. +#[derive(Default)] +struct MemoryLog { + ops: Mutex>, +} + +struct MemoryStamps { + next_seq: i64, + reserved: Vec, +} + +impl StampSource for MemoryStamps { + fn device_id(&self) -> &str { + "test-device" + } + + fn next_stamp(&mut self) -> Result { + let seq = self.next_seq + self.reserved.len() as i64; + self.reserved.push(seq); + Ok(OpStamp::new( + format!("test-device:{seq}"), + "test-device", + NOW + seq, + )) + } +} + +impl OpLog for MemoryLog { + fn record( + &self, + action: &mut dyn FnMut(&mut dyn StampSource) -> Result>, + ) -> Result<()> { + let mut ops = self.ops.lock().unwrap_or_else(PoisonError::into_inner); + let mut stamps = MemoryStamps { + next_seq: ops.len() as i64 + 1, + reserved: Vec::new(), + }; + for (op, seq) in action(&mut stamps)?.into_iter().zip(stamps.reserved) { + ops.push(op.into_wire(seq)); + } + Ok(()) + } +} + +impl MemoryLog { + fn kinds(&self) -> Vec { + self.ops + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .map(|op| { + serde_json::to_value(&op.payload).unwrap()["kind"] + .as_str() + .unwrap() + .to_string() + }) + .collect() + } +} + +struct Fixture { + _dir: tempfile::TempDir, + services: Services, + log: Arc, + server: TestServer, +} + +fn fixture() -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let log = Arc::new(MemoryLog::default()); + let services = super::open_services( + &dir.path().join("podcasts.sqlite3"), + dir.path().join("covers"), + log.clone(), + ) + .unwrap(); + Fixture { + _dir: dir, + services, + log, + server: TestServer::start(), + } +} + +fn rss(items: &[(&str, &str)]) -> Vec { + let items: String = items + .iter() + .map(|(guid, title)| { + format!( + "{title}{guid}\ + Sat, 15 Jun 2024 10:00:00 GMT\ + 45:00\ + " + ) + }) + .collect(); + format!( + "\ + Test ShowHost{items}" + ) + .into_bytes() +} + +fn episode_count(store: &PodcastStore, id: &PodcastId) -> usize { + store + .episodes(&EpisodeQuery::new(id.clone())) + .unwrap() + .len() +} + +#[tokio::test] +async fn subscribe_then_conditional_refresh_uses_validators() { + let f = fixture(); + f.server.set( + "/feed.xml", + Route::new( + rss(&[("a", "Pilot"), ("b", "Second")]), + "application/rss+xml", + ) + .etag("\"v1\"") + .last_modified("Sat, 15 Jun 2024 10:00:00 GMT"), + ); + let report = subscribe(&f.services, &f.server.url("/feed.xml"), NOW) + .await + .unwrap(); + assert!(!report.already_subscribed); + assert_eq!(report.new_episodes, 2); + assert_eq!(report.title, "Test Show"); + assert_eq!(f.log.kinds(), ["podcast_subscription_set"]); + let store = &f.services.store; + let rows = store.list_podcasts(PodcastListFilter::Subscribed).unwrap(); + assert_eq!(rows.len(), 1); + assert!(!rows[0].shared, "a new subscription is private"); + let state = store.fetch_state(&report.podcast_id).unwrap(); + assert_eq!(state.etag.as_deref(), Some("\"v1\"")); + assert_eq!(state.next_check_at_ms, Some(next_check_after_success(NOW))); + let first = &f.server.requests("/feed.xml")[0]; + assert!(!first.contains_key("if-none-match"), "{first:?}"); + assert!( + first + .get("user-agent") + .is_some_and(|agent| agent.starts_with("furumi/")), + "{first:?}" + ); + + // Unchanged: 304, nothing re-parsed, validators sent. + let url = rows[0].meta.feed_url.clone(); + let checked = refresh(&f.services, &report.podcast_id, &url, NOW + 1) + .await + .unwrap(); + assert!(checked.not_modified); + let second = &f.server.requests("/feed.xml")[1]; + assert_eq!( + second.get("if-none-match").map(String::as_str), + Some("\"v1\"") + ); + assert_eq!( + second.get("if-modified-since").map(String::as_str), + Some("Sat, 15 Jun 2024 10:00:00 GMT") + ); + + // Changed: a new episode arrives, the known ones are not duplicated. + f.server.set( + "/feed.xml", + Route::new( + rss(&[("a", "Pilot (remastered)"), ("b", "Second"), ("c", "Third")]), + "application/rss+xml", + ) + .etag("\"v2\""), + ); + let changed = refresh(&f.services, &report.podcast_id, &url, NOW + 2) + .await + .unwrap(); + assert!(!changed.not_modified); + assert_eq!(changed.new_episodes, 1); + assert_eq!(changed.updated_episodes, 1); + assert_eq!(episode_count(store, &report.podcast_id), 3); + // A refresh never writes personal state: no new operation. + assert_eq!(f.log.kinds(), ["podcast_subscription_set"]); + + // Subscribing again is recognised without another fetch. + let requests = f.server.requests("/feed.xml").len(); + let again = subscribe(&f.services, &f.server.url("/feed.xml"), NOW + 3) + .await + .unwrap(); + assert!(again.already_subscribed); + assert_eq!(again.podcast_id, report.podcast_id); + assert_eq!(f.server.requests("/feed.xml").len(), requests); +} + +#[tokio::test] +async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() { + let f = fixture(); + f.server + .set("/old.xml", Route::redirect(301, &f.server.url("/new.xml"))); + f.server.set( + "/new.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"), + ); + let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap(); + let new = FeedUrl::parse(&f.server.url("/new.xml")).unwrap(); + let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap(); + // The id comes from the URL of the first subscription. + assert_eq!(report.podcast_id, PodcastId::from_feed_url(&old)); + assert_eq!( + f.log.kinds(), + ["podcast_subscription_set", "podcast_feed_aliases_added"] + ); + let store = &f.services.store; + assert!( + store + .feed_aliases(&report.podcast_id) + .unwrap() + .contains(&new) + ); + let summary = store.podcast(&report.podcast_id).unwrap().unwrap(); + assert_eq!(summary.meta.feed_url, new, "fetched from the new URL now"); + assert_eq!(store.resolve_podcast_id(&new).unwrap(), report.podcast_id); + + // The next refresh goes straight to the new URL; no new alias op. + let checked = refresh(&f.services, &report.podcast_id, &new, NOW + 1) + .await + .unwrap(); + assert!(checked.moved_to.is_none()); + assert_eq!(f.server.requests("/old.xml").len(), 1); + assert_eq!(f.log.kinds().len(), 2); + + // Subscribing through the new URL finds the same podcast. + let again = subscribe(&f.services, new.as_str(), NOW + 2).await.unwrap(); + assert!(again.already_subscribed); + assert_eq!(again.podcast_id, report.podcast_id); +} + +#[tokio::test] +async fn temporary_redirect_keeps_the_feed_url() { + let f = fixture(); + f.server + .set("/tracked.xml", Route::redirect(302, "/real.xml")); + f.server.set( + "/real.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"), + ); + let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap(); + let report = subscribe(&f.services, tracked.as_str(), NOW).await.unwrap(); + assert_eq!(f.log.kinds(), ["podcast_subscription_set"]); + let summary = f + .services + .store + .podcast(&report.podcast_id) + .unwrap() + .unwrap(); + assert_eq!(summary.meta.feed_url, tracked); + // Episode URLs resolve against where the document was served from. + let episodes = f + .services + .store + .episodes(&EpisodeQuery::new(report.podcast_id.clone())) + .unwrap(); + assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3")); +} + +#[tokio::test] +async fn malformed_feed_and_server_errors_back_off_and_keep_the_catalog() { + let f = fixture(); + f.server.set( + "/feed.xml", + Route::new( + rss(&[("a", "Pilot"), ("b", "Second")]), + "application/rss+xml", + ), + ); + let report = subscribe(&f.services, &f.server.url("/feed.xml"), NOW) + .await + .unwrap(); + let id = report.podcast_id; + let url = FeedUrl::parse(&f.server.url("/feed.xml")).unwrap(); + + f.server.set( + "/feed.xml", + Route::new("Moved to our app!", "text/html"), + ); + let err = refresh(&f.services, &id, &url, NOW + 1).await.unwrap_err(); + assert!(format!("{err:#}").contains("unusable feed"), "{err:#}"); + let store = &f.services.store; + let state = store.fetch_state(&id).unwrap(); + assert_eq!(state.consecutive_failures, 1); + assert!( + state + .last_error + .as_deref() + .unwrap() + .contains("unusable feed") + ); + assert_eq!( + state.next_check_at_ms, + Some(next_check_after_failure(NOW + 1, 1, None)) + ); + assert_eq!(episode_count(store, &id), 2, "an error keeps the catalog"); + + f.server.set( + "/feed.xml", + Route::new("busy", "text/plain") + .status(503) + .retry_after("7200"), + ); + let err = refresh(&f.services, &id, &url, NOW + 2).await.unwrap_err(); + assert!(format!("{err:#}").contains("HTTP 503"), "{err:#}"); + let state = store.fetch_state(&id).unwrap(); + assert_eq!(state.consecutive_failures, 2); + assert_eq!(state.next_check_at_ms, Some(NOW + 2 + 7_200_000)); + assert_eq!(episode_count(store, &id), 2); + // A failing feed is not due before its back-off expires. + assert!( + store + .podcasts_due_for_refresh(NOW + 3, 10) + .unwrap() + .is_empty() + ); + assert_eq!( + store + .podcasts_due_for_refresh(NOW + 2 + 7_200_000, 10) + .unwrap() + .len(), + 1 + ); + + // Recovery clears the error. + f.server.set( + "/feed.xml", + Route::new( + rss(&[("a", "Pilot"), ("b", "Second")]), + "application/rss+xml", + ), + ); + refresh(&f.services, &id, &url, NOW + 3).await.unwrap(); + let state = store.fetch_state(&id).unwrap(); + assert_eq!(state.consecutive_failures, 0); + assert!(state.last_error.is_none()); +} + +#[tokio::test] +async fn subscribing_to_a_non_feed_stores_nothing() { + let f = fixture(); + f.server + .set("/page", Route::new("hello", "text/html")); + let err = subscribe(&f.services, &f.server.url("/page"), NOW) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("unusable feed"), "{err:#}"); + let err = subscribe(&f.services, &f.server.url("/missing"), NOW) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("HTTP 404"), "{err:#}"); + let err = subscribe(&f.services, "ftp://example.test/feed", NOW) + .await + .unwrap_err(); + assert!(!format!("{err:#}").is_empty()); + assert!( + f.services + .store + .list_podcasts(PodcastListFilter::All) + .unwrap() + .is_empty() + ); + assert!(f.log.kinds().is_empty()); +} + +#[tokio::test] +async fn unsubscribe_then_resubscribe_keeps_the_podcast_id() { + let f = fixture(); + f.server.set( + "/feed.xml", + Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"), + ); + let first = subscribe(&f.services, &f.server.url("/feed.xml"), NOW) + .await + .unwrap(); + let store = Arc::clone(&f.services.store); + let id = first.podcast_id.clone(); + f.log + .record(&mut |stamps| store.unsubscribe(stamps, &id)) + .unwrap(); + assert!( + store + .list_podcasts(PodcastListFilter::Subscribed) + .unwrap() + .is_empty() + ); + let again = subscribe(&f.services, &f.server.url("/feed.xml"), NOW + 1) + .await + .unwrap(); + assert!(!again.already_subscribed); + assert_eq!(again.podcast_id, first.podcast_id); + assert_eq!( + f.log.kinds(), + [ + "podcast_subscription_set", + "podcast_subscription_set", + "podcast_share_set", + "podcast_subscription_set" + ] + ); + let ops = f.log.ops.lock().unwrap(); + assert!(matches!( + ops.last().map(|op| &op.payload), + Some(SyncOpPayload::PodcastSubscriptionSet { + subscribed: true, + .. + }) + )); +} + +#[test] +fn back_off_doubles_and_honours_retry_after() { + let base = FAILURE_BACKOFF_BASE.as_millis() as i64; + assert_eq!(next_check_after_failure(0, 1, None), base); + assert_eq!(next_check_after_failure(0, 2, None), 2 * base); + assert_eq!(next_check_after_failure(0, 4, None), 8 * base); + let max = FAILURE_BACKOFF_MAX.as_millis() as i64; + assert_eq!(next_check_after_failure(0, 40, None), max); + assert_eq!( + next_check_after_failure(0, 1, Some(Duration::from_secs(3_600))), + 3_600_000 + ); + assert_eq!( + next_check_after_failure(0, 1, Some(Duration::from_secs(10 * 86_400))), + max + ); + assert_eq!( + next_check_after_success(0), + REFRESH_INTERVAL.as_millis() as i64 + ); +} + +/// The prototype podcast build's schema (user_version 1), verbatim. +const PROTOTYPE_SCHEMA: &str = " + CREATE TABLE IF NOT EXISTS podcasts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_url TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + author TEXT, + description TEXT, + link TEXT, + image_url TEXT, + cover_url TEXT, + cover_path TEXT, + etag TEXT, + last_modified TEXT, + added_at INTEGER NOT NULL, + last_checked_at INTEGER, + last_error TEXT + ); + CREATE TABLE IF NOT EXISTS podcast_episodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE, + guid TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT, + published_at INTEGER, + duration_seconds REAL, + audio_url TEXT NOT NULL, + mime_type TEXT, + byte_len INTEGER, + link TEXT, + image_url TEXT, + first_seen_at INTEGER NOT NULL, + UNIQUE (podcast_id, guid) + ); + CREATE INDEX IF NOT EXISTS podcast_episodes_by_date + ON podcast_episodes (podcast_id, published_at DESC, id DESC); +"; + +#[test] +fn prototype_database_is_migrated_in_place() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(furumi_library::podcasts::PODCAST_DB_FILE); + { + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute_batch(PROTOTYPE_SCHEMA).unwrap(); + conn.execute( + "INSERT INTO podcasts (feed_url, title, etag, added_at, last_checked_at) + VALUES ('https://example.test/feed.xml', 'Old Show', '\"e1\"', 1700000000, 1700000100)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO podcast_episodes + (podcast_id, guid, title, published_at, duration_seconds, audio_url, first_seen_at) + VALUES (1, 'ep-1', 'Old Pilot', 1700000000, 1800.0, + 'https://example.test/1.mp3', 1700000000)", + [], + ) + .unwrap(); + conn.pragma_update(None, "user_version", 1).unwrap(); + } + let services = super::open_services( + &path, + dir.path().join("covers"), + Arc::new(MemoryLog::default()), + ) + .unwrap(); + let store = &services.store; + assert_eq!( + store.schema_version().unwrap(), + furumi_library::podcasts::PODCAST_SCHEMA_VERSION + ); + let rows = store.list_podcasts(PodcastListFilter::Subscribed).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].meta.title, "Old Show"); + assert_eq!(rows[0].episode_count, 1); + let expected = + PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed.xml").unwrap()); + assert_eq!(rows[0].podcast_id, expected); + let fetch = store.fetch_state(&expected).unwrap(); + assert_eq!(fetch.etag.as_deref(), Some("\"e1\"")); + drop(services); + + // The original rows are kept, renamed. + let conn = rusqlite::Connection::open(&path).unwrap(); + let legacy: i64 = conn + .query_row("SELECT COUNT(*) FROM legacy_podcast_episodes", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(legacy, 1); +} + +#[test] +fn foreign_or_newer_databases_are_refused_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let log: Arc = Arc::new(MemoryLog::default()); + + let foreign = dir.path().join("foreign.sqlite3"); + { + let conn = rusqlite::Connection::open(&foreign).unwrap(); + conn.execute_batch("CREATE TABLE tracks (id INTEGER PRIMARY KEY, title TEXT);") + .unwrap(); + } + let newer = dir.path().join("newer.sqlite3"); + { + let conn = rusqlite::Connection::open(&newer).unwrap(); + conn.pragma_update(None, "user_version", 99).unwrap(); + } + let garbage = dir.path().join("garbage.sqlite3"); + std::fs::write( + &garbage, + b"this is not a database at all, just some text bytes", + ) + .unwrap(); + + for (path, expected) in [ + (&foreign, "not a Furumi podcast database"), + (&newer, "newer than this Furumi supports"), + (&garbage, ""), + ] { + let before = std::fs::read(path).unwrap(); + let Err(err) = super::open_services(path, dir.path().join("covers"), Arc::clone(&log)) + else { + panic!("{} opened", path.display()); + }; + let message = super::describe_open_error(&err); + assert!(message.contains(expected), "{message}"); + assert_eq!(std::fs::read(path).unwrap(), before, "{}", path.display()); + } +}