Fetch podcast feeds and stamp store actions from the sync log

Add src/podcasts: an HTTP client with conditional requests, hand-followed
redirects (permanent ones become feed aliases via PodcastStore::apply_feed),
size and time limits and a furumi user agent; subscribe and refresh flows
that parse feeds with the shared music-dht parser and keep the store's
local fetch state with exponential back-off and Retry-After.

DeviceSync::run_podcast_action hands PodcastStore actions stamps reserved
in the device-sync log and appends the operations they return with exactly
those stamps. Podcast operations stay out of outgoing sync batches until
this client speaks sync protocol v4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-19 19:54:48 +02:00
1 parent bc04587252
commit 07dcd0587c
8 files changed
+1892

No files matched your search

+129
View File
@@ -13,6 +13,9 @@ use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context as _, Result};
use furumi_library::podcasts::{
OpStamp as PodcastOpStamp, PodcastLocalOp, StampSource as PodcastStampSource,
};
use music_dht::device_sync::{ListenEvent, ListenTrackMetadata};
use music_dht::{ByteStream, MusicDhtService, NetworkId, PeerTicket, SecretKey, StreamAcceptor};
use rusqlite::{Connection, OptionalExtension, params};
@@ -1325,6 +1328,30 @@ impl DeviceSync {
Ok(())
}
/// Runs a `PodcastStore` local action with stamps reserved in this
/// device's sync log and appends the operations it returns with exactly
/// those stamps (`op_id`, `seq`, `hlc_ms`).
///
/// The log connection stays locked from the first stamp until the
/// operations are appended, so no sync batch can read past a reserved
/// `seq` before its operation exists. Lock order is podcast store, then
/// this log: never call the podcast store while holding `self.conn`.
pub fn run_podcast_action<T>(
&self,
action: impl FnOnce(&mut dyn PodcastStampSource) -> Result<(T, Vec<PodcastLocalOp>)>,
) -> Result<T> {
let identity = self.ensure_identity()?;
let mut log = PodcastLogStamps {
conn: &self.conn,
guard: None,
device_id: identity.device_id,
reserved: BTreeMap::new(),
};
let (value, ops) = action(&mut log)?;
log.append(ops)?;
Ok(value)
}
pub async fn sync_once(
&self,
service: Arc<MusicDhtService>,
@@ -2303,6 +2330,9 @@ impl DeviceSync {
}
fn ops_for_peer(&self, peer_device_id: &str) -> Result<Vec<SyncOpWire>> {
// Podcast operations (`podcast_*`, protocol v4) stay in the log but
// are not sent yet: this client still announces protocol v3 and its
// wire types cannot carry them. Gated sending comes with sync v4.
let conn = lock(&self.conn);
let mut stmt = conn.prepare(
"SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json
@@ -2310,6 +2340,7 @@ impl DeviceSync {
LEFT JOIN sync_peer_acks a
ON a.peer_device_id = ?1 AND a.origin_device_id = o.origin_device_id
WHERE o.seq > COALESCE(a.max_seq, 0)
AND substr(o.kind, 1, 8) != 'podcast_'
AND (
o.kind != 'playback_command'
OR (
@@ -2800,6 +2831,104 @@ impl DeviceSync {
}
}
/// Stamp source over the sync log for [`DeviceSync::run_podcast_action`].
struct PodcastLogStamps<'a> {
conn: &'a std::sync::Mutex<Connection>,
/// Taken at the first stamp and held until the operations are appended.
guard: Option<std::sync::MutexGuard<'a, Connection>>,
device_id: String,
/// `op_id` -> reserved `seq`.
reserved: BTreeMap<String, i64>,
}
impl PodcastStampSource for PodcastLogStamps<'_> {
fn device_id(&self) -> &str {
&self.device_id
}
fn next_stamp(&mut self) -> Result<PodcastOpStamp> {
let conn = self.guard.get_or_insert_with(|| lock(self.conn));
let seq = get_meta(conn, "local_seq")?
.and_then(|v| v.parse::<i64>().ok())
.unwrap_or(0)
+ 1;
let last_hlc = get_meta(conn, "last_hlc_ms")?
.and_then(|v| v.parse::<i64>().ok())
.unwrap_or(0);
let hlc_ms = now_ms().max(last_hlc + 1);
set_meta(conn, "local_seq", &seq.to_string())?;
set_meta(conn, "last_hlc_ms", &hlc_ms.to_string())?;
let op_id = format!("{}:{seq}", self.device_id);
self.reserved.insert(op_id.clone(), seq);
Ok(PodcastOpStamp::new(op_id, self.device_id.clone(), hlc_ms))
}
}
impl PodcastLogStamps<'_> {
fn append(mut self, ops: Vec<PodcastLocalOp>) -> Result<()> {
if ops.len() != self.reserved.len() {
tracing::warn!(
reserved = self.reserved.len(),
returned = ops.len(),
"podcast action reserved stamps it did not return operations for"
);
}
if ops.is_empty() {
return Ok(());
}
let conn = self.guard.get_or_insert_with(|| lock(self.conn));
for op in ops {
let Some(&seq) = self.reserved.get(&op.stamp.op_id) else {
anyhow::bail!(
"podcast operation {} was not stamped by this sync log",
op.stamp.op_id
);
};
let op = op.into_wire(seq);
let payload = serde_json::to_value(&op.payload)?;
let kind = payload
.get("kind")
.and_then(serde_json::Value::as_str)
.unwrap_or("podcast_unknown")
.to_string();
conn.execute(
"INSERT OR IGNORE INTO sync_ops
(op_id, origin_device_id, seq, kind, payload_json, hlc_ms,
received_at_ms, tombstone)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![
op.op_id,
op.origin_device_id,
op.seq,
kind,
payload.to_string(),
op.hlc_ms,
now_ms(),
i64::from(op.payload.is_tombstone()),
],
)?;
conn.execute(
"INSERT INTO sync_vectors (device_id, max_seq)
VALUES (?1, ?2)
ON CONFLICT(device_id) DO UPDATE SET
max_seq = MAX(max_seq, excluded.max_seq)",
params![op.origin_device_id, op.seq],
)?;
tracing::debug!(op_id = %op.op_id, kind, "recorded podcast sync op");
}
Ok(())
}
}
impl crate::podcasts::OpLog for DeviceSync {
fn record(
&self,
action: &mut dyn FnMut(&mut dyn PodcastStampSource) -> Result<Vec<PodcastLocalOp>>,
) -> Result<()> {
self.run_podcast_action(|stamps| Ok(((), action(stamps)?)))
}
}
#[derive(Debug)]
struct StoredDevice {
device_id: String,
+133
View File
@@ -937,3 +937,136 @@ fn stale_synced_playlist_item_metadata_repairs_pending_fed_track() {
assert!(detail.tracks[0].is_fed_pending());
assert_eq!(detail.tracks[0].title, fed.title);
}
fn podcast_meta() -> music_dht::podcast::PodcastMeta {
music_dht::podcast::PodcastMeta {
feed_url: music_dht::podcast::FeedUrl::parse("https://example.test/feed.xml").unwrap(),
title: "Show".into(),
author: None,
description: None,
link: None,
image_url: None,
}
}
/// Rows of the local sync log: (op_id, seq, kind, payload_json, hlc_ms).
fn logged_ops(sync: &DeviceSync) -> Vec<(String, i64, String, String, i64)> {
let conn = lock(&sync.conn);
let mut stmt = conn
.prepare("SELECT op_id, seq, kind, payload_json, hlc_ms FROM sync_ops ORDER BY seq")
.unwrap();
stmt.query_map([], |row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap()
}
#[test]
fn podcast_actions_are_appended_to_the_sync_log_with_their_stamps() {
let sync = test_sync();
let device_id = sync.ensure_identity().unwrap().device_id;
let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap();
sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true)
.unwrap();
let outcome = sync
.run_podcast_action(|stamps| {
let outcome = store.subscribe(stamps, &podcast_meta())?;
let ops = outcome.ops.clone();
Ok((outcome, ops))
})
.unwrap();
let ops = logged_ops(&sync);
assert_eq!(ops.len(), 2, "{ops:?}");
let (op_id, seq, kind, payload_json, hlc_ms) = &ops[1];
assert_eq!(kind, "podcast_subscription_set");
assert_eq!(*seq, 2);
assert_eq!(*op_id, format!("{device_id}:2"));
assert!(*hlc_ms > ops[0].4, "hlc keeps increasing");
// The log carries the shared FRID payload, and the store's register
// was written with exactly this stamp.
let payload: music_dht::device_sync::SyncOpPayload =
serde_json::from_str(payload_json).unwrap();
assert!(payload.is_podcast());
let register = store.subscription(&outcome.podcast_id).unwrap().unwrap();
assert_eq!(register.op_id, *op_id);
assert_eq!(register.hlc_ms, *hlc_ms);
let vector = sync.vector().unwrap();
assert_eq!(vector.get(&device_id), Some(&2));
// Unsubscribing emits two operations with consecutive stamps; a music
// operation afterwards continues the same sequence.
let unsubscribe = sync
.run_podcast_action(|stamps| {
let ops = store.unsubscribe(stamps, &outcome.podcast_id)?;
Ok((ops.len(), ops))
})
.unwrap();
assert_eq!(unsubscribe, 2);
sync.record_content_like(&format!("b3:{}", "b".repeat(64)), true)
.unwrap();
let ops = logged_ops(&sync);
let kinds: Vec<(i64, &str)> = ops
.iter()
.map(|(_, seq, kind, ..)| (*seq, kind.as_str()))
.collect();
assert_eq!(
kinds,
[
(1, "track_like_set"),
(2, "podcast_subscription_set"),
(3, "podcast_subscription_set"),
(4, "podcast_share_set"),
(5, "track_like_set"),
]
);
// An action that emits nothing leaves the log alone.
sync.run_podcast_action(|stamps| {
let ops = store.unsubscribe(stamps, &outcome.podcast_id)?;
Ok(((), ops))
})
.unwrap();
assert_eq!(logged_ops(&sync).len(), 5);
}
#[test]
fn podcast_ops_are_not_sent_to_peers_before_sync_v4() {
let sync = test_sync();
let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap();
sync.run_podcast_action(|stamps| {
let outcome = store.subscribe(stamps, &podcast_meta())?;
let ops = outcome.ops.clone();
Ok(((), ops))
})
.unwrap();
sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true)
.unwrap();
// This client still speaks protocol v3; its wire types cannot carry the
// podcast payload, so the batch keeps only the music operation.
let ops = sync.ops_for_peer("dev_peer").unwrap();
assert_eq!(ops.len(), 1);
assert!(matches!(ops[0].payload, SyncOpPayload::TrackLikeSet { .. }));
}
#[test]
fn failed_podcast_actions_append_nothing() {
let sync = test_sync();
let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap();
let unknown = music_dht::podcast::PodcastId::from_feed_url(
&music_dht::podcast::FeedUrl::parse("https://example.test/other.xml").unwrap(),
);
let result = sync.run_podcast_action(|stamps| {
let ops = store.set_shared(stamps, &unknown, true)?;
Ok(((), ops))
});
assert!(result.is_err());
assert!(logged_ops(&sync).is_empty());
}
+1
View File
@@ -7,6 +7,7 @@ mod jam;
mod library;
mod media;
mod player;
mod podcasts;
mod share;
mod similarity;
mod status;
+347
View File
@@ -0,0 +1,347 @@
//! HTTP for podcast feeds and cover images.
//!
//! Redirects are followed by hand so the caller learns where the feed
//! really lives and whether every hop was permanent (301/308): only then
//! does the store record the new URL as a feed alias. Conditional requests
//! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and
//! the user agent are fixed here; parsing and storage happen elsewhere.
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use anyhow::{Context, Result, anyhow, bail};
use music_dht::podcast::{FeedUrl, PodcastId};
use reqwest::header::{
ACCEPT, ETAG, HeaderMap, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED, LOCATION, RETRY_AFTER,
};
use reqwest::{StatusCode, Url};
/// Hard cap on a feed document (large catalogs are a few MiB).
pub const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
/// Hard cap on a cover image.
pub const MAX_COVER_BYTES: usize = 10 * 1024 * 1024;
/// Redirect hops followed before giving up.
pub const MAX_REDIRECTS: usize = 10;
/// TCP + TLS connection setup.
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
/// Whole request, including the body, per hop.
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
/// Sent with every podcast request.
pub const USER_AGENT: &str = concat!("furumi/", env!("CARGO_PKG_VERSION"), " (podcast reader)");
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
text/xml;q=0.9, */*;q=0.5";
/// HTTP validators of the last successful fetch.
#[derive(Debug, Clone, Copy, Default)]
pub struct Validators<'a> {
pub etag: Option<&'a str>,
pub last_modified: Option<&'a str>,
}
/// What a feed request returned.
#[derive(Debug)]
pub enum FeedBody {
/// `304 Not Modified` to a conditional request.
NotModified,
/// A new document with its validators.
Modified {
body: Vec<u8>,
etag: Option<String>,
last_modified: Option<String>,
},
}
/// A feed response and the redirect chain that led to it.
#[derive(Debug)]
pub struct FeedResponse {
pub body: FeedBody,
/// URL that finally answered, canonicalized.
pub final_url: FeedUrl,
/// At least one redirect was followed and every one of them was
/// permanent (301/308). Temporary redirects (tracking prefixes, CDNs)
/// never move a feed.
pub moved_permanently: bool,
/// Redirect hops followed.
pub redirects: usize,
}
/// A non-success HTTP status, with the server's `Retry-After` (seconds form)
/// so the scheduler can back off as asked.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HttpStatusError {
pub status: u16,
pub retry_after: Option<Duration>,
}
impl std::fmt::Display for HttpStatusError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let reason = StatusCode::from_u16(self.status)
.ok()
.and_then(|status| status.canonical_reason())
.unwrap_or("error");
write!(f, "HTTP {} {reason}", self.status)
}
}
impl std::error::Error for HttpStatusError {}
/// Shared HTTP client for feeds and covers.
pub struct Fetcher {
client: reqwest::Client,
covers_dir: PathBuf,
}
impl Fetcher {
pub fn new(covers_dir: PathBuf) -> Result<Fetcher> {
let _ = rustls::crypto::ring::default_provider().install_default();
let client = reqwest::Client::builder()
.connect_timeout(CONNECT_TIMEOUT)
.timeout(REQUEST_TIMEOUT)
.redirect(reqwest::redirect::Policy::none())
.user_agent(USER_AGENT)
.build()
.context("building the podcast HTTP client")?;
Ok(Fetcher { client, covers_dir })
}
/// GET of a feed. `validators` make it conditional; they are dropped
/// after a permanent redirect because they belong to the old URL.
pub async fn fetch_feed(
&self,
url: &FeedUrl,
validators: Validators<'_>,
) -> Result<FeedResponse> {
let Followed {
response,
url: final_url,
moved_permanently,
redirects,
conditional,
} = self
.get_following(url.as_str(), Some(FEED_ACCEPT), validators)
.await?;
let final_url = FeedUrl::parse(final_url.as_str()).map_err(|err| {
anyhow!(
"the feed was redirected to an unsupported URL {}: {err}",
redact(&final_url)
)
})?;
let status = response.status();
if status == StatusCode::NOT_MODIFIED {
if !conditional {
bail!("the server answered 304 Not Modified to an unconditional request");
}
return Ok(FeedResponse {
body: FeedBody::NotModified,
final_url,
moved_permanently,
redirects,
});
}
if !status.is_success() {
return Err(status_error(status, response.headers()).into());
}
let etag = header_string(response.headers(), ETAG);
let last_modified = header_string(response.headers(), LAST_MODIFIED);
let body = read_capped(response, MAX_FEED_BYTES, "feed").await?;
Ok(FeedResponse {
body: FeedBody::Modified {
body,
etag,
last_modified,
},
final_url,
moved_permanently,
redirects,
})
}
/// Downloads a cover image into the covers directory; returns its path.
pub async fn fetch_cover(&self, podcast_id: &PodcastId, image_url: &str) -> Result<PathBuf> {
let response = self
.get_following(image_url, None, Validators::default())
.await?
.response;
let status = response.status();
if !status.is_success() {
return Err(status_error(status, response.headers()).into());
}
let body = read_capped(response, MAX_COVER_BYTES, "cover image").await?;
let format = image::guess_format(&body).context("the cover is not a supported image")?;
let extension = format.extensions_str().first().copied().unwrap_or("img");
let name = cover_file_name(podcast_id, image_url, extension);
let dir = self.covers_dir.clone();
tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body))
.await
.map_err(|err| anyhow!("cover writer task failed: {err}"))?
}
/// GET following up to [`MAX_REDIRECTS`] http(s) redirects by hand.
async fn get_following(
&self,
url: &str,
accept: Option<&str>,
validators: Validators<'_>,
) -> Result<Followed> {
let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?;
check_scheme(&current)?;
let mut all_permanent = true;
let mut send_validators = true;
let mut hops = 0;
loop {
let mut request = self.client.get(current.clone());
if let Some(accept) = accept {
request = request.header(ACCEPT, accept);
}
if send_validators {
if let Some(etag) = validators.etag {
request = request.header(IF_NONE_MATCH, etag);
}
if let Some(last_modified) = validators.last_modified {
request = request.header(IF_MODIFIED_SINCE, last_modified);
}
}
let response = request
.send()
.await
// reqwest errors print the full URL, query tokens included.
.map_err(reqwest::Error::without_url)
.with_context(|| format!("requesting {}", redact(&current)))?;
let status = response.status();
if !status.is_redirection() || status == StatusCode::NOT_MODIFIED {
let conditional = send_validators
&& (validators.etag.is_some() || validators.last_modified.is_some());
return Ok(Followed {
response,
url: current,
moved_permanently: hops > 0 && all_permanent,
redirects: hops,
conditional,
});
}
let location = response
.headers()
.get(LOCATION)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| anyhow!("HTTP {status} redirect without a Location header"))?;
let next = current
.join(location.trim())
.with_context(|| format!("invalid redirect location {location:?}"))?;
check_scheme(&next)?;
hops += 1;
if hops > MAX_REDIRECTS {
bail!("too many redirects (more than {MAX_REDIRECTS})");
}
let permanent = matches!(
status,
StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT
);
all_permanent &= permanent;
if permanent {
send_validators = false;
}
current = next;
}
}
}
struct Followed {
response: reqwest::Response,
/// URL that answered.
url: Url,
/// At least one hop and all of them permanent.
moved_permanently: bool,
redirects: usize,
/// The final request carried validators.
conditional: bool,
}
fn check_scheme(url: &Url) -> Result<()> {
if !matches!(url.scheme(), "http" | "https") {
bail!("unsupported URL scheme {:?}", url.scheme());
}
if url.host_str().is_none_or(str::is_empty) {
bail!("URL has no host");
}
Ok(())
}
/// The URL without query and fragment, for messages and logs: feed URLs
/// may carry private tokens in their query.
pub fn redact(url: &Url) -> String {
let mut shown = url.clone();
if shown.query().is_some() {
shown.set_query(Some("…"));
}
shown.set_fragment(None);
shown.to_string()
}
fn header_string(headers: &HeaderMap, name: reqwest::header::HeaderName) -> Option<String> {
headers
.get(name)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string)
}
fn status_error(status: StatusCode, headers: &HeaderMap) -> HttpStatusError {
let retry_after = header_string(headers, RETRY_AFTER)
.and_then(|value| value.parse::<u64>().ok())
.map(Duration::from_secs);
HttpStatusError {
status: status.as_u16(),
retry_after,
}
}
async fn read_capped(mut response: reqwest::Response, cap: usize, what: &str) -> Result<Vec<u8>> {
if let Some(len) = response.content_length()
&& len > cap as u64
{
bail!("the {what} is too large ({len} bytes, limit {cap})");
}
let mut body = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(reqwest::Error::without_url)
.with_context(|| format!("reading the {what}"))?
{
if body.len() + chunk.len() > cap {
bail!("the {what} is too large (over {cap} bytes)");
}
body.extend_from_slice(&chunk);
}
Ok(body)
}
/// Stable per podcast and image URL, so a changed image gets a new file and
/// the previous one can be removed.
fn cover_file_name(podcast_id: &PodcastId, image_url: &str, extension: &str) -> String {
let podcast = blake3::hash(podcast_id.as_str().as_bytes()).to_hex();
let image = blake3::hash(image_url.as_bytes()).to_hex();
format!("{}-{}.{extension}", &podcast[..16], &image[..16])
}
fn write_atomically(dir: &Path, name: &str, bytes: &[u8]) -> Result<PathBuf> {
static COUNTER: AtomicU64 = AtomicU64::new(0);
std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
let target = dir.join(name);
let temp = dir.join(format!(
".{name}.{}-{}.tmp",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
));
let result = std::fs::write(&temp, bytes)
.and_then(|()| std::fs::rename(&temp, &target))
.with_context(|| format!("writing {}", target.display()));
if result.is_err() {
let _ = std::fs::remove_file(&temp);
}
result?;
Ok(target)
}
+107
View File
@@ -0,0 +1,107 @@
//! Podcast runtime of the TUI: feed HTTP, refresh scheduling and the glue
//! between the shared podcast store and the device-sync log.
//!
//! The data model, identity rules, merge rules and the SQLite store are not
//! defined here: they come from FRID (`music_dht::podcast` and
//! `furumi_library::podcasts::PodcastStore`, file `podcasts.sqlite3`). This
//! module only does what the contract leaves to clients:
//!
//! * [`fetch`] downloads feeds (conditional GET, redirects reported so
//! permanent moves become feed aliases, size and time limits) and covers;
//! * [`refresh`] subscribes and refreshes feeds: it parses them with the
//! shared `music_dht::podcast::feed` parser, ingests them with
//! `PodcastStore::apply_feed` and keeps the local fetch state (validators,
//! errors, back-off);
//! * [`OpLog`] hands store actions the stamps of the device-sync log and
//! appends the operations they return.
//!
//! Only RSS metadata is stored. Audio stays on the publisher's server.
pub mod fetch;
pub mod refresh;
#[cfg(test)]
mod test_server;
#[cfg(test)]
mod tests;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Result, anyhow};
use furumi_library::podcasts::{
PODCAST_DB_FILE, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource,
};
pub use fetch::Fetcher;
use refresh::Services;
/// Appends podcast operations to the device-sync log.
///
/// [`OpLog::record`] gives `action` a [`StampSource`] that reserves stamps
/// in the log. The action runs a `PodcastStore` local action and returns the
/// operations it emitted; they are appended with exactly those stamps.
pub trait OpLog: Send + Sync {
fn record(
&self,
action: &mut dyn FnMut(&mut dyn StampSource) -> Result<Vec<PodcastLocalOp>>,
) -> Result<()>;
}
/// The podcast store file. It is the file the prototype podcast build
/// used, so an existing prototype database is migrated in place.
pub fn db_path() -> Result<PathBuf> {
let dirs = crate::config::project_dirs()
.ok_or_else(|| anyhow!("no home directory for podcast data"))?;
Ok(dirs.data_dir().join(PODCAST_DB_FILE))
}
/// Replaceable cache of downloaded podcast covers.
pub fn covers_dir() -> Result<PathBuf> {
let dirs = crate::config::project_dirs()
.ok_or_else(|| anyhow!("no cache directory for podcast covers"))?;
Ok(dirs.cache_dir().join("podcasts").join("covers"))
}
/// Opens the podcast store at `db_path` and builds the feed client.
///
/// `PodcastStore::open` migrates a prototype database in place and refuses,
/// without writing anything, a database from a newer Furumi or a file that
/// is not a podcast database. Blocking: call it on the blocking pool.
pub fn open_services(db_path: &Path, covers_dir: PathBuf, log: Arc<dyn OpLog>) -> Result<Services> {
let store = PodcastStore::open(db_path)?;
tracing::info!(path = %db_path.display(), "podcast store opened");
match store.legacy_quarantine() {
Ok(entries) if !entries.is_empty() => tracing::warn!(
count = entries.len(),
"some prototype podcast rows could not be imported; they are kept in the legacy_* tables"
),
Ok(_) => {}
Err(err) => tracing::debug!("reading the podcast import quarantine failed: {err:#}"),
}
Ok(Services {
store: Arc::new(store),
fetcher: Arc::new(Fetcher::new(covers_dir)?),
log,
})
}
/// A readable reason why the store did not open; the typed store errors
/// already say that the file was left unchanged.
pub fn describe_open_error(err: &anyhow::Error) -> String {
match err.downcast_ref::<PodcastStoreError>() {
Some(store_error) => store_error.to_string(),
None => format!("{err:#}"),
}
}
/// Runs blocking store or filesystem work off the Tokio workers.
pub(crate) async fn blocking<T, F>(work: F) -> Result<T>
where
T: Send + 'static,
F: FnOnce() -> Result<T> + Send + 'static,
{
tokio::task::spawn_blocking(work)
.await
.map_err(|err| anyhow!("background task failed: {err}"))?
}
+380
View File
@@ -0,0 +1,380 @@
//! Subscribing and refreshing feeds.
//!
//! Every function records its HTTP outcome in the store's local fetch state
//! (validators, last error, consecutive failures, next check) so the
//! scheduler never needs its own bookkeeping, and an error never clears the
//! stored catalog.
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context as _, Result, anyhow, bail};
use furumi_library::podcasts::{FeedRefresh, PodcastStore};
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use music_dht::podcast::{FeedUrl, PodcastId};
use super::fetch::{FeedBody, Fetcher, HttpStatusError, Validators};
use super::{OpLog, blocking};
/// A successfully checked feed is checked again after this long.
pub const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60);
/// First retry after a failed check; doubles per consecutive failure.
pub const FAILURE_BACKOFF_BASE: Duration = Duration::from_secs(5 * 60);
/// Longest wait between checks of a failing feed (also caps `Retry-After`).
pub const FAILURE_BACKOFF_MAX: Duration = Duration::from_secs(24 * 60 * 60);
/// Store, HTTP client and sync log shared by podcast background tasks.
#[derive(Clone)]
pub struct Services {
pub store: Arc<PodcastStore>,
pub fetcher: Arc<Fetcher>,
pub log: Arc<dyn OpLog>,
}
impl std::fmt::Debug for Services {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Services")
.field("store", &self.store)
.finish_non_exhaustive()
}
}
/// Result of [`subscribe`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SubscribeReport {
pub podcast_id: PodcastId,
pub title: String,
/// The podcast was already subscribed; nothing was fetched or written.
pub already_subscribed: bool,
/// Episodes the first refresh added to the catalog.
pub new_episodes: usize,
}
/// Result of [`refresh`].
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RefreshReport {
/// The server answered 304 Not Modified.
pub not_modified: bool,
pub new_episodes: usize,
pub updated_episodes: usize,
/// Feed items without identity or with invalid metadata.
pub skipped_items: usize,
/// New fetch URL after a permanent redirect or `itunes:new-feed-url`.
pub moved_to: Option<FeedUrl>,
}
/// When to check a feed again after a success.
pub fn next_check_after_success(now_ms: i64) -> i64 {
now_ms.saturating_add(duration_ms(REFRESH_INTERVAL))
}
/// When to check a feed again after its `failures`-th consecutive failure:
/// exponential back-off, at least the server's `Retry-After`, at most
/// [`FAILURE_BACKOFF_MAX`].
pub fn next_check_after_failure(now_ms: i64, failures: u32, retry_after: Option<Duration>) -> i64 {
let exponent = failures.saturating_sub(1).min(16);
let backoff = FAILURE_BACKOFF_BASE
.saturating_mul(1 << exponent)
.max(retry_after.unwrap_or_default())
.min(FAILURE_BACKOFF_MAX);
now_ms.saturating_add(duration_ms(backoff))
}
fn duration_ms(duration: Duration) -> i64 {
i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)
}
/// Subscribes to the feed the user typed (`FeedUrl::from_user_input`).
///
/// Nothing is stored unless the URL serves a parsable feed. A URL that
/// resolves (directly or through a known alias or redirect) to a podcast
/// this device already knows reuses its id, so resubscribing continues the
/// saved progress and history. The subscription is private.
pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<SubscribeReport> {
let url = FeedUrl::from_user_input(input).map_err(|err| anyhow!("{err}"))?;
let store = Arc::clone(&services.store);
let lookup = url.clone();
let existing = blocking(move || {
let id = store.resolve_podcast_id(&lookup)?;
let subscribed = store
.subscription(&id)?
.is_some_and(|record| record.subscribed);
let title = store.podcast(&id)?.map(|summary| summary.meta.title);
Ok(subscribed.then(|| (id, title)))
})
.await?;
if let Some((podcast_id, title)) = existing {
return Ok(SubscribeReport {
podcast_id,
title: title.unwrap_or_default(),
already_subscribed: true,
new_episodes: 0,
});
}
let response = services
.fetcher
.fetch_feed(&url, Validators::default())
.await?;
let FeedBody::Modified {
body,
etag,
last_modified,
} = response.body
else {
bail!("the server answered 304 Not Modified to an unconditional request");
};
let final_url = response.final_url;
let feed = parse(body, &final_url).await?;
let store = Arc::clone(&services.store);
let log = Arc::clone(&services.log);
let moved_permanently = response.moved_permanently;
let image_url = feed.image_url.clone();
let report = blocking(move || {
// Identity comes from the URL of the first subscription. Reuse a
// podcast this device already knows under the typed or the final
// URL; otherwise the typed URL defines the id and a permanent
// redirect becomes an alias below.
let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? {
url.clone()
} else {
final_url.clone()
};
let meta = feed.to_podcast_meta(identity_url);
let mut subscribed = None;
log.record(&mut |stamps| {
let outcome = store.subscribe(stamps, &meta)?;
let ops = outcome.ops.clone();
subscribed = Some(outcome);
Ok(ops)
})?;
let outcome = subscribed.ok_or_else(|| anyhow!("subscription was not recorded"))?;
let podcast_id = outcome.podcast_id;
let mut refreshed = None;
log.record(&mut |stamps| {
let outcome = store.apply_feed(
stamps,
&FeedRefresh {
podcast_id: &podcast_id,
feed: &feed,
fetched_from: &final_url,
moved_permanently,
},
)?;
let ops = outcome.ops.clone();
refreshed = Some(outcome);
Ok(ops)
})?;
let refreshed = refreshed.ok_or_else(|| anyhow!("feed refresh was not recorded"))?;
store.record_fetch_success(
&podcast_id,
now_ms,
etag.as_deref(),
last_modified.as_deref(),
Some(next_check_after_success(now_ms)),
)?;
Ok(SubscribeReport {
podcast_id,
title: meta.title,
already_subscribed: outcome.already_subscribed,
new_episodes: refreshed.new_episodes,
})
})
.await?;
if let Some(image_url) = image_url {
update_cover(services, &report.podcast_id, &image_url).await;
}
Ok(report)
}
fn knows(store: &PodcastStore, url: &FeedUrl) -> Result<bool> {
let id = store.resolve_podcast_id(url)?;
Ok(store.podcast(&id)?.is_some() || store.subscription(&id)?.is_some())
}
/// Checks one feed (conditional GET from `url`, the store's fetch URL) and
/// ingests a changed document. Failures are recorded in the fetch state
/// with back-off and returned; the stored catalog is kept.
pub async fn refresh(
services: &Services,
podcast_id: &PodcastId,
url: &FeedUrl,
now_ms: i64,
) -> Result<RefreshReport> {
let store = Arc::clone(&services.store);
let id = podcast_id.clone();
let fetch_state = blocking(move || store.fetch_state(&id)).await?;
let result = fetch_and_apply(services, podcast_id, url, &fetch_state, now_ms).await;
if let Err(err) = &result {
let message = format!("{err:#}");
let retry_after = err
.downcast_ref::<HttpStatusError>()
.and_then(|status| status.retry_after);
let next = next_check_after_failure(
now_ms,
fetch_state.consecutive_failures.saturating_add(1),
retry_after,
);
let store = Arc::clone(&services.store);
let id = podcast_id.clone();
if let Err(record_err) = blocking(move || {
store
.record_fetch_failure(&id, now_ms, &message, Some(next))
.map(|_| ())
})
.await
{
tracing::warn!("recording a podcast refresh error failed: {record_err:#}");
}
}
result
}
async fn fetch_and_apply(
services: &Services,
podcast_id: &PodcastId,
url: &FeedUrl,
fetch_state: &furumi_library::podcasts::FetchState,
now_ms: i64,
) -> Result<RefreshReport> {
let response = services
.fetcher
.fetch_feed(
url,
Validators {
etag: fetch_state.etag.as_deref(),
last_modified: fetch_state.last_modified.as_deref(),
},
)
.await?;
tracing::debug!(
podcast = %podcast_id,
redirects = response.redirects,
moved_permanently = response.moved_permanently,
not_modified = matches!(response.body, FeedBody::NotModified),
"podcast feed checked"
);
let next = Some(next_check_after_success(now_ms));
let (body, etag, last_modified) = match response.body {
FeedBody::NotModified => {
let store = Arc::clone(&services.store);
let id = podcast_id.clone();
blocking(move || store.record_fetch_not_modified(&id, now_ms, next)).await?;
return Ok(RefreshReport {
not_modified: true,
..RefreshReport::default()
});
}
FeedBody::Modified {
body,
etag,
last_modified,
} => (body, etag, last_modified),
};
let final_url = response.final_url;
let feed = parse(body, &final_url).await?;
let image_url = feed.image_url.clone();
let store = Arc::clone(&services.store);
let log = Arc::clone(&services.log);
let id = podcast_id.clone();
let previous_url = url.clone();
let moved_permanently = response.moved_permanently;
let report = blocking(move || {
let mut refreshed = None;
log.record(&mut |stamps| {
let outcome = store.apply_feed(
stamps,
&FeedRefresh {
podcast_id: &id,
feed: &feed,
fetched_from: &final_url,
moved_permanently,
},
)?;
let ops = outcome.ops.clone();
refreshed = Some(outcome);
Ok(ops)
})?;
let outcome = refreshed.ok_or_else(|| anyhow!("feed refresh was not recorded"))?;
store.record_fetch_success(&id, now_ms, etag.as_deref(), last_modified.as_deref(), next)?;
Ok(RefreshReport {
not_modified: false,
new_episodes: outcome.new_episodes,
updated_episodes: outcome.updated_episodes,
skipped_items: outcome.skipped_items,
moved_to: outcome
.feed_url
.filter(|feed_url| *feed_url != previous_url),
})
})
.await?;
if let Some(image_url) = image_url {
update_cover(services, podcast_id, &image_url).await;
}
Ok(report)
}
/// Parses a feed document on the blocking pool; relative URLs resolve
/// against where the document was served from.
async fn parse(body: Vec<u8>, base: &FeedUrl) -> Result<ParsedFeed> {
let base = base.as_str().to_string();
blocking(move || {
parse_feed_with_base(&body, Some(&base)).map_err(|err| anyhow!("unusable feed: {err}"))
})
.await
}
/// Downloads the cover when the feed image changed or the cached file is
/// gone, records it and deletes the replaced file. Best effort: failures are
/// only logged, covers are a replaceable cache.
async fn update_cover(services: &Services, podcast_id: &PodcastId, image_url: &str) {
let store = Arc::clone(&services.store);
let id = podcast_id.clone();
let current = blocking(move || {
let state = store.fetch_state(&id)?;
Ok((state.cover_url, state.cover_path))
})
.await;
let (old_url, old_path) = match current {
Ok(current) => current,
Err(err) => {
tracing::warn!("reading podcast cover state failed: {err:#}");
return;
}
};
let cached = old_path
.as_deref()
.is_some_and(|path| Path::new(path).is_file());
if old_url.as_deref() == Some(image_url) && cached {
return;
}
let path = match services.fetcher.fetch_cover(podcast_id, image_url).await {
Ok(path) => path,
Err(err) => {
tracing::debug!(podcast = %podcast_id, "podcast cover download failed: {err:#}");
return;
}
};
let store = Arc::clone(&services.store);
let id = podcast_id.clone();
let image_url = image_url.to_string();
let result = blocking(move || {
let new_path = std::path::absolute(&path)
.unwrap_or(path)
.to_string_lossy()
.into_owned();
store
.set_cover_cache(&id, Some(&image_url), Some(&new_path))
.context("recording the cover")?;
if let Some(old_path) = old_path.filter(|old| *old != new_path) {
let _ = std::fs::remove_file(old_path);
}
Ok(())
})
.await;
if let Err(err) = result {
tracing::warn!("recording a podcast cover failed: {err:#}");
}
}
+207
View File
@@ -0,0 +1,207 @@
//! Minimal HTTP/1.1 server on 127.0.0.1 for podcast tests (no internet).
//! One request per connection (`Connection: close`).
use std::collections::HashMap;
use std::io::{BufRead, BufReader, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
#[derive(Clone)]
pub(super) struct Route {
pub status: u16,
pub body: Arc<Vec<u8>>,
pub content_type: String,
pub etag: Option<String>,
pub last_modified: Option<String>,
/// `Location` of a redirect status.
pub location: Option<String>,
pub retry_after: Option<String>,
}
impl Route {
pub fn new(body: impl Into<Vec<u8>>, content_type: &str) -> Route {
Route {
status: 200,
body: Arc::new(body.into()),
content_type: content_type.to_string(),
etag: None,
last_modified: None,
location: None,
retry_after: None,
}
}
pub fn redirect(status: u16, location: &str) -> Route {
Route {
status,
location: Some(location.to_string()),
..Route::new(Vec::new(), "text/plain")
}
}
pub fn status(mut self, status: u16) -> Route {
self.status = status;
self
}
pub fn etag(mut self, etag: &str) -> Route {
self.etag = Some(etag.to_string());
self
}
pub fn last_modified(mut self, value: &str) -> Route {
self.last_modified = Some(value.to_string());
self
}
pub fn retry_after(mut self, value: &str) -> Route {
self.retry_after = Some(value.to_string());
self
}
}
/// Headers of one received request (names lowercased).
pub(super) type RequestHeaders = HashMap<String, String>;
#[derive(Default)]
struct ServerState {
routes: HashMap<String, Route>,
requests: HashMap<String, Vec<RequestHeaders>>,
}
pub(super) struct TestServer {
base: String,
state: Arc<Mutex<ServerState>>,
}
impl TestServer {
pub fn start() -> TestServer {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server");
let base = format!("http://{}", listener.local_addr().expect("local addr"));
let state = Arc::new(Mutex::new(ServerState::default()));
let accept_state = Arc::clone(&state);
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
let state = Arc::clone(&accept_state);
std::thread::spawn(move || handle(stream, &state));
}
});
TestServer { base, state }
}
pub fn url(&self, path: &str) -> String {
format!("{}{path}", self.base)
}
pub fn set(&self, path: &str, route: Route) {
self.lock().routes.insert(path.to_string(), route);
}
/// Headers of every request received for `path`, in order.
pub fn requests(&self, path: &str) -> Vec<RequestHeaders> {
self.lock().requests.get(path).cloned().unwrap_or_default()
}
fn lock(&self) -> std::sync::MutexGuard<'_, ServerState> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
}
fn handle(stream: TcpStream, state: &Mutex<ServerState>) {
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
let mut reader = BufReader::new(match stream.try_clone() {
Ok(stream) => stream,
Err(_) => return,
});
let mut request_line = String::new();
if reader.read_line(&mut request_line).is_err() {
return;
}
let path = request_line
.split_whitespace()
.nth(1)
.unwrap_or("/")
.to_string();
let mut headers = RequestHeaders::new();
loop {
let mut line = String::new();
match reader.read_line(&mut line) {
Ok(0) | Err(_) => return,
Ok(_) => {}
}
let line = line.trim_end();
if line.is_empty() {
break;
}
if let Some((name, value)) = line.split_once(':') {
headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string());
}
}
let route = {
let mut state = state.lock().unwrap_or_else(PoisonError::into_inner);
state
.requests
.entry(path.clone())
.or_default()
.push(headers.clone());
state.routes.get(&path).cloned()
};
let mut out = stream;
let Some(route) = route else {
let _ = out
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
return;
};
let etag_matches = route
.etag
.as_ref()
.is_some_and(|etag| headers.get("if-none-match") == Some(etag));
let date_matches = route
.last_modified
.as_ref()
.is_some_and(|date| headers.get("if-modified-since") == Some(date));
if route.status == 200 && (etag_matches || date_matches) {
let _ = write!(
out,
"HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n"
);
return;
}
let mut extra = String::new();
for (name, value) in [
("ETag", &route.etag),
("Last-Modified", &route.last_modified),
("Location", &route.location),
("Retry-After", &route.retry_after),
] {
if let Some(value) = value {
extra.push_str(&format!("{name}: {value}\r\n"));
}
}
let reason = match route.status {
200 => "OK",
301 => "Moved Permanently",
302 => "Found",
307 => "Temporary Redirect",
308 => "Permanent Redirect",
404 => "Not Found",
503 => "Service Unavailable",
_ => "Status",
};
if write!(
out,
"HTTP/1.1 {} {reason}\r\nContent-Type: {}\r\nContent-Length: {}\r\n{extra}Connection: close\r\n\r\n",
route.status,
route.content_type,
route.body.len()
)
.is_err()
{
return;
}
let _ = out.write_all(&route.body);
let _ = out.flush();
let _ = out.shutdown(std::net::Shutdown::Both);
}
+588
View File
@@ -0,0 +1,588 @@
//! Store opening, subscribing and refreshing against a local HTTP server.
//! No internet, no real user directories: every test uses a temp dir.
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use anyhow::Result;
use furumi_library::podcasts::{
EpisodeQuery, OpStamp, PodcastListFilter, PodcastLocalOp, PodcastStore, StampSource,
};
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::{FeedUrl, PodcastId};
use super::OpLog;
use super::refresh::{
FAILURE_BACKOFF_BASE, FAILURE_BACKOFF_MAX, REFRESH_INTERVAL, Services,
next_check_after_failure, next_check_after_success, refresh, subscribe,
};
use super::test_server::{Route, TestServer};
const NOW: i64 = 1_700_000_000_000;
/// In-memory sync log: hands out stamps and keeps the appended operations.
#[derive(Default)]
struct MemoryLog {
ops: Mutex<Vec<SyncOpWire>>,
}
struct MemoryStamps {
next_seq: i64,
reserved: Vec<i64>,
}
impl StampSource for MemoryStamps {
fn device_id(&self) -> &str {
"test-device"
}
fn next_stamp(&mut self) -> Result<OpStamp> {
let seq = self.next_seq + self.reserved.len() as i64;
self.reserved.push(seq);
Ok(OpStamp::new(
format!("test-device:{seq}"),
"test-device",
NOW + seq,
))
}
}
impl OpLog for MemoryLog {
fn record(
&self,
action: &mut dyn FnMut(&mut dyn StampSource) -> Result<Vec<PodcastLocalOp>>,
) -> Result<()> {
let mut ops = self.ops.lock().unwrap_or_else(PoisonError::into_inner);
let mut stamps = MemoryStamps {
next_seq: ops.len() as i64 + 1,
reserved: Vec::new(),
};
for (op, seq) in action(&mut stamps)?.into_iter().zip(stamps.reserved) {
ops.push(op.into_wire(seq));
}
Ok(())
}
}
impl MemoryLog {
fn kinds(&self) -> Vec<String> {
self.ops
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.map(|op| {
serde_json::to_value(&op.payload).unwrap()["kind"]
.as_str()
.unwrap()
.to_string()
})
.collect()
}
}
struct Fixture {
_dir: tempfile::TempDir,
services: Services,
log: Arc<MemoryLog>,
server: TestServer,
}
fn fixture() -> Fixture {
let dir = tempfile::tempdir().unwrap();
let log = Arc::new(MemoryLog::default());
let services = super::open_services(
&dir.path().join("podcasts.sqlite3"),
dir.path().join("covers"),
log.clone(),
)
.unwrap();
Fixture {
_dir: dir,
services,
log,
server: TestServer::start(),
}
}
fn rss(items: &[(&str, &str)]) -> Vec<u8> {
let items: String = items
.iter()
.map(|(guid, title)| {
format!(
"<item><title>{title}</title><guid>{guid}</guid>\
<pubDate>Sat, 15 Jun 2024 10:00:00 GMT</pubDate>\
<itunes:duration>45:00</itunes:duration>\
<enclosure url=\"/audio/{guid}.mp3\" type=\"audio/mpeg\" length=\"100\"/></item>"
)
})
.collect();
format!(
"<?xml version=\"1.0\"?><rss version=\"2.0\" \
xmlns:itunes=\"http://www.itunes.com/dtds/podcast-1.0.dtd\"><channel>\
<title>Test Show</title><itunes:author>Host</itunes:author>{items}</channel></rss>"
)
.into_bytes()
}
fn episode_count(store: &PodcastStore, id: &PodcastId) -> usize {
store
.episodes(&EpisodeQuery::new(id.clone()))
.unwrap()
.len()
}
#[tokio::test]
async fn subscribe_then_conditional_refresh_uses_validators() {
let f = fixture();
f.server.set(
"/feed.xml",
Route::new(
rss(&[("a", "Pilot"), ("b", "Second")]),
"application/rss+xml",
)
.etag("\"v1\"")
.last_modified("Sat, 15 Jun 2024 10:00:00 GMT"),
);
let report = subscribe(&f.services, &f.server.url("/feed.xml"), NOW)
.await
.unwrap();
assert!(!report.already_subscribed);
assert_eq!(report.new_episodes, 2);
assert_eq!(report.title, "Test Show");
assert_eq!(f.log.kinds(), ["podcast_subscription_set"]);
let store = &f.services.store;
let rows = store.list_podcasts(PodcastListFilter::Subscribed).unwrap();
assert_eq!(rows.len(), 1);
assert!(!rows[0].shared, "a new subscription is private");
let state = store.fetch_state(&report.podcast_id).unwrap();
assert_eq!(state.etag.as_deref(), Some("\"v1\""));
assert_eq!(state.next_check_at_ms, Some(next_check_after_success(NOW)));
let first = &f.server.requests("/feed.xml")[0];
assert!(!first.contains_key("if-none-match"), "{first:?}");
assert!(
first
.get("user-agent")
.is_some_and(|agent| agent.starts_with("furumi/")),
"{first:?}"
);
// Unchanged: 304, nothing re-parsed, validators sent.
let url = rows[0].meta.feed_url.clone();
let checked = refresh(&f.services, &report.podcast_id, &url, NOW + 1)
.await
.unwrap();
assert!(checked.not_modified);
let second = &f.server.requests("/feed.xml")[1];
assert_eq!(
second.get("if-none-match").map(String::as_str),
Some("\"v1\"")
);
assert_eq!(
second.get("if-modified-since").map(String::as_str),
Some("Sat, 15 Jun 2024 10:00:00 GMT")
);
// Changed: a new episode arrives, the known ones are not duplicated.
f.server.set(
"/feed.xml",
Route::new(
rss(&[("a", "Pilot (remastered)"), ("b", "Second"), ("c", "Third")]),
"application/rss+xml",
)
.etag("\"v2\""),
);
let changed = refresh(&f.services, &report.podcast_id, &url, NOW + 2)
.await
.unwrap();
assert!(!changed.not_modified);
assert_eq!(changed.new_episodes, 1);
assert_eq!(changed.updated_episodes, 1);
assert_eq!(episode_count(store, &report.podcast_id), 3);
// A refresh never writes personal state: no new operation.
assert_eq!(f.log.kinds(), ["podcast_subscription_set"]);
// Subscribing again is recognised without another fetch.
let requests = f.server.requests("/feed.xml").len();
let again = subscribe(&f.services, &f.server.url("/feed.xml"), NOW + 3)
.await
.unwrap();
assert!(again.already_subscribed);
assert_eq!(again.podcast_id, report.podcast_id);
assert_eq!(f.server.requests("/feed.xml").len(), requests);
}
#[tokio::test]
async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() {
let f = fixture();
f.server
.set("/old.xml", Route::redirect(301, &f.server.url("/new.xml")));
f.server.set(
"/new.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
);
let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap();
let new = FeedUrl::parse(&f.server.url("/new.xml")).unwrap();
let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap();
// The id comes from the URL of the first subscription.
assert_eq!(report.podcast_id, PodcastId::from_feed_url(&old));
assert_eq!(
f.log.kinds(),
["podcast_subscription_set", "podcast_feed_aliases_added"]
);
let store = &f.services.store;
assert!(
store
.feed_aliases(&report.podcast_id)
.unwrap()
.contains(&new)
);
let summary = store.podcast(&report.podcast_id).unwrap().unwrap();
assert_eq!(summary.meta.feed_url, new, "fetched from the new URL now");
assert_eq!(store.resolve_podcast_id(&new).unwrap(), report.podcast_id);
// The next refresh goes straight to the new URL; no new alias op.
let checked = refresh(&f.services, &report.podcast_id, &new, NOW + 1)
.await
.unwrap();
assert!(checked.moved_to.is_none());
assert_eq!(f.server.requests("/old.xml").len(), 1);
assert_eq!(f.log.kinds().len(), 2);
// Subscribing through the new URL finds the same podcast.
let again = subscribe(&f.services, new.as_str(), NOW + 2).await.unwrap();
assert!(again.already_subscribed);
assert_eq!(again.podcast_id, report.podcast_id);
}
#[tokio::test]
async fn temporary_redirect_keeps_the_feed_url() {
let f = fixture();
f.server
.set("/tracked.xml", Route::redirect(302, "/real.xml"));
f.server.set(
"/real.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
);
let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap();
let report = subscribe(&f.services, tracked.as_str(), NOW).await.unwrap();
assert_eq!(f.log.kinds(), ["podcast_subscription_set"]);
let summary = f
.services
.store
.podcast(&report.podcast_id)
.unwrap()
.unwrap();
assert_eq!(summary.meta.feed_url, tracked);
// Episode URLs resolve against where the document was served from.
let episodes = f
.services
.store
.episodes(&EpisodeQuery::new(report.podcast_id.clone()))
.unwrap();
assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3"));
}
#[tokio::test]
async fn malformed_feed_and_server_errors_back_off_and_keep_the_catalog() {
let f = fixture();
f.server.set(
"/feed.xml",
Route::new(
rss(&[("a", "Pilot"), ("b", "Second")]),
"application/rss+xml",
),
);
let report = subscribe(&f.services, &f.server.url("/feed.xml"), NOW)
.await
.unwrap();
let id = report.podcast_id;
let url = FeedUrl::parse(&f.server.url("/feed.xml")).unwrap();
f.server.set(
"/feed.xml",
Route::new("<html><body>Moved to our app!</body></html>", "text/html"),
);
let err = refresh(&f.services, &id, &url, NOW + 1).await.unwrap_err();
assert!(format!("{err:#}").contains("unusable feed"), "{err:#}");
let store = &f.services.store;
let state = store.fetch_state(&id).unwrap();
assert_eq!(state.consecutive_failures, 1);
assert!(
state
.last_error
.as_deref()
.unwrap()
.contains("unusable feed")
);
assert_eq!(
state.next_check_at_ms,
Some(next_check_after_failure(NOW + 1, 1, None))
);
assert_eq!(episode_count(store, &id), 2, "an error keeps the catalog");
f.server.set(
"/feed.xml",
Route::new("busy", "text/plain")
.status(503)
.retry_after("7200"),
);
let err = refresh(&f.services, &id, &url, NOW + 2).await.unwrap_err();
assert!(format!("{err:#}").contains("HTTP 503"), "{err:#}");
let state = store.fetch_state(&id).unwrap();
assert_eq!(state.consecutive_failures, 2);
assert_eq!(state.next_check_at_ms, Some(NOW + 2 + 7_200_000));
assert_eq!(episode_count(store, &id), 2);
// A failing feed is not due before its back-off expires.
assert!(
store
.podcasts_due_for_refresh(NOW + 3, 10)
.unwrap()
.is_empty()
);
assert_eq!(
store
.podcasts_due_for_refresh(NOW + 2 + 7_200_000, 10)
.unwrap()
.len(),
1
);
// Recovery clears the error.
f.server.set(
"/feed.xml",
Route::new(
rss(&[("a", "Pilot"), ("b", "Second")]),
"application/rss+xml",
),
);
refresh(&f.services, &id, &url, NOW + 3).await.unwrap();
let state = store.fetch_state(&id).unwrap();
assert_eq!(state.consecutive_failures, 0);
assert!(state.last_error.is_none());
}
#[tokio::test]
async fn subscribing_to_a_non_feed_stores_nothing() {
let f = fixture();
f.server
.set("/page", Route::new("<html>hello</html>", "text/html"));
let err = subscribe(&f.services, &f.server.url("/page"), NOW)
.await
.unwrap_err();
assert!(format!("{err:#}").contains("unusable feed"), "{err:#}");
let err = subscribe(&f.services, &f.server.url("/missing"), NOW)
.await
.unwrap_err();
assert!(format!("{err:#}").contains("HTTP 404"), "{err:#}");
let err = subscribe(&f.services, "ftp://example.test/feed", NOW)
.await
.unwrap_err();
assert!(!format!("{err:#}").is_empty());
assert!(
f.services
.store
.list_podcasts(PodcastListFilter::All)
.unwrap()
.is_empty()
);
assert!(f.log.kinds().is_empty());
}
#[tokio::test]
async fn unsubscribe_then_resubscribe_keeps_the_podcast_id() {
let f = fixture();
f.server.set(
"/feed.xml",
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
);
let first = subscribe(&f.services, &f.server.url("/feed.xml"), NOW)
.await
.unwrap();
let store = Arc::clone(&f.services.store);
let id = first.podcast_id.clone();
f.log
.record(&mut |stamps| store.unsubscribe(stamps, &id))
.unwrap();
assert!(
store
.list_podcasts(PodcastListFilter::Subscribed)
.unwrap()
.is_empty()
);
let again = subscribe(&f.services, &f.server.url("/feed.xml"), NOW + 1)
.await
.unwrap();
assert!(!again.already_subscribed);
assert_eq!(again.podcast_id, first.podcast_id);
assert_eq!(
f.log.kinds(),
[
"podcast_subscription_set",
"podcast_subscription_set",
"podcast_share_set",
"podcast_subscription_set"
]
);
let ops = f.log.ops.lock().unwrap();
assert!(matches!(
ops.last().map(|op| &op.payload),
Some(SyncOpPayload::PodcastSubscriptionSet {
subscribed: true,
..
})
));
}
#[test]
fn back_off_doubles_and_honours_retry_after() {
let base = FAILURE_BACKOFF_BASE.as_millis() as i64;
assert_eq!(next_check_after_failure(0, 1, None), base);
assert_eq!(next_check_after_failure(0, 2, None), 2 * base);
assert_eq!(next_check_after_failure(0, 4, None), 8 * base);
let max = FAILURE_BACKOFF_MAX.as_millis() as i64;
assert_eq!(next_check_after_failure(0, 40, None), max);
assert_eq!(
next_check_after_failure(0, 1, Some(Duration::from_secs(3_600))),
3_600_000
);
assert_eq!(
next_check_after_failure(0, 1, Some(Duration::from_secs(10 * 86_400))),
max
);
assert_eq!(
next_check_after_success(0),
REFRESH_INTERVAL.as_millis() as i64
);
}
/// The prototype podcast build's schema (user_version 1), verbatim.
const PROTOTYPE_SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS podcasts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
feed_url TEXT NOT NULL UNIQUE,
title TEXT NOT NULL,
author TEXT,
description TEXT,
link TEXT,
image_url TEXT,
cover_url TEXT,
cover_path TEXT,
etag TEXT,
last_modified TEXT,
added_at INTEGER NOT NULL,
last_checked_at INTEGER,
last_error TEXT
);
CREATE TABLE IF NOT EXISTS podcast_episodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE,
guid TEXT NOT NULL,
title TEXT NOT NULL,
description TEXT,
published_at INTEGER,
duration_seconds REAL,
audio_url TEXT NOT NULL,
mime_type TEXT,
byte_len INTEGER,
link TEXT,
image_url TEXT,
first_seen_at INTEGER NOT NULL,
UNIQUE (podcast_id, guid)
);
CREATE INDEX IF NOT EXISTS podcast_episodes_by_date
ON podcast_episodes (podcast_id, published_at DESC, id DESC);
";
#[test]
fn prototype_database_is_migrated_in_place() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join(furumi_library::podcasts::PODCAST_DB_FILE);
{
let conn = rusqlite::Connection::open(&path).unwrap();
conn.execute_batch(PROTOTYPE_SCHEMA).unwrap();
conn.execute(
"INSERT INTO podcasts (feed_url, title, etag, added_at, last_checked_at)
VALUES ('https://example.test/feed.xml', 'Old Show', '\"e1\"', 1700000000, 1700000100)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO podcast_episodes
(podcast_id, guid, title, published_at, duration_seconds, audio_url, first_seen_at)
VALUES (1, 'ep-1', 'Old Pilot', 1700000000, 1800.0,
'https://example.test/1.mp3', 1700000000)",
[],
)
.unwrap();
conn.pragma_update(None, "user_version", 1).unwrap();
}
let services = super::open_services(
&path,
dir.path().join("covers"),
Arc::new(MemoryLog::default()),
)
.unwrap();
let store = &services.store;
assert_eq!(
store.schema_version().unwrap(),
furumi_library::podcasts::PODCAST_SCHEMA_VERSION
);
let rows = store.list_podcasts(PodcastListFilter::Subscribed).unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].meta.title, "Old Show");
assert_eq!(rows[0].episode_count, 1);
let expected =
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed.xml").unwrap());
assert_eq!(rows[0].podcast_id, expected);
let fetch = store.fetch_state(&expected).unwrap();
assert_eq!(fetch.etag.as_deref(), Some("\"e1\""));
drop(services);
// The original rows are kept, renamed.
let conn = rusqlite::Connection::open(&path).unwrap();
let legacy: i64 = conn
.query_row("SELECT COUNT(*) FROM legacy_podcast_episodes", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(legacy, 1);
}
#[test]
fn foreign_or_newer_databases_are_refused_unchanged() {
let dir = tempfile::tempdir().unwrap();
let log: Arc<dyn OpLog> = Arc::new(MemoryLog::default());
let foreign = dir.path().join("foreign.sqlite3");
{
let conn = rusqlite::Connection::open(&foreign).unwrap();
conn.execute_batch("CREATE TABLE tracks (id INTEGER PRIMARY KEY, title TEXT);")
.unwrap();
}
let newer = dir.path().join("newer.sqlite3");
{
let conn = rusqlite::Connection::open(&newer).unwrap();
conn.pragma_update(None, "user_version", 99).unwrap();
}
let garbage = dir.path().join("garbage.sqlite3");
std::fs::write(
&garbage,
b"this is not a database at all, just some text bytes",
)
.unwrap();
for (path, expected) in [
(&foreign, "not a Furumi podcast database"),
(&newer, "newer than this Furumi supports"),
(&garbage, ""),
] {
let before = std::fs::read(path).unwrap();
let Err(err) = super::open_services(path, dir.path().join("covers"), Arc::clone(&log))
else {
panic!("{} opened", path.display());
};
let message = super::describe_open_error(&err);
assert!(message.contains(expected), "{message}");
assert_eq!(std::fs::read(path).unwrap(), before, "{}", path.display());
}
}