Fetch podcast feeds and stamp store actions from the sync log
Add src/podcasts: an HTTP client with conditional requests, hand-followed redirects (permanent ones become feed aliases via PodcastStore::apply_feed), size and time limits and a furumi user agent; subscribe and refresh flows that parse feeds with the shared music-dht parser and keep the store's local fetch state with exponential back-off and Retry-After. DeviceSync::run_podcast_action hands PodcastStore actions stamps reserved in the device-sync log and appends the operations they return with exactly those stamps. Podcast operations stay out of outgoing sync batches until this client speaks sync protocol v4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc04587252
commit
07dcd0587c
8 files changed
+1892
No files matched your search
+129
@@ -13,6 +13,9 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use furumi_library::podcasts::{
|
||||
OpStamp as PodcastOpStamp, PodcastLocalOp, StampSource as PodcastStampSource,
|
||||
};
|
||||
use music_dht::device_sync::{ListenEvent, ListenTrackMetadata};
|
||||
use music_dht::{ByteStream, MusicDhtService, NetworkId, PeerTicket, SecretKey, StreamAcceptor};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
@@ -1325,6 +1328,30 @@ impl DeviceSync {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs a `PodcastStore` local action with stamps reserved in this
|
||||
/// device's sync log and appends the operations it returns with exactly
|
||||
/// those stamps (`op_id`, `seq`, `hlc_ms`).
|
||||
///
|
||||
/// The log connection stays locked from the first stamp until the
|
||||
/// operations are appended, so no sync batch can read past a reserved
|
||||
/// `seq` before its operation exists. Lock order is podcast store, then
|
||||
/// this log: never call the podcast store while holding `self.conn`.
|
||||
pub fn run_podcast_action<T>(
|
||||
&self,
|
||||
action: impl FnOnce(&mut dyn PodcastStampSource) -> Result<(T, Vec<PodcastLocalOp>)>,
|
||||
) -> Result<T> {
|
||||
let identity = self.ensure_identity()?;
|
||||
let mut log = PodcastLogStamps {
|
||||
conn: &self.conn,
|
||||
guard: None,
|
||||
device_id: identity.device_id,
|
||||
reserved: BTreeMap::new(),
|
||||
};
|
||||
let (value, ops) = action(&mut log)?;
|
||||
log.append(ops)?;
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub async fn sync_once(
|
||||
&self,
|
||||
service: Arc<MusicDhtService>,
|
||||
@@ -2303,6 +2330,9 @@ impl DeviceSync {
|
||||
}
|
||||
|
||||
fn ops_for_peer(&self, peer_device_id: &str) -> Result<Vec<SyncOpWire>> {
|
||||
// Podcast operations (`podcast_*`, protocol v4) stay in the log but
|
||||
// are not sent yet: this client still announces protocol v3 and its
|
||||
// wire types cannot carry them. Gated sending comes with sync v4.
|
||||
let conn = lock(&self.conn);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT o.op_id, o.origin_device_id, o.seq, o.hlc_ms, o.payload_json
|
||||
@@ -2310,6 +2340,7 @@ impl DeviceSync {
|
||||
LEFT JOIN sync_peer_acks a
|
||||
ON a.peer_device_id = ?1 AND a.origin_device_id = o.origin_device_id
|
||||
WHERE o.seq > COALESCE(a.max_seq, 0)
|
||||
AND substr(o.kind, 1, 8) != 'podcast_'
|
||||
AND (
|
||||
o.kind != 'playback_command'
|
||||
OR (
|
||||
@@ -2800,6 +2831,104 @@ impl DeviceSync {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stamp source over the sync log for [`DeviceSync::run_podcast_action`].
|
||||
struct PodcastLogStamps<'a> {
|
||||
conn: &'a std::sync::Mutex<Connection>,
|
||||
/// Taken at the first stamp and held until the operations are appended.
|
||||
guard: Option<std::sync::MutexGuard<'a, Connection>>,
|
||||
device_id: String,
|
||||
/// `op_id` -> reserved `seq`.
|
||||
reserved: BTreeMap<String, i64>,
|
||||
}
|
||||
|
||||
impl PodcastStampSource for PodcastLogStamps<'_> {
|
||||
fn device_id(&self) -> &str {
|
||||
&self.device_id
|
||||
}
|
||||
|
||||
fn next_stamp(&mut self) -> Result<PodcastOpStamp> {
|
||||
let conn = self.guard.get_or_insert_with(|| lock(self.conn));
|
||||
let seq = get_meta(conn, "local_seq")?
|
||||
.and_then(|v| v.parse::<i64>().ok())
|
||||
.unwrap_or(0)
|
||||
+ 1;
|
||||
let last_hlc = get_meta(conn, "last_hlc_ms")?
|
||||
.and_then(|v| v.parse::<i64>().ok())
|
||||
.unwrap_or(0);
|
||||
let hlc_ms = now_ms().max(last_hlc + 1);
|
||||
set_meta(conn, "local_seq", &seq.to_string())?;
|
||||
set_meta(conn, "last_hlc_ms", &hlc_ms.to_string())?;
|
||||
let op_id = format!("{}:{seq}", self.device_id);
|
||||
self.reserved.insert(op_id.clone(), seq);
|
||||
Ok(PodcastOpStamp::new(op_id, self.device_id.clone(), hlc_ms))
|
||||
}
|
||||
}
|
||||
|
||||
impl PodcastLogStamps<'_> {
|
||||
fn append(mut self, ops: Vec<PodcastLocalOp>) -> Result<()> {
|
||||
if ops.len() != self.reserved.len() {
|
||||
tracing::warn!(
|
||||
reserved = self.reserved.len(),
|
||||
returned = ops.len(),
|
||||
"podcast action reserved stamps it did not return operations for"
|
||||
);
|
||||
}
|
||||
if ops.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let conn = self.guard.get_or_insert_with(|| lock(self.conn));
|
||||
for op in ops {
|
||||
let Some(&seq) = self.reserved.get(&op.stamp.op_id) else {
|
||||
anyhow::bail!(
|
||||
"podcast operation {} was not stamped by this sync log",
|
||||
op.stamp.op_id
|
||||
);
|
||||
};
|
||||
let op = op.into_wire(seq);
|
||||
let payload = serde_json::to_value(&op.payload)?;
|
||||
let kind = payload
|
||||
.get("kind")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("podcast_unknown")
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO sync_ops
|
||||
(op_id, origin_device_id, seq, kind, payload_json, hlc_ms,
|
||||
received_at_ms, tombstone)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
params![
|
||||
op.op_id,
|
||||
op.origin_device_id,
|
||||
op.seq,
|
||||
kind,
|
||||
payload.to_string(),
|
||||
op.hlc_ms,
|
||||
now_ms(),
|
||||
i64::from(op.payload.is_tombstone()),
|
||||
],
|
||||
)?;
|
||||
conn.execute(
|
||||
"INSERT INTO sync_vectors (device_id, max_seq)
|
||||
VALUES (?1, ?2)
|
||||
ON CONFLICT(device_id) DO UPDATE SET
|
||||
max_seq = MAX(max_seq, excluded.max_seq)",
|
||||
params![op.origin_device_id, op.seq],
|
||||
)?;
|
||||
tracing::debug!(op_id = %op.op_id, kind, "recorded podcast sync op");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::podcasts::OpLog for DeviceSync {
|
||||
fn record(
|
||||
&self,
|
||||
action: &mut dyn FnMut(&mut dyn PodcastStampSource) -> Result<Vec<PodcastLocalOp>>,
|
||||
) -> Result<()> {
|
||||
self.run_podcast_action(|stamps| Ok(((), action(stamps)?)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct StoredDevice {
|
||||
device_id: String,
|
||||
|
||||
@@ -937,3 +937,136 @@ fn stale_synced_playlist_item_metadata_repairs_pending_fed_track() {
|
||||
assert!(detail.tracks[0].is_fed_pending());
|
||||
assert_eq!(detail.tracks[0].title, fed.title);
|
||||
}
|
||||
|
||||
fn podcast_meta() -> music_dht::podcast::PodcastMeta {
|
||||
music_dht::podcast::PodcastMeta {
|
||||
feed_url: music_dht::podcast::FeedUrl::parse("https://example.test/feed.xml").unwrap(),
|
||||
title: "Show".into(),
|
||||
author: None,
|
||||
description: None,
|
||||
link: None,
|
||||
image_url: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rows of the local sync log: (op_id, seq, kind, payload_json, hlc_ms).
|
||||
fn logged_ops(sync: &DeviceSync) -> Vec<(String, i64, String, String, i64)> {
|
||||
let conn = lock(&sync.conn);
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT op_id, seq, kind, payload_json, hlc_ms FROM sync_ops ORDER BY seq")
|
||||
.unwrap();
|
||||
stmt.query_map([], |row| {
|
||||
Ok((
|
||||
row.get(0)?,
|
||||
row.get(1)?,
|
||||
row.get(2)?,
|
||||
row.get(3)?,
|
||||
row.get(4)?,
|
||||
))
|
||||
})
|
||||
.unwrap()
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_actions_are_appended_to_the_sync_log_with_their_stamps() {
|
||||
let sync = test_sync();
|
||||
let device_id = sync.ensure_identity().unwrap().device_id;
|
||||
let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap();
|
||||
sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true)
|
||||
.unwrap();
|
||||
let outcome = sync
|
||||
.run_podcast_action(|stamps| {
|
||||
let outcome = store.subscribe(stamps, &podcast_meta())?;
|
||||
let ops = outcome.ops.clone();
|
||||
Ok((outcome, ops))
|
||||
})
|
||||
.unwrap();
|
||||
let ops = logged_ops(&sync);
|
||||
assert_eq!(ops.len(), 2, "{ops:?}");
|
||||
let (op_id, seq, kind, payload_json, hlc_ms) = &ops[1];
|
||||
assert_eq!(kind, "podcast_subscription_set");
|
||||
assert_eq!(*seq, 2);
|
||||
assert_eq!(*op_id, format!("{device_id}:2"));
|
||||
assert!(*hlc_ms > ops[0].4, "hlc keeps increasing");
|
||||
// The log carries the shared FRID payload, and the store's register
|
||||
// was written with exactly this stamp.
|
||||
let payload: music_dht::device_sync::SyncOpPayload =
|
||||
serde_json::from_str(payload_json).unwrap();
|
||||
assert!(payload.is_podcast());
|
||||
let register = store.subscription(&outcome.podcast_id).unwrap().unwrap();
|
||||
assert_eq!(register.op_id, *op_id);
|
||||
assert_eq!(register.hlc_ms, *hlc_ms);
|
||||
let vector = sync.vector().unwrap();
|
||||
assert_eq!(vector.get(&device_id), Some(&2));
|
||||
|
||||
// Unsubscribing emits two operations with consecutive stamps; a music
|
||||
// operation afterwards continues the same sequence.
|
||||
let unsubscribe = sync
|
||||
.run_podcast_action(|stamps| {
|
||||
let ops = store.unsubscribe(stamps, &outcome.podcast_id)?;
|
||||
Ok((ops.len(), ops))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(unsubscribe, 2);
|
||||
sync.record_content_like(&format!("b3:{}", "b".repeat(64)), true)
|
||||
.unwrap();
|
||||
let ops = logged_ops(&sync);
|
||||
let kinds: Vec<(i64, &str)> = ops
|
||||
.iter()
|
||||
.map(|(_, seq, kind, ..)| (*seq, kind.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
[
|
||||
(1, "track_like_set"),
|
||||
(2, "podcast_subscription_set"),
|
||||
(3, "podcast_subscription_set"),
|
||||
(4, "podcast_share_set"),
|
||||
(5, "track_like_set"),
|
||||
]
|
||||
);
|
||||
|
||||
// An action that emits nothing leaves the log alone.
|
||||
sync.run_podcast_action(|stamps| {
|
||||
let ops = store.unsubscribe(stamps, &outcome.podcast_id)?;
|
||||
Ok(((), ops))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(logged_ops(&sync).len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_ops_are_not_sent_to_peers_before_sync_v4() {
|
||||
let sync = test_sync();
|
||||
let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap();
|
||||
sync.run_podcast_action(|stamps| {
|
||||
let outcome = store.subscribe(stamps, &podcast_meta())?;
|
||||
let ops = outcome.ops.clone();
|
||||
Ok(((), ops))
|
||||
})
|
||||
.unwrap();
|
||||
sync.record_content_like(&format!("b3:{}", "a".repeat(64)), true)
|
||||
.unwrap();
|
||||
// This client still speaks protocol v3; its wire types cannot carry the
|
||||
// podcast payload, so the batch keeps only the music operation.
|
||||
let ops = sync.ops_for_peer("dev_peer").unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert!(matches!(ops[0].payload, SyncOpPayload::TrackLikeSet { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_podcast_actions_append_nothing() {
|
||||
let sync = test_sync();
|
||||
let store = furumi_library::podcasts::PodcastStore::open_in_memory().unwrap();
|
||||
let unknown = music_dht::podcast::PodcastId::from_feed_url(
|
||||
&music_dht::podcast::FeedUrl::parse("https://example.test/other.xml").unwrap(),
|
||||
);
|
||||
let result = sync.run_podcast_action(|stamps| {
|
||||
let ops = store.set_shared(stamps, &unknown, true)?;
|
||||
Ok(((), ops))
|
||||
});
|
||||
assert!(result.is_err());
|
||||
assert!(logged_ops(&sync).is_empty());
|
||||
}
|
||||
@@ -7,6 +7,7 @@ mod jam;
|
||||
mod library;
|
||||
mod media;
|
||||
mod player;
|
||||
mod podcasts;
|
||||
mod share;
|
||||
mod similarity;
|
||||
mod status;
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
//! HTTP for podcast feeds and cover images.
|
||||
//!
|
||||
//! Redirects are followed by hand so the caller learns where the feed
|
||||
//! really lives and whether every hop was permanent (301/308): only then
|
||||
//! does the store record the new URL as a feed alias. Conditional requests
|
||||
//! (`If-None-Match` / `If-Modified-Since`), response size caps, timeouts and
|
||||
//! the user agent are fixed here; parsing and storage happen elsewhere.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
use reqwest::header::{
|
||||
ACCEPT, ETAG, HeaderMap, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED, LOCATION, RETRY_AFTER,
|
||||
};
|
||||
use reqwest::{StatusCode, Url};
|
||||
|
||||
/// Hard cap on a feed document (large catalogs are a few MiB).
|
||||
pub const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// Hard cap on a cover image.
|
||||
pub const MAX_COVER_BYTES: usize = 10 * 1024 * 1024;
|
||||
/// Redirect hops followed before giving up.
|
||||
pub const MAX_REDIRECTS: usize = 10;
|
||||
/// TCP + TLS connection setup.
|
||||
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
/// Whole request, including the body, per hop.
|
||||
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
/// Sent with every podcast request.
|
||||
pub const USER_AGENT: &str = concat!("furumi/", env!("CARGO_PKG_VERSION"), " (podcast reader)");
|
||||
|
||||
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
|
||||
text/xml;q=0.9, */*;q=0.5";
|
||||
|
||||
/// HTTP validators of the last successful fetch.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct Validators<'a> {
|
||||
pub etag: Option<&'a str>,
|
||||
pub last_modified: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// What a feed request returned.
|
||||
#[derive(Debug)]
|
||||
pub enum FeedBody {
|
||||
/// `304 Not Modified` to a conditional request.
|
||||
NotModified,
|
||||
/// A new document with its validators.
|
||||
Modified {
|
||||
body: Vec<u8>,
|
||||
etag: Option<String>,
|
||||
last_modified: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// A feed response and the redirect chain that led to it.
|
||||
#[derive(Debug)]
|
||||
pub struct FeedResponse {
|
||||
pub body: FeedBody,
|
||||
/// URL that finally answered, canonicalized.
|
||||
pub final_url: FeedUrl,
|
||||
/// At least one redirect was followed and every one of them was
|
||||
/// permanent (301/308). Temporary redirects (tracking prefixes, CDNs)
|
||||
/// never move a feed.
|
||||
pub moved_permanently: bool,
|
||||
/// Redirect hops followed.
|
||||
pub redirects: usize,
|
||||
}
|
||||
|
||||
/// A non-success HTTP status, with the server's `Retry-After` (seconds form)
|
||||
/// so the scheduler can back off as asked.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HttpStatusError {
|
||||
pub status: u16,
|
||||
pub retry_after: Option<Duration>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for HttpStatusError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let reason = StatusCode::from_u16(self.status)
|
||||
.ok()
|
||||
.and_then(|status| status.canonical_reason())
|
||||
.unwrap_or("error");
|
||||
write!(f, "HTTP {} {reason}", self.status)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for HttpStatusError {}
|
||||
|
||||
/// Shared HTTP client for feeds and covers.
|
||||
pub struct Fetcher {
|
||||
client: reqwest::Client,
|
||||
covers_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Fetcher {
|
||||
pub fn new(covers_dir: PathBuf) -> Result<Fetcher> {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent(USER_AGENT)
|
||||
.build()
|
||||
.context("building the podcast HTTP client")?;
|
||||
Ok(Fetcher { client, covers_dir })
|
||||
}
|
||||
|
||||
/// GET of a feed. `validators` make it conditional; they are dropped
|
||||
/// after a permanent redirect because they belong to the old URL.
|
||||
pub async fn fetch_feed(
|
||||
&self,
|
||||
url: &FeedUrl,
|
||||
validators: Validators<'_>,
|
||||
) -> Result<FeedResponse> {
|
||||
let Followed {
|
||||
response,
|
||||
url: final_url,
|
||||
moved_permanently,
|
||||
redirects,
|
||||
conditional,
|
||||
} = self
|
||||
.get_following(url.as_str(), Some(FEED_ACCEPT), validators)
|
||||
.await?;
|
||||
let final_url = FeedUrl::parse(final_url.as_str()).map_err(|err| {
|
||||
anyhow!(
|
||||
"the feed was redirected to an unsupported URL {}: {err}",
|
||||
redact(&final_url)
|
||||
)
|
||||
})?;
|
||||
let status = response.status();
|
||||
if status == StatusCode::NOT_MODIFIED {
|
||||
if !conditional {
|
||||
bail!("the server answered 304 Not Modified to an unconditional request");
|
||||
}
|
||||
return Ok(FeedResponse {
|
||||
body: FeedBody::NotModified,
|
||||
final_url,
|
||||
moved_permanently,
|
||||
redirects,
|
||||
});
|
||||
}
|
||||
if !status.is_success() {
|
||||
return Err(status_error(status, response.headers()).into());
|
||||
}
|
||||
let etag = header_string(response.headers(), ETAG);
|
||||
let last_modified = header_string(response.headers(), LAST_MODIFIED);
|
||||
let body = read_capped(response, MAX_FEED_BYTES, "feed").await?;
|
||||
Ok(FeedResponse {
|
||||
body: FeedBody::Modified {
|
||||
body,
|
||||
etag,
|
||||
last_modified,
|
||||
},
|
||||
final_url,
|
||||
moved_permanently,
|
||||
redirects,
|
||||
})
|
||||
}
|
||||
|
||||
/// Downloads a cover image into the covers directory; returns its path.
|
||||
pub async fn fetch_cover(&self, podcast_id: &PodcastId, image_url: &str) -> Result<PathBuf> {
|
||||
let response = self
|
||||
.get_following(image_url, None, Validators::default())
|
||||
.await?
|
||||
.response;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(status_error(status, response.headers()).into());
|
||||
}
|
||||
let body = read_capped(response, MAX_COVER_BYTES, "cover image").await?;
|
||||
let format = image::guess_format(&body).context("the cover is not a supported image")?;
|
||||
let extension = format.extensions_str().first().copied().unwrap_or("img");
|
||||
let name = cover_file_name(podcast_id, image_url, extension);
|
||||
let dir = self.covers_dir.clone();
|
||||
tokio::task::spawn_blocking(move || write_atomically(&dir, &name, &body))
|
||||
.await
|
||||
.map_err(|err| anyhow!("cover writer task failed: {err}"))?
|
||||
}
|
||||
|
||||
/// GET following up to [`MAX_REDIRECTS`] http(s) redirects by hand.
|
||||
async fn get_following(
|
||||
&self,
|
||||
url: &str,
|
||||
accept: Option<&str>,
|
||||
validators: Validators<'_>,
|
||||
) -> Result<Followed> {
|
||||
let mut current = Url::parse(url).with_context(|| format!("invalid URL {url:?}"))?;
|
||||
check_scheme(¤t)?;
|
||||
let mut all_permanent = true;
|
||||
let mut send_validators = true;
|
||||
let mut hops = 0;
|
||||
loop {
|
||||
let mut request = self.client.get(current.clone());
|
||||
if let Some(accept) = accept {
|
||||
request = request.header(ACCEPT, accept);
|
||||
}
|
||||
if send_validators {
|
||||
if let Some(etag) = validators.etag {
|
||||
request = request.header(IF_NONE_MATCH, etag);
|
||||
}
|
||||
if let Some(last_modified) = validators.last_modified {
|
||||
request = request.header(IF_MODIFIED_SINCE, last_modified);
|
||||
}
|
||||
}
|
||||
let response = request
|
||||
.send()
|
||||
.await
|
||||
// reqwest errors print the full URL, query tokens included.
|
||||
.map_err(reqwest::Error::without_url)
|
||||
.with_context(|| format!("requesting {}", redact(¤t)))?;
|
||||
let status = response.status();
|
||||
if !status.is_redirection() || status == StatusCode::NOT_MODIFIED {
|
||||
let conditional = send_validators
|
||||
&& (validators.etag.is_some() || validators.last_modified.is_some());
|
||||
return Ok(Followed {
|
||||
response,
|
||||
url: current,
|
||||
moved_permanently: hops > 0 && all_permanent,
|
||||
redirects: hops,
|
||||
conditional,
|
||||
});
|
||||
}
|
||||
let location = response
|
||||
.headers()
|
||||
.get(LOCATION)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| anyhow!("HTTP {status} redirect without a Location header"))?;
|
||||
let next = current
|
||||
.join(location.trim())
|
||||
.with_context(|| format!("invalid redirect location {location:?}"))?;
|
||||
check_scheme(&next)?;
|
||||
hops += 1;
|
||||
if hops > MAX_REDIRECTS {
|
||||
bail!("too many redirects (more than {MAX_REDIRECTS})");
|
||||
}
|
||||
let permanent = matches!(
|
||||
status,
|
||||
StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT
|
||||
);
|
||||
all_permanent &= permanent;
|
||||
if permanent {
|
||||
send_validators = false;
|
||||
}
|
||||
current = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Followed {
|
||||
response: reqwest::Response,
|
||||
/// URL that answered.
|
||||
url: Url,
|
||||
/// At least one hop and all of them permanent.
|
||||
moved_permanently: bool,
|
||||
redirects: usize,
|
||||
/// The final request carried validators.
|
||||
conditional: bool,
|
||||
}
|
||||
|
||||
fn check_scheme(url: &Url) -> Result<()> {
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
bail!("unsupported URL scheme {:?}", url.scheme());
|
||||
}
|
||||
if url.host_str().is_none_or(str::is_empty) {
|
||||
bail!("URL has no host");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The URL without query and fragment, for messages and logs: feed URLs
|
||||
/// may carry private tokens in their query.
|
||||
pub fn redact(url: &Url) -> String {
|
||||
let mut shown = url.clone();
|
||||
if shown.query().is_some() {
|
||||
shown.set_query(Some("…"));
|
||||
}
|
||||
shown.set_fragment(None);
|
||||
shown.to_string()
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: reqwest::header::HeaderName) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
fn status_error(status: StatusCode, headers: &HeaderMap) -> HttpStatusError {
|
||||
let retry_after = header_string(headers, RETRY_AFTER)
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.map(Duration::from_secs);
|
||||
HttpStatusError {
|
||||
status: status.as_u16(),
|
||||
retry_after,
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_capped(mut response: reqwest::Response, cap: usize, what: &str) -> Result<Vec<u8>> {
|
||||
if let Some(len) = response.content_length()
|
||||
&& len > cap as u64
|
||||
{
|
||||
bail!("the {what} is too large ({len} bytes, limit {cap})");
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response
|
||||
.chunk()
|
||||
.await
|
||||
.map_err(reqwest::Error::without_url)
|
||||
.with_context(|| format!("reading the {what}"))?
|
||||
{
|
||||
if body.len() + chunk.len() > cap {
|
||||
bail!("the {what} is too large (over {cap} bytes)");
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Stable per podcast and image URL, so a changed image gets a new file and
|
||||
/// the previous one can be removed.
|
||||
fn cover_file_name(podcast_id: &PodcastId, image_url: &str, extension: &str) -> String {
|
||||
let podcast = blake3::hash(podcast_id.as_str().as_bytes()).to_hex();
|
||||
let image = blake3::hash(image_url.as_bytes()).to_hex();
|
||||
format!("{}-{}.{extension}", &podcast[..16], &image[..16])
|
||||
}
|
||||
|
||||
fn write_atomically(dir: &Path, name: &str, bytes: &[u8]) -> Result<PathBuf> {
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
|
||||
let target = dir.join(name);
|
||||
let temp = dir.join(format!(
|
||||
".{name}.{}-{}.tmp",
|
||||
std::process::id(),
|
||||
COUNTER.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let result = std::fs::write(&temp, bytes)
|
||||
.and_then(|()| std::fs::rename(&temp, &target))
|
||||
.with_context(|| format!("writing {}", target.display()));
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&temp);
|
||||
}
|
||||
result?;
|
||||
Ok(target)
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
//! Podcast runtime of the TUI: feed HTTP, refresh scheduling and the glue
|
||||
//! between the shared podcast store and the device-sync log.
|
||||
//!
|
||||
//! The data model, identity rules, merge rules and the SQLite store are not
|
||||
//! defined here: they come from FRID (`music_dht::podcast` and
|
||||
//! `furumi_library::podcasts::PodcastStore`, file `podcasts.sqlite3`). This
|
||||
//! module only does what the contract leaves to clients:
|
||||
//!
|
||||
//! * [`fetch`] downloads feeds (conditional GET, redirects reported so
|
||||
//! permanent moves become feed aliases, size and time limits) and covers;
|
||||
//! * [`refresh`] subscribes and refreshes feeds: it parses them with the
|
||||
//! shared `music_dht::podcast::feed` parser, ingests them with
|
||||
//! `PodcastStore::apply_feed` and keeps the local fetch state (validators,
|
||||
//! errors, back-off);
|
||||
//! * [`OpLog`] hands store actions the stamps of the device-sync log and
|
||||
//! appends the operations they return.
|
||||
//!
|
||||
//! Only RSS metadata is stored. Audio stays on the publisher's server.
|
||||
|
||||
pub mod fetch;
|
||||
pub mod refresh;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_server;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use furumi_library::podcasts::{
|
||||
PODCAST_DB_FILE, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource,
|
||||
};
|
||||
|
||||
pub use fetch::Fetcher;
|
||||
use refresh::Services;
|
||||
|
||||
/// Appends podcast operations to the device-sync log.
|
||||
///
|
||||
/// [`OpLog::record`] gives `action` a [`StampSource`] that reserves stamps
|
||||
/// in the log. The action runs a `PodcastStore` local action and returns the
|
||||
/// operations it emitted; they are appended with exactly those stamps.
|
||||
pub trait OpLog: Send + Sync {
|
||||
fn record(
|
||||
&self,
|
||||
action: &mut dyn FnMut(&mut dyn StampSource) -> Result<Vec<PodcastLocalOp>>,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
/// The podcast store file. It is the file the prototype podcast build
|
||||
/// used, so an existing prototype database is migrated in place.
|
||||
pub fn db_path() -> Result<PathBuf> {
|
||||
let dirs = crate::config::project_dirs()
|
||||
.ok_or_else(|| anyhow!("no home directory for podcast data"))?;
|
||||
Ok(dirs.data_dir().join(PODCAST_DB_FILE))
|
||||
}
|
||||
|
||||
/// Replaceable cache of downloaded podcast covers.
|
||||
pub fn covers_dir() -> Result<PathBuf> {
|
||||
let dirs = crate::config::project_dirs()
|
||||
.ok_or_else(|| anyhow!("no cache directory for podcast covers"))?;
|
||||
Ok(dirs.cache_dir().join("podcasts").join("covers"))
|
||||
}
|
||||
|
||||
/// Opens the podcast store at `db_path` and builds the feed client.
|
||||
///
|
||||
/// `PodcastStore::open` migrates a prototype database in place and refuses,
|
||||
/// without writing anything, a database from a newer Furumi or a file that
|
||||
/// is not a podcast database. Blocking: call it on the blocking pool.
|
||||
pub fn open_services(db_path: &Path, covers_dir: PathBuf, log: Arc<dyn OpLog>) -> Result<Services> {
|
||||
let store = PodcastStore::open(db_path)?;
|
||||
tracing::info!(path = %db_path.display(), "podcast store opened");
|
||||
match store.legacy_quarantine() {
|
||||
Ok(entries) if !entries.is_empty() => tracing::warn!(
|
||||
count = entries.len(),
|
||||
"some prototype podcast rows could not be imported; they are kept in the legacy_* tables"
|
||||
),
|
||||
Ok(_) => {}
|
||||
Err(err) => tracing::debug!("reading the podcast import quarantine failed: {err:#}"),
|
||||
}
|
||||
Ok(Services {
|
||||
store: Arc::new(store),
|
||||
fetcher: Arc::new(Fetcher::new(covers_dir)?),
|
||||
log,
|
||||
})
|
||||
}
|
||||
|
||||
/// A readable reason why the store did not open; the typed store errors
|
||||
/// already say that the file was left unchanged.
|
||||
pub fn describe_open_error(err: &anyhow::Error) -> String {
|
||||
match err.downcast_ref::<PodcastStoreError>() {
|
||||
Some(store_error) => store_error.to_string(),
|
||||
None => format!("{err:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs blocking store or filesystem work off the Tokio workers.
|
||||
pub(crate) async fn blocking<T, F>(work: F) -> Result<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> Result<T> + Send + 'static,
|
||||
{
|
||||
tokio::task::spawn_blocking(work)
|
||||
.await
|
||||
.map_err(|err| anyhow!("background task failed: {err}"))?
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
//! Subscribing and refreshing feeds.
|
||||
//!
|
||||
//! Every function records its HTTP outcome in the store's local fetch state
|
||||
//! (validators, last error, consecutive failures, next check) so the
|
||||
//! scheduler never needs its own bookkeeping, and an error never clears the
|
||||
//! stored catalog.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result, anyhow, bail};
|
||||
use furumi_library::podcasts::{FeedRefresh, PodcastStore};
|
||||
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
|
||||
use super::fetch::{FeedBody, Fetcher, HttpStatusError, Validators};
|
||||
use super::{OpLog, blocking};
|
||||
|
||||
/// A successfully checked feed is checked again after this long.
|
||||
pub const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
||||
/// First retry after a failed check; doubles per consecutive failure.
|
||||
pub const FAILURE_BACKOFF_BASE: Duration = Duration::from_secs(5 * 60);
|
||||
/// Longest wait between checks of a failing feed (also caps `Retry-After`).
|
||||
pub const FAILURE_BACKOFF_MAX: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
/// Store, HTTP client and sync log shared by podcast background tasks.
|
||||
#[derive(Clone)]
|
||||
pub struct Services {
|
||||
pub store: Arc<PodcastStore>,
|
||||
pub fetcher: Arc<Fetcher>,
|
||||
pub log: Arc<dyn OpLog>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Services {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Services")
|
||||
.field("store", &self.store)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of [`subscribe`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SubscribeReport {
|
||||
pub podcast_id: PodcastId,
|
||||
pub title: String,
|
||||
/// The podcast was already subscribed; nothing was fetched or written.
|
||||
pub already_subscribed: bool,
|
||||
/// Episodes the first refresh added to the catalog.
|
||||
pub new_episodes: usize,
|
||||
}
|
||||
|
||||
/// Result of [`refresh`].
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct RefreshReport {
|
||||
/// The server answered 304 Not Modified.
|
||||
pub not_modified: bool,
|
||||
pub new_episodes: usize,
|
||||
pub updated_episodes: usize,
|
||||
/// Feed items without identity or with invalid metadata.
|
||||
pub skipped_items: usize,
|
||||
/// New fetch URL after a permanent redirect or `itunes:new-feed-url`.
|
||||
pub moved_to: Option<FeedUrl>,
|
||||
}
|
||||
|
||||
/// When to check a feed again after a success.
|
||||
pub fn next_check_after_success(now_ms: i64) -> i64 {
|
||||
now_ms.saturating_add(duration_ms(REFRESH_INTERVAL))
|
||||
}
|
||||
|
||||
/// When to check a feed again after its `failures`-th consecutive failure:
|
||||
/// exponential back-off, at least the server's `Retry-After`, at most
|
||||
/// [`FAILURE_BACKOFF_MAX`].
|
||||
pub fn next_check_after_failure(now_ms: i64, failures: u32, retry_after: Option<Duration>) -> i64 {
|
||||
let exponent = failures.saturating_sub(1).min(16);
|
||||
let backoff = FAILURE_BACKOFF_BASE
|
||||
.saturating_mul(1 << exponent)
|
||||
.max(retry_after.unwrap_or_default())
|
||||
.min(FAILURE_BACKOFF_MAX);
|
||||
now_ms.saturating_add(duration_ms(backoff))
|
||||
}
|
||||
|
||||
fn duration_ms(duration: Duration) -> i64 {
|
||||
i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Subscribes to the feed the user typed (`FeedUrl::from_user_input`).
|
||||
///
|
||||
/// Nothing is stored unless the URL serves a parsable feed. A URL that
|
||||
/// resolves (directly or through a known alias or redirect) to a podcast
|
||||
/// this device already knows reuses its id, so resubscribing continues the
|
||||
/// saved progress and history. The subscription is private.
|
||||
pub async fn subscribe(services: &Services, input: &str, now_ms: i64) -> Result<SubscribeReport> {
|
||||
let url = FeedUrl::from_user_input(input).map_err(|err| anyhow!("{err}"))?;
|
||||
let store = Arc::clone(&services.store);
|
||||
let lookup = url.clone();
|
||||
let existing = blocking(move || {
|
||||
let id = store.resolve_podcast_id(&lookup)?;
|
||||
let subscribed = store
|
||||
.subscription(&id)?
|
||||
.is_some_and(|record| record.subscribed);
|
||||
let title = store.podcast(&id)?.map(|summary| summary.meta.title);
|
||||
Ok(subscribed.then(|| (id, title)))
|
||||
})
|
||||
.await?;
|
||||
if let Some((podcast_id, title)) = existing {
|
||||
return Ok(SubscribeReport {
|
||||
podcast_id,
|
||||
title: title.unwrap_or_default(),
|
||||
already_subscribed: true,
|
||||
new_episodes: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let response = services
|
||||
.fetcher
|
||||
.fetch_feed(&url, Validators::default())
|
||||
.await?;
|
||||
let FeedBody::Modified {
|
||||
body,
|
||||
etag,
|
||||
last_modified,
|
||||
} = response.body
|
||||
else {
|
||||
bail!("the server answered 304 Not Modified to an unconditional request");
|
||||
};
|
||||
let final_url = response.final_url;
|
||||
let feed = parse(body, &final_url).await?;
|
||||
|
||||
let store = Arc::clone(&services.store);
|
||||
let log = Arc::clone(&services.log);
|
||||
let moved_permanently = response.moved_permanently;
|
||||
let image_url = feed.image_url.clone();
|
||||
let report = blocking(move || {
|
||||
// Identity comes from the URL of the first subscription. Reuse a
|
||||
// podcast this device already knows under the typed or the final
|
||||
// URL; otherwise the typed URL defines the id and a permanent
|
||||
// redirect becomes an alias below.
|
||||
let identity_url = if knows(&store, &url)? || !knows(&store, &final_url)? {
|
||||
url.clone()
|
||||
} else {
|
||||
final_url.clone()
|
||||
};
|
||||
let meta = feed.to_podcast_meta(identity_url);
|
||||
let mut subscribed = None;
|
||||
log.record(&mut |stamps| {
|
||||
let outcome = store.subscribe(stamps, &meta)?;
|
||||
let ops = outcome.ops.clone();
|
||||
subscribed = Some(outcome);
|
||||
Ok(ops)
|
||||
})?;
|
||||
let outcome = subscribed.ok_or_else(|| anyhow!("subscription was not recorded"))?;
|
||||
let podcast_id = outcome.podcast_id;
|
||||
let mut refreshed = None;
|
||||
log.record(&mut |stamps| {
|
||||
let outcome = store.apply_feed(
|
||||
stamps,
|
||||
&FeedRefresh {
|
||||
podcast_id: &podcast_id,
|
||||
feed: &feed,
|
||||
fetched_from: &final_url,
|
||||
moved_permanently,
|
||||
},
|
||||
)?;
|
||||
let ops = outcome.ops.clone();
|
||||
refreshed = Some(outcome);
|
||||
Ok(ops)
|
||||
})?;
|
||||
let refreshed = refreshed.ok_or_else(|| anyhow!("feed refresh was not recorded"))?;
|
||||
store.record_fetch_success(
|
||||
&podcast_id,
|
||||
now_ms,
|
||||
etag.as_deref(),
|
||||
last_modified.as_deref(),
|
||||
Some(next_check_after_success(now_ms)),
|
||||
)?;
|
||||
Ok(SubscribeReport {
|
||||
podcast_id,
|
||||
title: meta.title,
|
||||
already_subscribed: outcome.already_subscribed,
|
||||
new_episodes: refreshed.new_episodes,
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
if let Some(image_url) = image_url {
|
||||
update_cover(services, &report.podcast_id, &image_url).await;
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
fn knows(store: &PodcastStore, url: &FeedUrl) -> Result<bool> {
|
||||
let id = store.resolve_podcast_id(url)?;
|
||||
Ok(store.podcast(&id)?.is_some() || store.subscription(&id)?.is_some())
|
||||
}
|
||||
|
||||
/// Checks one feed (conditional GET from `url`, the store's fetch URL) and
|
||||
/// ingests a changed document. Failures are recorded in the fetch state
|
||||
/// with back-off and returned; the stored catalog is kept.
|
||||
pub async fn refresh(
|
||||
services: &Services,
|
||||
podcast_id: &PodcastId,
|
||||
url: &FeedUrl,
|
||||
now_ms: i64,
|
||||
) -> Result<RefreshReport> {
|
||||
let store = Arc::clone(&services.store);
|
||||
let id = podcast_id.clone();
|
||||
let fetch_state = blocking(move || store.fetch_state(&id)).await?;
|
||||
let result = fetch_and_apply(services, podcast_id, url, &fetch_state, now_ms).await;
|
||||
if let Err(err) = &result {
|
||||
let message = format!("{err:#}");
|
||||
let retry_after = err
|
||||
.downcast_ref::<HttpStatusError>()
|
||||
.and_then(|status| status.retry_after);
|
||||
let next = next_check_after_failure(
|
||||
now_ms,
|
||||
fetch_state.consecutive_failures.saturating_add(1),
|
||||
retry_after,
|
||||
);
|
||||
let store = Arc::clone(&services.store);
|
||||
let id = podcast_id.clone();
|
||||
if let Err(record_err) = blocking(move || {
|
||||
store
|
||||
.record_fetch_failure(&id, now_ms, &message, Some(next))
|
||||
.map(|_| ())
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!("recording a podcast refresh error failed: {record_err:#}");
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn fetch_and_apply(
|
||||
services: &Services,
|
||||
podcast_id: &PodcastId,
|
||||
url: &FeedUrl,
|
||||
fetch_state: &furumi_library::podcasts::FetchState,
|
||||
now_ms: i64,
|
||||
) -> Result<RefreshReport> {
|
||||
let response = services
|
||||
.fetcher
|
||||
.fetch_feed(
|
||||
url,
|
||||
Validators {
|
||||
etag: fetch_state.etag.as_deref(),
|
||||
last_modified: fetch_state.last_modified.as_deref(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
tracing::debug!(
|
||||
podcast = %podcast_id,
|
||||
redirects = response.redirects,
|
||||
moved_permanently = response.moved_permanently,
|
||||
not_modified = matches!(response.body, FeedBody::NotModified),
|
||||
"podcast feed checked"
|
||||
);
|
||||
let next = Some(next_check_after_success(now_ms));
|
||||
let (body, etag, last_modified) = match response.body {
|
||||
FeedBody::NotModified => {
|
||||
let store = Arc::clone(&services.store);
|
||||
let id = podcast_id.clone();
|
||||
blocking(move || store.record_fetch_not_modified(&id, now_ms, next)).await?;
|
||||
return Ok(RefreshReport {
|
||||
not_modified: true,
|
||||
..RefreshReport::default()
|
||||
});
|
||||
}
|
||||
FeedBody::Modified {
|
||||
body,
|
||||
etag,
|
||||
last_modified,
|
||||
} => (body, etag, last_modified),
|
||||
};
|
||||
let final_url = response.final_url;
|
||||
let feed = parse(body, &final_url).await?;
|
||||
let image_url = feed.image_url.clone();
|
||||
let store = Arc::clone(&services.store);
|
||||
let log = Arc::clone(&services.log);
|
||||
let id = podcast_id.clone();
|
||||
let previous_url = url.clone();
|
||||
let moved_permanently = response.moved_permanently;
|
||||
let report = blocking(move || {
|
||||
let mut refreshed = None;
|
||||
log.record(&mut |stamps| {
|
||||
let outcome = store.apply_feed(
|
||||
stamps,
|
||||
&FeedRefresh {
|
||||
podcast_id: &id,
|
||||
feed: &feed,
|
||||
fetched_from: &final_url,
|
||||
moved_permanently,
|
||||
},
|
||||
)?;
|
||||
let ops = outcome.ops.clone();
|
||||
refreshed = Some(outcome);
|
||||
Ok(ops)
|
||||
})?;
|
||||
let outcome = refreshed.ok_or_else(|| anyhow!("feed refresh was not recorded"))?;
|
||||
store.record_fetch_success(&id, now_ms, etag.as_deref(), last_modified.as_deref(), next)?;
|
||||
Ok(RefreshReport {
|
||||
not_modified: false,
|
||||
new_episodes: outcome.new_episodes,
|
||||
updated_episodes: outcome.updated_episodes,
|
||||
skipped_items: outcome.skipped_items,
|
||||
moved_to: outcome
|
||||
.feed_url
|
||||
.filter(|feed_url| *feed_url != previous_url),
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
if let Some(image_url) = image_url {
|
||||
update_cover(services, podcast_id, &image_url).await;
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Parses a feed document on the blocking pool; relative URLs resolve
|
||||
/// against where the document was served from.
|
||||
async fn parse(body: Vec<u8>, base: &FeedUrl) -> Result<ParsedFeed> {
|
||||
let base = base.as_str().to_string();
|
||||
blocking(move || {
|
||||
parse_feed_with_base(&body, Some(&base)).map_err(|err| anyhow!("unusable feed: {err}"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Downloads the cover when the feed image changed or the cached file is
|
||||
/// gone, records it and deletes the replaced file. Best effort: failures are
|
||||
/// only logged, covers are a replaceable cache.
|
||||
async fn update_cover(services: &Services, podcast_id: &PodcastId, image_url: &str) {
|
||||
let store = Arc::clone(&services.store);
|
||||
let id = podcast_id.clone();
|
||||
let current = blocking(move || {
|
||||
let state = store.fetch_state(&id)?;
|
||||
Ok((state.cover_url, state.cover_path))
|
||||
})
|
||||
.await;
|
||||
let (old_url, old_path) = match current {
|
||||
Ok(current) => current,
|
||||
Err(err) => {
|
||||
tracing::warn!("reading podcast cover state failed: {err:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let cached = old_path
|
||||
.as_deref()
|
||||
.is_some_and(|path| Path::new(path).is_file());
|
||||
if old_url.as_deref() == Some(image_url) && cached {
|
||||
return;
|
||||
}
|
||||
let path = match services.fetcher.fetch_cover(podcast_id, image_url).await {
|
||||
Ok(path) => path,
|
||||
Err(err) => {
|
||||
tracing::debug!(podcast = %podcast_id, "podcast cover download failed: {err:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let store = Arc::clone(&services.store);
|
||||
let id = podcast_id.clone();
|
||||
let image_url = image_url.to_string();
|
||||
let result = blocking(move || {
|
||||
let new_path = std::path::absolute(&path)
|
||||
.unwrap_or(path)
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
store
|
||||
.set_cover_cache(&id, Some(&image_url), Some(&new_path))
|
||||
.context("recording the cover")?;
|
||||
if let Some(old_path) = old_path.filter(|old| *old != new_path) {
|
||||
let _ = std::fs::remove_file(old_path);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
tracing::warn!("recording a podcast cover failed: {err:#}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
//! Minimal HTTP/1.1 server on 127.0.0.1 for podcast tests (no internet).
|
||||
//! One request per connection (`Connection: close`).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct Route {
|
||||
pub status: u16,
|
||||
pub body: Arc<Vec<u8>>,
|
||||
pub content_type: String,
|
||||
pub etag: Option<String>,
|
||||
pub last_modified: Option<String>,
|
||||
/// `Location` of a redirect status.
|
||||
pub location: Option<String>,
|
||||
pub retry_after: Option<String>,
|
||||
}
|
||||
|
||||
impl Route {
|
||||
pub fn new(body: impl Into<Vec<u8>>, content_type: &str) -> Route {
|
||||
Route {
|
||||
status: 200,
|
||||
body: Arc::new(body.into()),
|
||||
content_type: content_type.to_string(),
|
||||
etag: None,
|
||||
last_modified: None,
|
||||
location: None,
|
||||
retry_after: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redirect(status: u16, location: &str) -> Route {
|
||||
Route {
|
||||
status,
|
||||
location: Some(location.to_string()),
|
||||
..Route::new(Vec::new(), "text/plain")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn status(mut self, status: u16) -> Route {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn etag(mut self, etag: &str) -> Route {
|
||||
self.etag = Some(etag.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn last_modified(mut self, value: &str) -> Route {
|
||||
self.last_modified = Some(value.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn retry_after(mut self, value: &str) -> Route {
|
||||
self.retry_after = Some(value.to_string());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Headers of one received request (names lowercased).
|
||||
pub(super) type RequestHeaders = HashMap<String, String>;
|
||||
|
||||
#[derive(Default)]
|
||||
struct ServerState {
|
||||
routes: HashMap<String, Route>,
|
||||
requests: HashMap<String, Vec<RequestHeaders>>,
|
||||
}
|
||||
|
||||
pub(super) struct TestServer {
|
||||
base: String,
|
||||
state: Arc<Mutex<ServerState>>,
|
||||
}
|
||||
|
||||
impl TestServer {
|
||||
pub fn start() -> TestServer {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server");
|
||||
let base = format!("http://{}", listener.local_addr().expect("local addr"));
|
||||
let state = Arc::new(Mutex::new(ServerState::default()));
|
||||
let accept_state = Arc::clone(&state);
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let Ok(stream) = stream else { continue };
|
||||
let state = Arc::clone(&accept_state);
|
||||
std::thread::spawn(move || handle(stream, &state));
|
||||
}
|
||||
});
|
||||
TestServer { base, state }
|
||||
}
|
||||
|
||||
pub fn url(&self, path: &str) -> String {
|
||||
format!("{}{path}", self.base)
|
||||
}
|
||||
|
||||
pub fn set(&self, path: &str, route: Route) {
|
||||
self.lock().routes.insert(path.to_string(), route);
|
||||
}
|
||||
|
||||
/// Headers of every request received for `path`, in order.
|
||||
pub fn requests(&self, path: &str) -> Vec<RequestHeaders> {
|
||||
self.lock().requests.get(path).cloned().unwrap_or_default()
|
||||
}
|
||||
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, ServerState> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(stream: TcpStream, state: &Mutex<ServerState>) {
|
||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
|
||||
let mut reader = BufReader::new(match stream.try_clone() {
|
||||
Ok(stream) => stream,
|
||||
Err(_) => return,
|
||||
});
|
||||
let mut request_line = String::new();
|
||||
if reader.read_line(&mut request_line).is_err() {
|
||||
return;
|
||||
}
|
||||
let path = request_line
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.unwrap_or("/")
|
||||
.to_string();
|
||||
let mut headers = RequestHeaders::new();
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
match reader.read_line(&mut line) {
|
||||
Ok(0) | Err(_) => return,
|
||||
Ok(_) => {}
|
||||
}
|
||||
let line = line.trim_end();
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
if let Some((name, value)) = line.split_once(':') {
|
||||
headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string());
|
||||
}
|
||||
}
|
||||
let route = {
|
||||
let mut state = state.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
state
|
||||
.requests
|
||||
.entry(path.clone())
|
||||
.or_default()
|
||||
.push(headers.clone());
|
||||
state.routes.get(&path).cloned()
|
||||
};
|
||||
let mut out = stream;
|
||||
let Some(route) = route else {
|
||||
let _ = out
|
||||
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
|
||||
return;
|
||||
};
|
||||
let etag_matches = route
|
||||
.etag
|
||||
.as_ref()
|
||||
.is_some_and(|etag| headers.get("if-none-match") == Some(etag));
|
||||
let date_matches = route
|
||||
.last_modified
|
||||
.as_ref()
|
||||
.is_some_and(|date| headers.get("if-modified-since") == Some(date));
|
||||
if route.status == 200 && (etag_matches || date_matches) {
|
||||
let _ = write!(
|
||||
out,
|
||||
"HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let mut extra = String::new();
|
||||
for (name, value) in [
|
||||
("ETag", &route.etag),
|
||||
("Last-Modified", &route.last_modified),
|
||||
("Location", &route.location),
|
||||
("Retry-After", &route.retry_after),
|
||||
] {
|
||||
if let Some(value) = value {
|
||||
extra.push_str(&format!("{name}: {value}\r\n"));
|
||||
}
|
||||
}
|
||||
let reason = match route.status {
|
||||
200 => "OK",
|
||||
301 => "Moved Permanently",
|
||||
302 => "Found",
|
||||
307 => "Temporary Redirect",
|
||||
308 => "Permanent Redirect",
|
||||
404 => "Not Found",
|
||||
503 => "Service Unavailable",
|
||||
_ => "Status",
|
||||
};
|
||||
if write!(
|
||||
out,
|
||||
"HTTP/1.1 {} {reason}\r\nContent-Type: {}\r\nContent-Length: {}\r\n{extra}Connection: close\r\n\r\n",
|
||||
route.status,
|
||||
route.content_type,
|
||||
route.body.len()
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let _ = out.write_all(&route.body);
|
||||
let _ = out.flush();
|
||||
let _ = out.shutdown(std::net::Shutdown::Both);
|
||||
}
|
||||
@@ -0,0 +1,588 @@
|
||||
//! Store opening, subscribing and refreshing against a local HTTP server.
|
||||
//! No internet, no real user directories: every test uses a temp dir.
|
||||
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use furumi_library::podcasts::{
|
||||
EpisodeQuery, OpStamp, PodcastListFilter, PodcastLocalOp, PodcastStore, StampSource,
|
||||
};
|
||||
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
|
||||
use super::OpLog;
|
||||
use super::refresh::{
|
||||
FAILURE_BACKOFF_BASE, FAILURE_BACKOFF_MAX, REFRESH_INTERVAL, Services,
|
||||
next_check_after_failure, next_check_after_success, refresh, subscribe,
|
||||
};
|
||||
use super::test_server::{Route, TestServer};
|
||||
|
||||
const NOW: i64 = 1_700_000_000_000;
|
||||
|
||||
/// In-memory sync log: hands out stamps and keeps the appended operations.
|
||||
#[derive(Default)]
|
||||
struct MemoryLog {
|
||||
ops: Mutex<Vec<SyncOpWire>>,
|
||||
}
|
||||
|
||||
struct MemoryStamps {
|
||||
next_seq: i64,
|
||||
reserved: Vec<i64>,
|
||||
}
|
||||
|
||||
impl StampSource for MemoryStamps {
|
||||
fn device_id(&self) -> &str {
|
||||
"test-device"
|
||||
}
|
||||
|
||||
fn next_stamp(&mut self) -> Result<OpStamp> {
|
||||
let seq = self.next_seq + self.reserved.len() as i64;
|
||||
self.reserved.push(seq);
|
||||
Ok(OpStamp::new(
|
||||
format!("test-device:{seq}"),
|
||||
"test-device",
|
||||
NOW + seq,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl OpLog for MemoryLog {
|
||||
fn record(
|
||||
&self,
|
||||
action: &mut dyn FnMut(&mut dyn StampSource) -> Result<Vec<PodcastLocalOp>>,
|
||||
) -> Result<()> {
|
||||
let mut ops = self.ops.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let mut stamps = MemoryStamps {
|
||||
next_seq: ops.len() as i64 + 1,
|
||||
reserved: Vec::new(),
|
||||
};
|
||||
for (op, seq) in action(&mut stamps)?.into_iter().zip(stamps.reserved) {
|
||||
ops.push(op.into_wire(seq));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl MemoryLog {
|
||||
fn kinds(&self) -> Vec<String> {
|
||||
self.ops
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.iter()
|
||||
.map(|op| {
|
||||
serde_json::to_value(&op.payload).unwrap()["kind"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
_dir: tempfile::TempDir,
|
||||
services: Services,
|
||||
log: Arc<MemoryLog>,
|
||||
server: TestServer,
|
||||
}
|
||||
|
||||
fn fixture() -> Fixture {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let log = Arc::new(MemoryLog::default());
|
||||
let services = super::open_services(
|
||||
&dir.path().join("podcasts.sqlite3"),
|
||||
dir.path().join("covers"),
|
||||
log.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
Fixture {
|
||||
_dir: dir,
|
||||
services,
|
||||
log,
|
||||
server: TestServer::start(),
|
||||
}
|
||||
}
|
||||
|
||||
fn rss(items: &[(&str, &str)]) -> Vec<u8> {
|
||||
let items: String = items
|
||||
.iter()
|
||||
.map(|(guid, title)| {
|
||||
format!(
|
||||
"<item><title>{title}</title><guid>{guid}</guid>\
|
||||
<pubDate>Sat, 15 Jun 2024 10:00:00 GMT</pubDate>\
|
||||
<itunes:duration>45:00</itunes:duration>\
|
||||
<enclosure url=\"/audio/{guid}.mp3\" type=\"audio/mpeg\" length=\"100\"/></item>"
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
format!(
|
||||
"<?xml version=\"1.0\"?><rss version=\"2.0\" \
|
||||
xmlns:itunes=\"http://www.itunes.com/dtds/podcast-1.0.dtd\"><channel>\
|
||||
<title>Test Show</title><itunes:author>Host</itunes:author>{items}</channel></rss>"
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn episode_count(store: &PodcastStore, id: &PodcastId) -> usize {
|
||||
store
|
||||
.episodes(&EpisodeQuery::new(id.clone()))
|
||||
.unwrap()
|
||||
.len()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscribe_then_conditional_refresh_uses_validators() {
|
||||
let f = fixture();
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(
|
||||
rss(&[("a", "Pilot"), ("b", "Second")]),
|
||||
"application/rss+xml",
|
||||
)
|
||||
.etag("\"v1\"")
|
||||
.last_modified("Sat, 15 Jun 2024 10:00:00 GMT"),
|
||||
);
|
||||
let report = subscribe(&f.services, &f.server.url("/feed.xml"), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!report.already_subscribed);
|
||||
assert_eq!(report.new_episodes, 2);
|
||||
assert_eq!(report.title, "Test Show");
|
||||
assert_eq!(f.log.kinds(), ["podcast_subscription_set"]);
|
||||
let store = &f.services.store;
|
||||
let rows = store.list_podcasts(PodcastListFilter::Subscribed).unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert!(!rows[0].shared, "a new subscription is private");
|
||||
let state = store.fetch_state(&report.podcast_id).unwrap();
|
||||
assert_eq!(state.etag.as_deref(), Some("\"v1\""));
|
||||
assert_eq!(state.next_check_at_ms, Some(next_check_after_success(NOW)));
|
||||
let first = &f.server.requests("/feed.xml")[0];
|
||||
assert!(!first.contains_key("if-none-match"), "{first:?}");
|
||||
assert!(
|
||||
first
|
||||
.get("user-agent")
|
||||
.is_some_and(|agent| agent.starts_with("furumi/")),
|
||||
"{first:?}"
|
||||
);
|
||||
|
||||
// Unchanged: 304, nothing re-parsed, validators sent.
|
||||
let url = rows[0].meta.feed_url.clone();
|
||||
let checked = refresh(&f.services, &report.podcast_id, &url, NOW + 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(checked.not_modified);
|
||||
let second = &f.server.requests("/feed.xml")[1];
|
||||
assert_eq!(
|
||||
second.get("if-none-match").map(String::as_str),
|
||||
Some("\"v1\"")
|
||||
);
|
||||
assert_eq!(
|
||||
second.get("if-modified-since").map(String::as_str),
|
||||
Some("Sat, 15 Jun 2024 10:00:00 GMT")
|
||||
);
|
||||
|
||||
// Changed: a new episode arrives, the known ones are not duplicated.
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(
|
||||
rss(&[("a", "Pilot (remastered)"), ("b", "Second"), ("c", "Third")]),
|
||||
"application/rss+xml",
|
||||
)
|
||||
.etag("\"v2\""),
|
||||
);
|
||||
let changed = refresh(&f.services, &report.podcast_id, &url, NOW + 2)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!changed.not_modified);
|
||||
assert_eq!(changed.new_episodes, 1);
|
||||
assert_eq!(changed.updated_episodes, 1);
|
||||
assert_eq!(episode_count(store, &report.podcast_id), 3);
|
||||
// A refresh never writes personal state: no new operation.
|
||||
assert_eq!(f.log.kinds(), ["podcast_subscription_set"]);
|
||||
|
||||
// Subscribing again is recognised without another fetch.
|
||||
let requests = f.server.requests("/feed.xml").len();
|
||||
let again = subscribe(&f.services, &f.server.url("/feed.xml"), NOW + 3)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(again.already_subscribed);
|
||||
assert_eq!(again.podcast_id, report.podcast_id);
|
||||
assert_eq!(f.server.requests("/feed.xml").len(), requests);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn permanent_redirect_becomes_an_alias_and_the_new_fetch_url() {
|
||||
let f = fixture();
|
||||
f.server
|
||||
.set("/old.xml", Route::redirect(301, &f.server.url("/new.xml")));
|
||||
f.server.set(
|
||||
"/new.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
|
||||
);
|
||||
let old = FeedUrl::parse(&f.server.url("/old.xml")).unwrap();
|
||||
let new = FeedUrl::parse(&f.server.url("/new.xml")).unwrap();
|
||||
let report = subscribe(&f.services, old.as_str(), NOW).await.unwrap();
|
||||
// The id comes from the URL of the first subscription.
|
||||
assert_eq!(report.podcast_id, PodcastId::from_feed_url(&old));
|
||||
assert_eq!(
|
||||
f.log.kinds(),
|
||||
["podcast_subscription_set", "podcast_feed_aliases_added"]
|
||||
);
|
||||
let store = &f.services.store;
|
||||
assert!(
|
||||
store
|
||||
.feed_aliases(&report.podcast_id)
|
||||
.unwrap()
|
||||
.contains(&new)
|
||||
);
|
||||
let summary = store.podcast(&report.podcast_id).unwrap().unwrap();
|
||||
assert_eq!(summary.meta.feed_url, new, "fetched from the new URL now");
|
||||
assert_eq!(store.resolve_podcast_id(&new).unwrap(), report.podcast_id);
|
||||
|
||||
// The next refresh goes straight to the new URL; no new alias op.
|
||||
let checked = refresh(&f.services, &report.podcast_id, &new, NOW + 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(checked.moved_to.is_none());
|
||||
assert_eq!(f.server.requests("/old.xml").len(), 1);
|
||||
assert_eq!(f.log.kinds().len(), 2);
|
||||
|
||||
// Subscribing through the new URL finds the same podcast.
|
||||
let again = subscribe(&f.services, new.as_str(), NOW + 2).await.unwrap();
|
||||
assert!(again.already_subscribed);
|
||||
assert_eq!(again.podcast_id, report.podcast_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn temporary_redirect_keeps_the_feed_url() {
|
||||
let f = fixture();
|
||||
f.server
|
||||
.set("/tracked.xml", Route::redirect(302, "/real.xml"));
|
||||
f.server.set(
|
||||
"/real.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
|
||||
);
|
||||
let tracked = FeedUrl::parse(&f.server.url("/tracked.xml")).unwrap();
|
||||
let report = subscribe(&f.services, tracked.as_str(), NOW).await.unwrap();
|
||||
assert_eq!(f.log.kinds(), ["podcast_subscription_set"]);
|
||||
let summary = f
|
||||
.services
|
||||
.store
|
||||
.podcast(&report.podcast_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(summary.meta.feed_url, tracked);
|
||||
// Episode URLs resolve against where the document was served from.
|
||||
let episodes = f
|
||||
.services
|
||||
.store
|
||||
.episodes(&EpisodeQuery::new(report.podcast_id.clone()))
|
||||
.unwrap();
|
||||
assert_eq!(episodes[0].episode.audio_url, f.server.url("/audio/a.mp3"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_feed_and_server_errors_back_off_and_keep_the_catalog() {
|
||||
let f = fixture();
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(
|
||||
rss(&[("a", "Pilot"), ("b", "Second")]),
|
||||
"application/rss+xml",
|
||||
),
|
||||
);
|
||||
let report = subscribe(&f.services, &f.server.url("/feed.xml"), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
let id = report.podcast_id;
|
||||
let url = FeedUrl::parse(&f.server.url("/feed.xml")).unwrap();
|
||||
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new("<html><body>Moved to our app!</body></html>", "text/html"),
|
||||
);
|
||||
let err = refresh(&f.services, &id, &url, NOW + 1).await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("unusable feed"), "{err:#}");
|
||||
let store = &f.services.store;
|
||||
let state = store.fetch_state(&id).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 1);
|
||||
assert!(
|
||||
state
|
||||
.last_error
|
||||
.as_deref()
|
||||
.unwrap()
|
||||
.contains("unusable feed")
|
||||
);
|
||||
assert_eq!(
|
||||
state.next_check_at_ms,
|
||||
Some(next_check_after_failure(NOW + 1, 1, None))
|
||||
);
|
||||
assert_eq!(episode_count(store, &id), 2, "an error keeps the catalog");
|
||||
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new("busy", "text/plain")
|
||||
.status(503)
|
||||
.retry_after("7200"),
|
||||
);
|
||||
let err = refresh(&f.services, &id, &url, NOW + 2).await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("HTTP 503"), "{err:#}");
|
||||
let state = store.fetch_state(&id).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 2);
|
||||
assert_eq!(state.next_check_at_ms, Some(NOW + 2 + 7_200_000));
|
||||
assert_eq!(episode_count(store, &id), 2);
|
||||
// A failing feed is not due before its back-off expires.
|
||||
assert!(
|
||||
store
|
||||
.podcasts_due_for_refresh(NOW + 3, 10)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.podcasts_due_for_refresh(NOW + 2 + 7_200_000, 10)
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
// Recovery clears the error.
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(
|
||||
rss(&[("a", "Pilot"), ("b", "Second")]),
|
||||
"application/rss+xml",
|
||||
),
|
||||
);
|
||||
refresh(&f.services, &id, &url, NOW + 3).await.unwrap();
|
||||
let state = store.fetch_state(&id).unwrap();
|
||||
assert_eq!(state.consecutive_failures, 0);
|
||||
assert!(state.last_error.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscribing_to_a_non_feed_stores_nothing() {
|
||||
let f = fixture();
|
||||
f.server
|
||||
.set("/page", Route::new("<html>hello</html>", "text/html"));
|
||||
let err = subscribe(&f.services, &f.server.url("/page"), NOW)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("unusable feed"), "{err:#}");
|
||||
let err = subscribe(&f.services, &f.server.url("/missing"), NOW)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("HTTP 404"), "{err:#}");
|
||||
let err = subscribe(&f.services, "ftp://example.test/feed", NOW)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(!format!("{err:#}").is_empty());
|
||||
assert!(
|
||||
f.services
|
||||
.store
|
||||
.list_podcasts(PodcastListFilter::All)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(f.log.kinds().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsubscribe_then_resubscribe_keeps_the_podcast_id() {
|
||||
let f = fixture();
|
||||
f.server.set(
|
||||
"/feed.xml",
|
||||
Route::new(rss(&[("a", "Pilot")]), "application/rss+xml"),
|
||||
);
|
||||
let first = subscribe(&f.services, &f.server.url("/feed.xml"), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
let store = Arc::clone(&f.services.store);
|
||||
let id = first.podcast_id.clone();
|
||||
f.log
|
||||
.record(&mut |stamps| store.unsubscribe(stamps, &id))
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.list_podcasts(PodcastListFilter::Subscribed)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
let again = subscribe(&f.services, &f.server.url("/feed.xml"), NOW + 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!again.already_subscribed);
|
||||
assert_eq!(again.podcast_id, first.podcast_id);
|
||||
assert_eq!(
|
||||
f.log.kinds(),
|
||||
[
|
||||
"podcast_subscription_set",
|
||||
"podcast_subscription_set",
|
||||
"podcast_share_set",
|
||||
"podcast_subscription_set"
|
||||
]
|
||||
);
|
||||
let ops = f.log.ops.lock().unwrap();
|
||||
assert!(matches!(
|
||||
ops.last().map(|op| &op.payload),
|
||||
Some(SyncOpPayload::PodcastSubscriptionSet {
|
||||
subscribed: true,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn back_off_doubles_and_honours_retry_after() {
|
||||
let base = FAILURE_BACKOFF_BASE.as_millis() as i64;
|
||||
assert_eq!(next_check_after_failure(0, 1, None), base);
|
||||
assert_eq!(next_check_after_failure(0, 2, None), 2 * base);
|
||||
assert_eq!(next_check_after_failure(0, 4, None), 8 * base);
|
||||
let max = FAILURE_BACKOFF_MAX.as_millis() as i64;
|
||||
assert_eq!(next_check_after_failure(0, 40, None), max);
|
||||
assert_eq!(
|
||||
next_check_after_failure(0, 1, Some(Duration::from_secs(3_600))),
|
||||
3_600_000
|
||||
);
|
||||
assert_eq!(
|
||||
next_check_after_failure(0, 1, Some(Duration::from_secs(10 * 86_400))),
|
||||
max
|
||||
);
|
||||
assert_eq!(
|
||||
next_check_after_success(0),
|
||||
REFRESH_INTERVAL.as_millis() as i64
|
||||
);
|
||||
}
|
||||
|
||||
/// The prototype podcast build's schema (user_version 1), verbatim.
|
||||
const PROTOTYPE_SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcasts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_url TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
description TEXT,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
cover_url TEXT,
|
||||
cover_path TEXT,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
added_at INTEGER NOT NULL,
|
||||
last_checked_at INTEGER,
|
||||
last_error TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_episodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE,
|
||||
guid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
published_at INTEGER,
|
||||
duration_seconds REAL,
|
||||
audio_url TEXT NOT NULL,
|
||||
mime_type TEXT,
|
||||
byte_len INTEGER,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
UNIQUE (podcast_id, guid)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_episodes_by_date
|
||||
ON podcast_episodes (podcast_id, published_at DESC, id DESC);
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn prototype_database_is_migrated_in_place() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join(furumi_library::podcasts::PODCAST_DB_FILE);
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(PROTOTYPE_SCHEMA).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO podcasts (feed_url, title, etag, added_at, last_checked_at)
|
||||
VALUES ('https://example.test/feed.xml', 'Old Show', '\"e1\"', 1700000000, 1700000100)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_episodes
|
||||
(podcast_id, guid, title, published_at, duration_seconds, audio_url, first_seen_at)
|
||||
VALUES (1, 'ep-1', 'Old Pilot', 1700000000, 1800.0,
|
||||
'https://example.test/1.mp3', 1700000000)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.pragma_update(None, "user_version", 1).unwrap();
|
||||
}
|
||||
let services = super::open_services(
|
||||
&path,
|
||||
dir.path().join("covers"),
|
||||
Arc::new(MemoryLog::default()),
|
||||
)
|
||||
.unwrap();
|
||||
let store = &services.store;
|
||||
assert_eq!(
|
||||
store.schema_version().unwrap(),
|
||||
furumi_library::podcasts::PODCAST_SCHEMA_VERSION
|
||||
);
|
||||
let rows = store.list_podcasts(PodcastListFilter::Subscribed).unwrap();
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(rows[0].meta.title, "Old Show");
|
||||
assert_eq!(rows[0].episode_count, 1);
|
||||
let expected =
|
||||
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed.xml").unwrap());
|
||||
assert_eq!(rows[0].podcast_id, expected);
|
||||
let fetch = store.fetch_state(&expected).unwrap();
|
||||
assert_eq!(fetch.etag.as_deref(), Some("\"e1\""));
|
||||
drop(services);
|
||||
|
||||
// The original rows are kept, renamed.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
let legacy: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM legacy_podcast_episodes", [], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(legacy, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreign_or_newer_databases_are_refused_unchanged() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let log: Arc<dyn OpLog> = Arc::new(MemoryLog::default());
|
||||
|
||||
let foreign = dir.path().join("foreign.sqlite3");
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&foreign).unwrap();
|
||||
conn.execute_batch("CREATE TABLE tracks (id INTEGER PRIMARY KEY, title TEXT);")
|
||||
.unwrap();
|
||||
}
|
||||
let newer = dir.path().join("newer.sqlite3");
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&newer).unwrap();
|
||||
conn.pragma_update(None, "user_version", 99).unwrap();
|
||||
}
|
||||
let garbage = dir.path().join("garbage.sqlite3");
|
||||
std::fs::write(
|
||||
&garbage,
|
||||
b"this is not a database at all, just some text bytes",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
for (path, expected) in [
|
||||
(&foreign, "not a Furumi podcast database"),
|
||||
(&newer, "newer than this Furumi supports"),
|
||||
(&garbage, ""),
|
||||
] {
|
||||
let before = std::fs::read(path).unwrap();
|
||||
let Err(err) = super::open_services(path, dir.path().join("covers"), Arc::clone(&log))
|
||||
else {
|
||||
panic!("{} opened", path.display());
|
||||
};
|
||||
let message = super::describe_open_error(&err);
|
||||
assert!(message.contains(expected), "{message}");
|
||||
assert_eq!(std::fs::read(path).unwrap(), before, "{}", path.display());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user