Parse imported prototype feeds on their first check
The prototype import keeps the prototype's ETag and Last-Modified, but the prototype stored no seasons, episode numbers or types, and GUID-less episodes stay provisional until the feed is parsed again. The first check was answered 304 and a feed that no longer changes was never parsed. Drop the imported validators right after the migration so the first check downloads and applies the feed. Contract gap for the FRID import. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
42f22f0fea
commit
fe373e4169
2 files changed
+157
-2
No files matched your search
+65
-1
@@ -42,7 +42,8 @@ use std::sync::Arc;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use furumi_library::podcasts::{
|
||||
PODCAST_DB_FILE, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource,
|
||||
LEGACY_PROTOTYPE_SCHEMA_VERSION, PODCAST_DB_FILE, PodcastListFilter, PodcastLocalOp,
|
||||
PodcastStore, PodcastStoreError, StampSource,
|
||||
};
|
||||
|
||||
pub use fetch::Fetcher;
|
||||
@@ -81,8 +82,12 @@ pub fn covers_dir() -> Result<PathBuf> {
|
||||
/// without writing anything, a database from a newer Furumi or a file that
|
||||
/// is not a podcast database. Blocking: call it on the blocking pool.
|
||||
pub fn open_services(db_path: &Path, covers_dir: PathBuf, log: Arc<dyn OpLog>) -> Result<Services> {
|
||||
let from_prototype = is_prototype_database(db_path);
|
||||
let store = PodcastStore::open(db_path)?;
|
||||
tracing::info!(path = %db_path.display(), "podcast store opened");
|
||||
if from_prototype {
|
||||
forget_prototype_validators(&store);
|
||||
}
|
||||
match store.legacy_quarantine() {
|
||||
Ok(entries) if !entries.is_empty() => tracing::warn!(
|
||||
count = entries.len(),
|
||||
@@ -98,6 +103,65 @@ pub fn open_services(db_path: &Path, covers_dir: PathBuf, log: Arc<dyn OpLog>) -
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether `path` is a database of the prototype podcast build, which
|
||||
/// `PodcastStore::open` is about to import. Read-only; any doubt is "no".
|
||||
fn is_prototype_database(path: &Path) -> bool {
|
||||
if !path.is_file() {
|
||||
return false;
|
||||
}
|
||||
let flags =
|
||||
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX;
|
||||
rusqlite::Connection::open_with_flags(path, flags)
|
||||
.and_then(|conn| conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0)))
|
||||
.is_ok_and(|version| version == LEGACY_PROTOTYPE_SCHEMA_VERSION)
|
||||
}
|
||||
|
||||
/// Drops the HTTP validators imported from a prototype database.
|
||||
///
|
||||
/// The prototype stored no seasons, episode numbers or episode types, and
|
||||
/// its GUID-less episodes stay provisional until a feed is parsed again. With
|
||||
/// the imported `ETag` / `Last-Modified` the first check would be answered
|
||||
/// "304 Not Modified" and the feed never parsed while it does not change, so
|
||||
/// a finished series would never show its seasons. Without them the first
|
||||
/// check downloads and applies the feed. The fetch state is local (never
|
||||
/// synced). Contract gap: FRID's prototype import should not carry the
|
||||
/// validators over.
|
||||
fn forget_prototype_validators(store: &PodcastStore) {
|
||||
let result = (|| -> Result<usize> {
|
||||
let mut cleared = 0;
|
||||
for podcast in store.list_podcasts(PodcastListFilter::All)? {
|
||||
let id = &podcast.podcast_id;
|
||||
let state = store.fetch_state(id)?;
|
||||
if state.etag.is_none() && state.last_modified.is_none() {
|
||||
continue;
|
||||
}
|
||||
let checked = state
|
||||
.last_success_at_ms
|
||||
.or(state.checked_at_ms)
|
||||
.unwrap_or(0);
|
||||
store.record_fetch_success(id, checked, None, None, state.next_check_at_ms)?;
|
||||
if let Some(error) = &state.last_error {
|
||||
store.record_fetch_failure(
|
||||
id,
|
||||
state.checked_at_ms.unwrap_or(checked),
|
||||
error,
|
||||
state.next_check_at_ms,
|
||||
)?;
|
||||
}
|
||||
cleared += 1;
|
||||
}
|
||||
Ok(cleared)
|
||||
})();
|
||||
match result {
|
||||
Ok(0) => {}
|
||||
Ok(cleared) => tracing::info!(
|
||||
cleared,
|
||||
"imported podcasts get a full first feed check (prototype validators dropped)"
|
||||
),
|
||||
Err(err) => tracing::warn!("dropping imported podcast validators failed: {err:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A readable reason why the store did not open; the typed store errors
|
||||
/// already say that the file was left unchanged.
|
||||
pub fn describe_open_error(err: &anyhow::Error) -> String {
|
||||
|
||||
+92
-1
@@ -534,8 +534,12 @@ fn prototype_database_is_migrated_in_place() {
|
||||
let expected =
|
||||
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed.xml").unwrap());
|
||||
assert_eq!(rows[0].podcast_id, expected);
|
||||
// The prototype's validators are dropped: the first check parses the
|
||||
// feed (`imported_podcasts_get_their_seasons_on_the_first_check`).
|
||||
let fetch = store.fetch_state(&expected).unwrap();
|
||||
assert_eq!(fetch.etag.as_deref(), Some("\"e1\""));
|
||||
assert_eq!(fetch.etag, None);
|
||||
assert_eq!(fetch.last_success_at_ms, Some(1_700_000_100_000));
|
||||
assert_eq!(fetch.next_check_at_ms, None, "due at once");
|
||||
drop(services);
|
||||
|
||||
// The original rows are kept, renamed.
|
||||
@@ -586,3 +590,90 @@ fn foreign_or_newer_databases_are_refused_unchanged() {
|
||||
assert_eq!(std::fs::read(path).unwrap(), before, "{}", path.display());
|
||||
}
|
||||
}
|
||||
|
||||
/// A prototype database knew no seasons or episode numbers. Its stored
|
||||
/// `ETag` would make the first check "304 Not Modified" forever for a feed
|
||||
/// that no longer changes, so the seasons would never appear.
|
||||
#[tokio::test]
|
||||
async fn imported_podcasts_get_their_seasons_on_the_first_check() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let server = TestServer::start();
|
||||
let feed = "<?xml version=\"1.0\"?><rss version=\"2.0\" \
|
||||
xmlns:itunes=\"http://www.itunes.com/dtds/podcast-1.0.dtd\"><channel>\
|
||||
<title>Old Show</title><item><title>Old Pilot</title><guid>ep-1</guid>\
|
||||
<pubDate>Sat, 15 Jun 2024 10:00:00 GMT</pubDate>\
|
||||
<itunes:season>1</itunes:season><itunes:episode>1</itunes:episode>\
|
||||
<enclosure url=\"/audio/1.mp3\" type=\"audio/mpeg\" length=\"100\"/></item>\
|
||||
</channel></rss>";
|
||||
// The server answers the prototype's validator with 304.
|
||||
server.set(
|
||||
"/feed.xml",
|
||||
Route::new(feed, "application/rss+xml").etag("\"e1\""),
|
||||
);
|
||||
let feed_url = server.url("/feed.xml");
|
||||
let path = dir.path().join(furumi_library::podcasts::PODCAST_DB_FILE);
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(PROTOTYPE_SCHEMA).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO podcasts (feed_url, title, etag, last_modified, added_at, last_checked_at)
|
||||
VALUES (?1, 'Old Show', '\"e1\"', 'Sat, 15 Jun 2024 10:00:00 GMT',
|
||||
1700000000, 1700000100)",
|
||||
[&feed_url],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_episodes
|
||||
(podcast_id, guid, title, published_at, duration_seconds, audio_url, first_seen_at)
|
||||
VALUES (1, 'ep-1', 'Old Pilot', 1718445600, 1800.0, ?1, 1700000000)",
|
||||
[server.url("/audio/1.mp3")],
|
||||
)
|
||||
.unwrap();
|
||||
conn.pragma_update(None, "user_version", 1).unwrap();
|
||||
}
|
||||
let services = super::open_services(
|
||||
&path,
|
||||
dir.path().join("covers"),
|
||||
Arc::new(MemoryLog::default()),
|
||||
)
|
||||
.unwrap();
|
||||
let store = Arc::clone(&services.store);
|
||||
let id = PodcastId::from_feed_url(&FeedUrl::parse(&feed_url).unwrap());
|
||||
assert!(
|
||||
store
|
||||
.seasons(&id)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|s| s.season.is_none()),
|
||||
"the prototype had no seasons"
|
||||
);
|
||||
let due = store.podcasts_due_for_refresh(NOW, 16).unwrap();
|
||||
assert_eq!(due.len(), 1);
|
||||
let report = refresh(&services, &id, &due[0].1, NOW).await.unwrap();
|
||||
assert!(!report.not_modified);
|
||||
let requests = server.requests("/feed.xml");
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(!requests[0].contains_key("if-none-match"));
|
||||
assert!(!requests[0].contains_key("if-modified-since"));
|
||||
let seasons = store.seasons(&id).unwrap();
|
||||
assert_eq!(
|
||||
seasons.iter().map(|s| s.season).collect::<Vec<_>>(),
|
||||
[Some(1)]
|
||||
);
|
||||
// From then on the checks are conditional again.
|
||||
let report = refresh(&services, &id, &due[0].1, NOW + 1).await.unwrap();
|
||||
assert!(report.not_modified);
|
||||
drop(services);
|
||||
|
||||
// A store that was already migrated keeps its validators on reopen.
|
||||
let services = super::open_services(
|
||||
&path,
|
||||
dir.path().join("covers"),
|
||||
Arc::new(MemoryLog::default()),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
services.store.fetch_state(&id).unwrap().etag.as_deref(),
|
||||
Some("\"e1\"")
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user