//! Podcast store against PostgreSQL. //! //! Ignored by default: set `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a //! disposable database (for example a throwaway `postgres:16-alpine` //! container). The tests migrate it with the production migrations, use //! random account ids and remove their rows again; the migration test //! creates and drops its own database next to it. use std::collections::BTreeSet; use cot::db::migrations::{MigrationEngine, SyncDynMigration}; use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot, random_hex}; use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; use music_dht::podcast::{ EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, }; use super::*; use crate::player::queue_items::test_episode; const DATABASE_URL: &str = "FURU_DEVICE_SYNC_TEST_DATABASE_URL"; /// Every production migration (the list of `admin::App::migrations`). fn all_migrations() -> Vec<&'static SyncDynMigration> { [ crate::config::db_migrations::MIGRATIONS, crate::user::db_migrations::MIGRATIONS, crate::music::db_migrations::MIGRATIONS, crate::scheduler::db_migrations::MIGRATIONS, crate::auth::db_migrations::MIGRATIONS, crate::youtube::db_migrations::MIGRATIONS, ] .into_iter() .flatten() .copied() .collect() } async fn migrate(url: &str, migrations: Vec<&'static SyncDynMigration>) { let database = cot::db::Database::new(url).await.unwrap(); MigrationEngine::new(migrations) .unwrap() .run(&database) .await .unwrap(); database.close().await.unwrap(); } /// Pool on the test database, migrated once per test binary with the /// production migrations (also used by the device-sync Postgres tests). pub(crate) async fn test_pool() -> sqlx::PgPool { static MIGRATED: tokio::sync::Mutex = tokio::sync::Mutex::const_new(false); let url = std::env::var(DATABASE_URL).unwrap(); let mut migrated = MIGRATED.lock().await; if !*migrated { migrate(&url, all_migrations()).await; *migrated = true; } sqlx::PgPool::connect(&url).await.unwrap() } fn random_user() -> i64 { let bytes = blake3::hash(random_hex(8).as_bytes()); 9_000_000_000 + i64::from(u32::from_le_bytes(bytes.as_bytes()[..4].try_into().unwrap()) % 1_000_000_000) } const ACCOUNT_TABLES: &[&str] = &[ "furumusic__podcast_fetch_state", "furumusic__podcast_listen_session", "furumusic__podcast_progress", "furumusic__podcast_share", "furumusic__podcast_subscription", "furumusic__podcast_episode", "furumusic__podcast_feed_alias", "furumusic__podcast", "furumusic__fed_podcast_relay", "furumusic__fed_device_identity", "furumusic__fed_device", "furumusic__fed_sync_ops", "furumusic__fed_sync_vector", "furumusic__fed_state_like", "furumusic__fed_state_playlist", "furumusic__fed_state_playlist_item", ]; async fn cleanup(pool: &sqlx::PgPool, users: impl IntoIterator) { let users: Vec = users.into_iter().collect(); for table in ACCOUNT_TABLES { sqlx::query(&format!("DELETE FROM {table} WHERE user_id = ANY($1)")) .bind(&users) .execute(pool) .await .unwrap(); } } fn feed_url(path: &str) -> FeedUrl { FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap() } fn meta(path: &str, title: &str) -> PodcastMeta { PodcastMeta { feed_url: feed_url(path), title: title.into(), author: Some("Host".into()), description: Some("Channel description".into()), link: None, image_url: None, } } fn op(origin: &str, seq: i64, hlc_ms: i64, payload: SyncOpPayload) -> SyncOpWire { SyncOpWire { op_id: format!("{origin}:{seq}"), origin_device_id: origin.into(), seq, hlc_ms, payload, } } fn session( id: &str, device: &str, episode: &PodcastEpisodeRef, listened_ms: i64, completed: bool, ) -> ListenSession { ListenSession { session_id: id.into(), episode_id: episode.episode_id.clone(), podcast_id: episode.podcast_id.clone(), device_id: device.into(), started_at_ms: 1_000, ended_at_ms: 1_000 + listened_ms, start_position_ms: 0, end_position_ms: listened_ms, listened_ms, completed, } } fn progress_op( origin: &str, seq: i64, hlc_ms: i64, episode: &PodcastEpisodeRef, generation: u64, position_ms: i64, ) -> SyncOpWire { op( origin, seq, hlc_ms, SyncOpPayload::PodcastProgressSet { episode_id: episode.episode_id.clone(), podcast_id: episode.podcast_id.clone(), position_ms, status: ProgressStatus::InProgress, generation, session_id: format!("{origin}-g{generation}"), episode: Some(PodcastEpisodeRef { progress_generation: None, ..episode.clone() }), }, ) } /// Every permutation of `0..n` (Heap's algorithm). fn permutations(n: usize) -> Vec> { fn heap(k: usize, items: &mut Vec, out: &mut Vec>) { if k <= 1 { out.push(items.clone()); return; } for i in 0..k { heap(k - 1, items, out); let j = if k.is_multiple_of(2) { i } else { 0 }; items.swap(j, k - 1); } } let mut out = Vec::new(); heap(n, &mut (0..n).collect(), &mut out); out } async fn apply_all(store: &PodcastStore, user_id: i64, ops: &[SyncOpWire]) { for op in ops { let outcome = store.apply_remote_op(user_id, op).await.unwrap(); assert!( matches!(outcome, ApplyOutcome::Changed | ApplyOutcome::Unchanged), "{outcome:?}" ); } } /// The registers and sessions of one podcast and episode from three /// devices, with a duplicate, a late checkpoint of an older session, a /// concurrent offline action and a growing session. fn convergence_ops() -> Vec { let episode = test_episode(); let podcast_id = episode.podcast_id.clone(); let podcast = PodcastMeta { feed_url: episode.feed_url.clone(), title: episode.podcast_title.clone(), author: None, description: Some("dropped from snapshots".into()), link: None, image_url: None, }; let subscribe = op( "tui_a", 1, 100, SyncOpPayload::PodcastSubscriptionSet { podcast_id: podcast_id.clone(), subscribed: true, podcast: podcast.clone(), feed_aliases: vec![], generation: 1, }, ); vec![ subscribe.clone(), // B saw g1 and unsubscribes with a slow clock: g2 still wins. op( "tui_b", 1, 50, SyncOpPayload::PodcastSubscriptionSet { podcast_id: podcast_id.clone(), subscribed: false, podcast, feed_aliases: vec![], generation: 2, }, ), // A shared concurrently at g2: stored, but not effective. op( "tui_a", 2, 300, SyncOpPayload::PodcastShareSet { podcast_id: podcast_id.clone(), shared: true, generation: 2, }, ), op( "tui_c", 1, 400, SyncOpPayload::PodcastFeedAliasesAdded { podcast_id, feed_aliases: vec![feed_url("moved.xml")], }, ), subscribe, // A late g3 checkpoint of an old session with a fast clock. progress_op("tui_a", 3, 9_000, &episode, 3, 600_000), // B and C both seek at g4 with the same clock: device id decides. progress_op("tui_b", 2, 1_000, &episode, 4, 10_000), progress_op("tui_c", 2, 1_000, &episode, 4, 20_000), op( "tui_a", 4, 9_100, SyncOpPayload::PodcastListenRecorded { session: session("s1", "tui_a", &episode, 40_000, false), episode: None, }, ), op( "tui_a", 5, 9_200, SyncOpPayload::PodcastListenRecorded { session: session("s1", "tui_a", &episode, 90_000, true), episode: None, }, ), op( "tui_b", 3, 1_100, SyncOpPayload::PodcastListenRecorded { session: session("s2", "tui_b", &episode, 31_000, false), episode: Some(PodcastEpisodeRef { progress_generation: None, ..episode }), }, ), ] } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_ops_converge_in_any_order_and_match_snapshot_import() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let ops = convergence_ops(); let reference_user = random_user(); let mut users = vec![reference_user]; apply_all(&store, reference_user, &ops).await; let reference = store.export_snapshot(reference_user).await.unwrap(); // The contract outcome, independently of the order. let episode = test_episode(); assert_eq!(reference.podcast_subscriptions.len(), 1); let subscription = &reference.podcast_subscriptions[0]; assert!(!subscription.subscribed); assert_eq!(subscription.generation, 2); assert!(subscription.podcast.description.is_none()); assert_eq!( subscription.feed_urls(), vec![episode.feed_url.clone(), feed_url("moved.xml")] ); assert!(reference.podcast_shares[0].shared); let summary = store .podcast(reference_user, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(!summary.subscribed && !summary.shared); assert_eq!(reference.podcast_progress.len(), 1); assert_eq!(reference.podcast_progress[0].position_ms, 20_000); assert_eq!(reference.podcast_progress[0].device_id, "tui_c"); let sessions: BTreeSet<_> = reference .podcast_listen_sessions .iter() .map(|session| { ( session.session_id.as_str(), session.listened_ms, session.completed, ) }) .collect(); assert_eq!( sessions, BTreeSet::from([("s1", 90_000, true), ("s2", 31_000, false)]) ); assert_eq!( store .completed_listen_count(reference_user, &episode.episode_id) .await .unwrap(), 1 ); // The episode reference created a catalog row (never touching progress). let item = store .episode(reference_user, &episode.episode_id) .await .unwrap() .unwrap(); assert!(!item.in_feed); assert_eq!(item.resume_position_ms(), 20_000); // Register operations: all 120 orders of the first five operations // (subscription, unsubscription, share, aliases, duplicate) with the // rest appended; progress and session operations: all 720 orders of // the last six with the first five prepended. let (registers, personal) = ops.split_at(5); for order in permutations(registers.len()) { let user_id = random_user(); users.push(user_id); let mut shuffled: Vec = order.iter().map(|&i| registers[i].clone()).collect(); shuffled.extend_from_slice(personal); apply_all(&store, user_id, &shuffled).await; assert_eq!( store.export_snapshot(user_id).await.unwrap(), reference, "{order:?}" ); } for order in permutations(personal.len()) { let user_id = random_user(); users.push(user_id); let mut shuffled = registers.to_vec(); shuffled.extend(order.iter().map(|&i| personal[i].clone())); apply_all(&store, user_id, &shuffled).await; assert_eq!( store.export_snapshot(user_id).await.unwrap(), reference, "{order:?}" ); } // A new device recovering from the snapshot reaches the same state, and // the operations delivered afterwards change nothing. (Snapshots omit // the channel description, so the winning subscription operation only // adds it back to the stored register: the contract's tie-break by // metadata. Every other operation is already reflected.) let recovered = random_user(); users.push(recovered); let stats = store.apply_snapshot(recovered, &reference).await.unwrap(); assert_eq!(stats.rejected, 0); assert!(!stats.ignored_over_limits); assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); for op in ops.iter().rev() { let expected = if op.op_id == "tui_b:1" { ApplyOutcome::Changed } else { ApplyOutcome::Unchanged }; assert_eq!( store.apply_remote_op(recovered, op).await.unwrap(), expected, "{}", op.op_id ); } assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); // A stale snapshot never replaces newer state. let stale = SyncSnapshot { podcast_progress: vec![ProgressRecord { position_ms: 1, generation: 3, hlc_ms: i64::MAX, ..reference.podcast_progress[0].clone() }], ..SyncSnapshot::default() }; let stats = store.apply_snapshot(recovered, &stale).await.unwrap(); assert_eq!((stats.changed, stats.unchanged), (0, 1)); assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); // Invalid operations are rejected without touching state. let mut invalid = progress_op("tui_a", 9, 1, &episode, 9, 1); if let SyncOpPayload::PodcastProgressSet { session_id, .. } = &mut invalid.payload { session_id.clear(); } assert!(matches!( store.apply_remote_op(recovered, &invalid).await.unwrap(), ApplyOutcome::Rejected(_) )); let mut huge = progress_op("tui_a", 10, 1, &episode, u64::MAX, 1); huge.op_id = "tui_a:10".into(); assert!(matches!( store.apply_remote_op(recovered, &huge).await.unwrap(), ApplyOutcome::Rejected(_) )); assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); cleanup(&pool, users).await; } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_local_actions_keep_progress_through_unsubscribe() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user_id = random_user(); let device = crate::federation::devices::local_device_id(&pool, user_id) .await .unwrap(); let episode = test_episode(); let show = PodcastMeta { feed_url: episode.feed_url.clone(), ..meta("unused.xml", &episode.podcast_title) }; let subscribed = store.subscribe(user_id, &show).await.unwrap(); assert_eq!(subscribed.podcast_id, episode.podcast_id); assert!(!subscribed.already_subscribed); let [first] = subscribed.ops.as_slice() else { panic!("one operation"); }; assert_eq!(first.origin_device_id, device); assert_eq!(first.op_id, format!("{device}:{}", first.seq)); // The operation is in the log with exactly this stamp; applying it again // is a no-op. let logged: serde_json::Value = sqlx::query_scalar( "SELECT payload_json FROM furumusic__fed_sync_ops WHERE user_id = $1 AND op_id = $2 AND hlc_ms = $3 AND kind = 'podcast_subscription_set'", ) .bind(user_id) .bind(&first.op_id) .bind(first.hlc_ms) .fetch_one(&pool) .await .unwrap(); assert_eq!(logged, serde_json::to_value(&first.payload).unwrap()); assert_eq!( store.apply_remote_op(user_id, first).await.unwrap(), ApplyOutcome::Unchanged ); assert!( store .subscribe(user_id, &show) .await .unwrap() .already_subscribed ); assert_eq!( store .set_shared(user_id, &episode.podcast_id, true) .await .unwrap() .len(), 1 ); assert!( store .set_shared(user_id, &episode.podcast_id, true) .await .unwrap() .is_empty() ); assert!( store .podcast(user_id, &episode.podcast_id) .await .unwrap() .unwrap() .shared ); // Listening: an explicit start, checkpoints keep the generation. let start = store .start_session(user_id, &episode, "web-s1", 0, episode.progress_generation) .await .unwrap(); assert_eq!(start.record.generation, 8, "max(local, carried 7) + 1"); let CheckpointOutcome::Written(checkpoint) = store .checkpoint(user_id, &start.session, 30_000) .await .unwrap() else { panic!("checkpoint written"); }; assert_eq!(checkpoint.record.generation, 8); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 30_000 ); // A seek is explicit: later checkpoints use its generation. let seek = store .seek(user_id, &checkpoint.session, 12_000) .await .unwrap(); assert_eq!(seek.record.generation, 9); assert_eq!(seek.session.generation, 9); let start = seek; // Another device seeks back at a greater generation with a slow clock: // later checkpoints of the old session are refused. store .apply_remote_op(user_id, &progress_op("tui_b", 1, 1, &episode, 10, 5_000)) .await .unwrap(); assert!(matches!( store .checkpoint(user_id, &start.session, 900_000) .await .unwrap(), CheckpointOutcome::Superseded { .. } )); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 5_000 ); let listened = ListenSession { device_id: device.clone(), ..session("web-s1", &device, &episode, 45_000, false) }; assert!( store .record_listen_session(user_id, &listened, None) .await .unwrap() .is_some() ); assert!( store .record_listen_session(user_id, &listened, None) .await .unwrap() .is_none() ); assert!( store .record_listen_session( user_id, &session("x", "tui_b", &episode, 45_000, false), None ) .await .is_err() ); // Unsubscribing keeps progress and history and revokes the share. let ops = store .unsubscribe(user_id, &episode.podcast_id) .await .unwrap(); assert_eq!(ops.len(), 2); assert!( store .unsubscribe(user_id, &episode.podcast_id) .await .unwrap() .is_empty() ); let summary = store .podcast(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(!summary.subscribed && !summary.shared); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 5_000 ); assert_eq!( store.listen_history(user_id, None, 10).await.unwrap().len(), 1 ); assert!( store .list_podcasts(user_id, PodcastListFilter::Subscribed) .await .unwrap() .is_empty() ); assert_eq!( store .list_podcasts(user_id, PodcastListFilter::All) .await .unwrap() .len(), 1 ); // Subscribing again resolves to the same podcast, continues the // progress and is private again. let again = store.subscribe(user_id, &show).await.unwrap(); assert_eq!(again.podcast_id, episode.podcast_id); let register = store .subscription(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); let share = store .share(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(register.subscribed); assert!(register.generation > share.generation); let summary = store .podcast(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(summary.subscribed && !summary.shared); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 5_000 ); // Mark played writes progress only; restart and unplayed reset it. let played = store.mark_played(user_id, &episode).await.unwrap(); assert_eq!(played.record.status, ProgressStatus::Completed); assert_eq!(played.record.generation, 11); assert_eq!( store.listen_history(user_id, None, 10).await.unwrap().len(), 1 ); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 0 ); let restarted = store.restart(user_id, &episode, "web-s2").await.unwrap(); assert_eq!( ( restarted.record.position_ms, restarted.record.status, restarted.record.generation ), (0, ProgressStatus::InProgress, 12) ); let CheckpointOutcome::Written(finished) = store .complete_if_current(user_id, &restarted.session, 1_790_000) .await .unwrap() else { panic!("the running session completes the episode"); }; assert_eq!( (finished.record.status, finished.record.generation), (ProgressStatus::Completed, 13) ); assert!(matches!( store .checkpoint(user_id, &finished.session, 1_795_000) .await .unwrap(), CheckpointOutcome::AlreadyCompleted { .. } )); let mut played_filter = EpisodeQuery::new(episode.podcast_id.clone()); played_filter.played = PlayedFilter::Played; assert_eq!( store.episodes(user_id, &played_filter).await.unwrap().len(), 1 ); played_filter.played = PlayedFilter::Unplayed; assert!( store .episodes(user_id, &played_filter) .await .unwrap() .is_empty() ); assert_eq!( store .mark_unplayed(user_id, &episode) .await .unwrap() .record .status, ProgressStatus::NotStarted ); played_filter.played = PlayedFilter::NotStarted; let items = store.episodes(user_id, &played_filter).await.unwrap(); assert_eq!(items[0].status(), ProgressStatus::NotStarted); // Local stamps: consecutive sequence numbers, strictly growing clock. let stamps: Vec<(i64, i64)> = sqlx::query_as( "SELECT seq, hlc_ms FROM furumusic__fed_sync_ops WHERE user_id = $1 AND origin_device_id = $2 ORDER BY seq", ) .bind(user_id) .bind(&device) .fetch_all(&pool) .await .unwrap(); assert!( stamps .windows(2) .all(|pair| pair[1].0 == pair[0].0 + 1 && pair[1].1 > pair[0].1) ); // Re-applying the whole local log changes nothing. let before = store.export_snapshot(user_id).await.unwrap(); let payloads: Vec<(String, i64, i64, serde_json::Value)> = sqlx::query_as( "SELECT op_id, seq, hlc_ms, payload_json FROM furumusic__fed_sync_ops WHERE user_id = $1 AND kind LIKE 'podcast\\_%' ORDER BY hlc_ms", ) .bind(user_id) .fetch_all(&pool) .await .unwrap(); for (op_id, seq, hlc_ms, payload) in payloads { let origin = op_id.rsplit_once(':').unwrap().0.to_string(); let op = SyncOpWire { op_id, origin_device_id: origin, seq, hlc_ms, payload: serde_json::from_value(payload).unwrap(), }; assert_eq!( store.apply_remote_op(user_id, &op).await.unwrap(), ApplyOutcome::Unchanged ); } assert_eq!(store.export_snapshot(user_id).await.unwrap(), before); // Sharing needs a subscription. let unknown = PodcastId::from_feed_url(&feed_url("unknown.xml")); let error = store.set_shared(user_id, &unknown, true).await.unwrap_err(); assert!(matches!( error.downcast_ref::(), Some(PodcastStoreError::NotSubscribed(_)) )); cleanup(&pool, [user_id]).await; } fn rss(items: &str) -> String { format!( r#" Refresh Show https://show.example/ Channel text Host {items} "# ) } fn parse(xml: &str, url: &FeedUrl) -> ParsedFeed { parse_feed_with_base(xml.as_bytes(), Some(url.as_str())).unwrap() } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_feed_refresh_keeps_ids_and_progress() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user_id = random_user(); let url = feed_url("refresh.xml"); let podcast_id = PodcastId::from_feed_url(&url); let v1 = parse( &rss(r#" g-1One1 1Mon, 01 Jan 2024 10:00:00 GMT Twohttps://show.example/ep2 12 Mon, 08 Jan 2024 10:00:00 GMT Talk about Ünïcode ThreeMon, 15 Jan 2024 10:00:00 GMT "#), &url, ); // What every device derives for this document. let ids: Vec = v1 .resolve_episode_ids(&podcast_id, &[]) .into_iter() .map(Option::unwrap) .collect(); // Another device played the GUID-less "Two" before this account ever // fetched the feed: its reference arrives with the progress. let two = PodcastEpisodeRef { podcast_id: podcast_id.clone(), episode_id: ids[1].clone(), feed_url: url.clone(), podcast_title: "Refresh Show".into(), episode_title: "Two".into(), audio_url: "https://cdn.example/a2.mp3".into(), mime: None, duration_ms: None, published_at_ms: None, season: None, episode_number: None, image_url: None, progress_generation: None, }; store .apply_remote_op(user_id, &progress_op("tui_a", 1, 10, &two, 3, 61_000)) .await .unwrap(); let subscribed = store .subscribe(user_id, &v1.to_podcast_meta(url.clone())) .await .unwrap(); assert_eq!(subscribed.podcast_id, podcast_id); let first = store .apply_feed( user_id, &FeedRefresh { podcast_id: &podcast_id, feed: &v1, fetched_from: &url, moved_permanently: false, }, ) .await .unwrap(); assert_eq!((first.new_episodes, first.updated_episodes), (2, 1)); assert!(first.ops.is_empty()); let stored: BTreeSet = store .episodes(user_id, &EpisodeQuery::new(podcast_id.clone())) .await .unwrap() .into_iter() .map(|item| item.episode.episode_id) .collect(); assert_eq!(stored, ids.iter().cloned().collect()); let progress_before = store.progress(user_id, &ids[1]).await.unwrap().unwrap(); // The publisher renames episodes, moves their audio, drops "Three", // adds "Four" and moves the feed permanently. let moved = feed_url("moved-refresh.xml"); let v2 = parse( &rss(r#" g-1One (remastered)1 1Mon, 01 Jan 2024 10:00:00 GMT Two, revisedhttps://show.example/ep2 12 Mon, 08 Jan 2024 10:00:00 GMT Talk about Ünïcode g-4Four2 1Mon, 22 Jan 2024 10:00:00 GMT "#), &moved, ); let refresh = FeedRefresh { podcast_id: &podcast_id, feed: &v2, fetched_from: &moved, moved_permanently: true, }; let second = store.apply_feed(user_id, &refresh).await.unwrap(); assert_eq!( ( second.new_episodes, second.updated_episodes, second.unchanged_episodes ), (1, 2, 0) ); assert_eq!(second.ops.len(), 1, "aliases changed once"); assert_eq!(second.feed_url.as_ref(), Some(&moved)); let again = store.apply_feed(user_id, &refresh).await.unwrap(); assert!(again.ops.is_empty()); assert_eq!(again.unchanged_episodes, 3); assert_eq!( store.resolve_podcast_id(user_id, &moved).await.unwrap(), podcast_id ); assert_eq!( store.resolve_podcast_id(user_id, &url).await.unwrap(), podcast_id ); // Ids and progress survive; the dropped episode stays out of the feed. let one = store.episode(user_id, &ids[0]).await.unwrap().unwrap(); assert_eq!(one.episode.title, "One (remastered)"); assert_eq!(one.episode.audio_url, "https://cdn2.example/a1.mp3"); let two_item = store.episode(user_id, &ids[1]).await.unwrap().unwrap(); assert_eq!(two_item.episode.audio_url, "https://cdn2.example/a2.mp3"); assert_eq!(two_item.progress.as_ref(), Some(&progress_before)); assert_eq!(two_item.resume_position_ms(), 61_000); let three = store.episode(user_id, &ids[2]).await.unwrap().unwrap(); assert!(!three.in_feed); // The subscription register is untouched by refreshes. let register = store .subscription(user_id, &podcast_id) .await .unwrap() .unwrap(); assert_eq!(register.op_id, subscribed.ops[0].op_id); assert!(register.feed_urls().contains(&moved)); let reference = store.episode_ref(user_id, &ids[1]).await.unwrap().unwrap(); assert_eq!(reference.progress_generation, Some(3)); assert_eq!(reference.audio_url, "https://cdn2.example/a2.mp3"); // UI queries: counters, seasons, filters, search and sorting. let summary = store.podcast(user_id, &podcast_id).await.unwrap().unwrap(); assert_eq!( ( summary.episode_count, summary.unplayed_count, summary.in_progress_count, summary.season_count ), (4, 4, 1, 2) ); assert_eq!(summary.meta.feed_url, moved); let seasons = store.seasons(user_id, &podcast_id).await.unwrap(); assert_eq!( seasons .iter() .map(|season| (season.season, season.episode_count)) .collect::>(), vec![(Some(1), 2), (Some(2), 1), (None, 1)] ); let query = |configure: &dyn Fn(&mut EpisodeQuery)| { let mut query = EpisodeQuery::new(podcast_id.clone()); configure(&mut query); query }; let titles = |items: Vec| { items .into_iter() .map(|item| item.episode.title) .collect::>() }; assert_eq!( titles(store.episodes(user_id, &query(&|_| {})).await.unwrap()), ["Four", "Three", "Two, revised", "One (remastered)"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.sort = EpisodeSort::EpisodeNumber)) .await .unwrap() ), ["One (remastered)", "Two, revised", "Four", "Three"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.season = SeasonFilter::Season(1))) .await .unwrap() ), ["Two, revised", "One (remastered)"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.season = SeasonFilter::NoSeason)) .await .unwrap() ), ["Three"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.played = PlayedFilter::InProgress)) .await .unwrap() ), ["Two, revised"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.search = Some("ÜNÏCODE".into()))) .await .unwrap() ), ["Two, revised"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.search = Some("remaster".into()))) .await .unwrap() ), ["One (remastered)"] ); assert_eq!( titles( store .episodes( user_id, &query(&|q| { q.limit = Some(1); q.offset = 1; }) ) .await .unwrap() ), ["Three"] ); assert!( store .episodes(user_id, &query(&|q| q.search = Some("one two".into()))) .await .unwrap() .is_empty() ); // A refresh of a podcast this account does not know is refused. let unknown = PodcastId::from_feed_url(&feed_url("never.xml")); let error = store .apply_feed( user_id, &FeedRefresh { podcast_id: &unknown, ..refresh }, ) .await .unwrap_err(); assert!(matches!( error.downcast_ref::(), Some(PodcastStoreError::UnknownPodcast(_)) )); cleanup(&pool, [user_id]).await; } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_fetch_state_is_local_only() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user_id = random_user(); let due_show = meta("due.xml", "Due"); let later_show = meta("later.xml", "Later"); let gone_show = meta("gone.xml", "Gone"); let mut ids = Vec::new(); for show in [&due_show, &later_show, &gone_show] { ids.push(store.subscribe(user_id, show).await.unwrap().podcast_id); } store.unsubscribe(user_id, &ids[2]).await.unwrap(); let before = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap(); let now = 1_800_000_000_000; store .record_fetch_success( user_id, &ids[1], now, Some("\"secret-etag\""), Some("Tue"), Some(now + 60_000), ) .await .unwrap(); assert_eq!( store .record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1)) .await .unwrap(), 1 ); assert_eq!( store .record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1)) .await .unwrap(), 2 ); store .set_cover_cache( user_id, &ids[0], Some("https://img.example/c.jpg"), Some("/var/cache/cover-secret.jpg"), ) .await .unwrap(); let state = store.fetch_state(user_id, &ids[0]).await.unwrap(); assert_eq!(state.consecutive_failures, 2); assert_eq!(state.last_error.as_deref(), Some("boom: 503")); assert_eq!( state.cover_path.as_deref(), Some("/var/cache/cover-secret.jpg") ); store .record_fetch_not_modified(user_id, &ids[1], now + 1, Some(now + 60_000)) .await .unwrap(); let later = store.fetch_state(user_id, &ids[1]).await.unwrap(); assert_eq!(later.etag.as_deref(), Some("\"secret-etag\"")); assert_eq!(later.last_success_at_ms, Some(now + 1)); // Fetch state changes nothing in the synced state. let after = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap(); assert_eq!(after, before); for secret in ["secret-etag", "boom", "cover-secret", "Channel description"] { assert!(!after.contains(secret), "{secret}"); } let ops: Vec = sqlx::query_scalar( "SELECT payload_json::text FROM furumusic__fed_sync_ops WHERE user_id = $1", ) .bind(user_id) .fetch_all(&pool) .await .unwrap(); assert!( ops.iter() .all(|op| !op.contains("secret-etag") && !op.contains("cover-secret")) ); // Scheduling: overdue first, future and unsubscribed podcasts skipped. let due: Vec = store .podcasts_due_for_refresh(now, 10_000, usize::MAX) .await .unwrap() .into_iter() .filter(|(user, ..)| *user == user_id) .map(|(_, id, _)| id) .collect(); assert_eq!(due, vec![ids[0].clone()]); let unknown = PodcastId::from_feed_url(&feed_url("unknown-fetch.xml")); assert!( store .record_fetch_failure(user_id, &unknown, now, "x", None) .await .is_err() ); cleanup(&pool, [user_id]).await; } /// Relay rows exactly as 0.13.0 stored them (`podcast_relay`): the contract /// records as JSON, keyed by section and entity. async fn insert_relay( pool: &sqlx::PgPool, user_id: i64, section: &str, key: &str, value: serde_json::Value, ) { sqlx::query( "INSERT INTO furumusic__fed_podcast_relay (user_id, section, record_key, record_json, updated_at_ms) VALUES ($1, $2, $3, $4, 1700000000000)", ) .bind(user_id) .bind(section) .bind(key) .bind(value) .execute(pool) .await .unwrap(); } async fn insert_logged_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire, kind: &str) { sqlx::query( "INSERT INTO furumusic__fed_sync_ops (user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms, received_at_ms, tombstone) VALUES ($1, $2, $3, $4, $5, $6, $7, 1700000000000, FALSE)", ) .bind(user_id) .bind(&op.op_id) .bind(&op.origin_device_id) .bind(op.seq) .bind(kind) .bind(serde_json::to_value(&op.payload).unwrap()) .bind(op.hlc_ms) .execute(pool) .await .unwrap(); } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_migrations_import_the_relay_state() { let admin_url = std::env::var(DATABASE_URL).unwrap(); let admin = sqlx::PgPool::connect(&admin_url).await.unwrap(); let name = format!("furu_podcast_migration_{}", random_hex(6)); sqlx::query(&format!("CREATE DATABASE {name}")) .execute(&admin) .await .unwrap(); let (base, _) = admin_url.rsplit_once('/').unwrap(); let url = format!("{base}/{name}"); // A 0.13.0 database: every migration up to the relay table. let new_migrations = ["m_0056_create_podcast_store", "m_0057_import_podcast_relay"]; let old: Vec<_> = all_migrations() .into_iter() .filter(|migration| { !(migration.app_name() == "furumusic" && new_migrations.contains(&migration.name())) }) .collect(); migrate(&url, old).await; let pool = sqlx::PgPool::connect(&url).await.unwrap(); // State relayed by 0.13.0 for two accounts. let episode = test_episode(); let user_a = 1; let user_b = 2; let subscription = SubscriptionRecord { podcast_id: episode.podcast_id.clone(), subscribed: true, podcast: PodcastMeta { feed_url: episode.feed_url.clone(), title: episode.podcast_title.clone(), author: Some("Host".into()), description: None, link: None, image_url: Some("https://img.example/show.jpg".into()), }, feed_aliases: vec![episode.feed_url.clone(), feed_url("old.xml")], generation: 3, hlc_ms: 30, op_id: "tui_a:3".into(), }; let share = ShareRecord { podcast_id: episode.podcast_id.clone(), shared: false, generation: 2, hlc_ms: 20, op_id: "tui_a:2".into(), }; let progress = ProgressRecord { episode_id: episode.episode_id.clone(), podcast_id: episode.podcast_id.clone(), position_ms: 1_200_000, status: ProgressStatus::InProgress, generation: 9, session_id: "tui_a-g9".into(), hlc_ms: 90, device_id: "tui_a".into(), op_id: "tui_a:9".into(), }; let listened = session("s9", "tui_a", &episode, 120_000, true); let pending_podcast = PodcastId::from_feed_url(&feed_url("pending.xml")); insert_relay( &pool, user_a, "subscription", subscription.podcast_id.as_str(), serde_json::to_value(&subscription).unwrap(), ) .await; insert_relay( &pool, user_a, "share", share.podcast_id.as_str(), serde_json::to_value(&share).unwrap(), ) .await; insert_relay( &pool, user_a, "progress", progress.episode_id.as_str(), serde_json::to_value(&progress).unwrap(), ) .await; insert_relay( &pool, user_a, "session", "s9", serde_json::to_value(&listened).unwrap(), ) .await; insert_relay( &pool, user_a, "aliases", pending_podcast.as_str(), serde_json::to_value(vec![feed_url("pending.xml")]).unwrap(), ) .await; // An out-of-range generation (no conforming client) is skipped. let huge = ProgressRecord { episode_id: EpisodeId::from_guid(&episode.podcast_id, "huge").unwrap(), generation: u64::MAX, ..progress.clone() }; insert_relay( &pool, user_a, "progress", huge.episode_id.as_str(), serde_json::to_value(&huge).unwrap(), ) .await; insert_relay( &pool, user_b, "progress", progress.episode_id.as_str(), serde_json::to_value(&ProgressRecord { position_ms: 5, ..progress.clone() }) .unwrap(), ) .await; // The relayed operations kept the episode references (two versions). let old_ref = PodcastEpisodeRef { episode_title: "Old title".into(), progress_generation: None, ..episode.clone() }; insert_logged_op( &pool, user_a, &progress_op("tui_a", 8, 80, &old_ref, 8, 1_000), "podcast_progress_set", ) .await; insert_logged_op( &pool, user_a, &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000), "podcast_progress_set", ) .await; // Upgrade. migrate(&url, all_migrations()).await; let store = PodcastStore::new(pool.clone()); let mut expected = SyncSnapshot { podcast_subscriptions: vec![subscription.clone()], podcast_shares: vec![share], podcast_progress: vec![progress.clone()], podcast_listen_sessions: vec![listened], ..SyncSnapshot::default() }; expected.bound_podcast_state(); assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected); assert_eq!( store .export_snapshot(user_b) .await .unwrap() .podcast_progress[0] .position_ms, 5 ); assert_eq!( store.feed_aliases(user_a, &pending_podcast).await.unwrap(), vec![feed_url("pending.xml")] ); // Catalog rows: the podcast from its register, the episode from the // newest reference in the log. let summary = store .podcast(user_a, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(summary.subscribed); assert_eq!( summary.meta.image_url.as_deref(), Some("https://img.example/show.jpg") ); let item = store .episode(user_a, &episode.episode_id) .await .unwrap() .unwrap(); assert_eq!(item.episode.title, episode.episode_title); assert_eq!(item.episode.season, episode.season); assert_eq!(item.resume_position_ms(), 1_200_000); assert_eq!(item.completed_count, 1); // Imported rows are searchable once their search text is computed. let mut search = EpisodeQuery::new(episode.podcast_id.clone()); search.search = Some("episode 42".into()); assert_eq!(store.episodes(user_a, &search).await.unwrap().len(), 1); // The store keeps working on imported rows, and the relay table stays. assert_eq!( store .apply_remote_op(user_a, &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000)) .await .unwrap(), ApplyOutcome::Unchanged ); let relay_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM furumusic__fed_podcast_relay") .fetch_one(&pool) .await .unwrap(); assert_eq!(relay_rows, 7); // Running the import again (or the migrations) changes nothing. for sql in schema::import_statements("TRUE") { sqlx::query(&sql).execute(&pool).await.unwrap(); } migrate(&url, all_migrations()).await; assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected); pool.close().await; sqlx::query(&format!("DROP DATABASE {name} WITH (FORCE)")) .execute(&admin) .await .unwrap(); } /// Alias sets are compared and exported in byte order (as /// `merge_feed_aliases` returns them) whatever the database collation is: /// an ICU database sorts "a" before "B", bytes sort "B" first. #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_alias_sets_ignore_the_database_collation() { let admin_url = std::env::var(DATABASE_URL).unwrap(); let admin = sqlx::PgPool::connect(&admin_url).await.unwrap(); let name = format!("furu_podcast_icu_{}", random_hex(6)); sqlx::query(&format!( "CREATE DATABASE {name} TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' LOCALE 'C'" )) .execute(&admin) .await .unwrap(); let (base, _) = admin_url.rsplit_once('/').unwrap(); let url = format!("{base}/{name}"); migrate(&url, all_migrations()).await; let pool = sqlx::PgPool::connect(&url).await.unwrap(); let ordered: Vec = sqlx::query_scalar("SELECT array_agg(x ORDER BY x) FROM (VALUES ('B'), ('a')) v(x)") .fetch_one(&pool) .await .unwrap(); assert_eq!(ordered, ["a", "B"], "the database must not sort bytes"); let store = PodcastStore::new(pool.clone()); let user_id = 1; let feed = feed_url("feed.xml"); let podcast_id = PodcastId::from_feed_url(&feed); let mixed = vec![feed_url("a"), feed_url("B"), feed_url("a-c")]; let mut sorted = mixed.clone(); sorted.push(feed.clone()); sorted.sort(); let subscribe = op( "tui_a", 1, 100, SyncOpPayload::PodcastSubscriptionSet { podcast_id: podcast_id.clone(), subscribed: true, podcast: PodcastMeta { feed_url: feed.clone(), title: "Refresh Show".into(), author: None, description: None, link: None, image_url: None, }, feed_aliases: mixed.clone(), generation: 1, }, ); let aliases = op( "tui_a", 2, 200, SyncOpPayload::PodcastFeedAliasesAdded { podcast_id: podcast_id.clone(), feed_aliases: mixed.clone(), }, ); assert_eq!( store.apply_remote_op(user_id, &subscribe).await.unwrap(), ApplyOutcome::Changed ); for op in [&subscribe, &aliases, &aliases, &subscribe] { assert_eq!( store.apply_remote_op(user_id, op).await.unwrap(), ApplyOutcome::Unchanged, "{}", op.op_id ); } assert_eq!( store.feed_aliases(user_id, &podcast_id).await.unwrap(), sorted ); let snapshot = store.export_snapshot(user_id).await.unwrap(); assert_eq!(snapshot.podcast_subscriptions[0].feed_aliases, sorted); // A permanent move to an alias already known emits nothing. let document = parse( &rss(r#" g-1One "#), &feed_url("B"), ); let refresh = store .apply_feed( user_id, &FeedRefresh { podcast_id: &podcast_id, feed: &document, fetched_from: &feed_url("B"), moved_permanently: true, }, ) .await .unwrap(); assert!(refresh.ops.is_empty(), "{:?}", refresh.ops); assert_eq!( store.feed_aliases(user_id, &podcast_id).await.unwrap(), sorted ); pool.close().await; sqlx::query(&format!("DROP DATABASE {name} WITH (FORCE)")) .execute(&admin) .await .unwrap(); } // --------------------------------------------------------------------------- // RSS refresh job // --------------------------------------------------------------------------- fn refresh_item(guid: &str) -> String { format!( r#"Episode {guid}{guid} "# ) } /// A fetcher that may reach the local test server (loopback is allowed /// only by the test policy). fn local_fetcher() -> FeedFetcher { FeedFetcher::with_http( net::SafeHttp::with_lookup( fetch::FEED_HTTP, net::AddressPolicy::test_allow_loopback(), std::sync::Arc::new(net::SystemLookup), ) .unwrap(), ) } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_refresh_takes_accounts_in_turns() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let (busy, quiet) = (random_user(), random_user()); let now = 1_900_000_000_000; let mut busy_ids = Vec::new(); for (index, path) in ["turn-a.xml", "turn-b.xml", "turn-c.xml"] .iter() .enumerate() { let id = store .subscribe(busy, &meta(path, "Busy")) .await .unwrap() .podcast_id; // The busy account's podcasts are the most overdue ones. store .record_fetch_failure(busy, &id, now, "x", Some(now - 10_000 + index as i64)) .await .unwrap(); busy_ids.push(id); } let quiet_id = store .subscribe(quiet, &meta("turn-q.xml", "Quiet")) .await .unwrap() .podcast_id; store .record_fetch_failure(quiet, &quiet_id, now, "x", Some(now - 1)) .await .unwrap(); let due = |per_account| { let store = store.clone(); async move { store .podcasts_due_for_refresh(now, 100_000, per_account) .await .unwrap() .into_iter() .filter(|(user, ..)| [busy, quiet].contains(user)) .map(|(user, id, _)| (user, id)) .collect::>() } }; assert_eq!( due(2).await, vec![ (busy, busy_ids[0].clone()), (quiet, quiet_id.clone()), (busy, busy_ids[1].clone()), ], "one turn per account, at most two each" ); assert_eq!(due(usize::MAX).await.len(), 4); // New subscriptions stop at the per-account limit. sqlx::query( "INSERT INTO furumusic__podcast_subscription (user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id) SELECT $1, 'pod1:cap' || n, TRUE, '{}'::jsonb, 1, 1, 'x:' || n FROM generate_series(1, $2) AS n", ) .bind(quiet) .bind(i32::try_from(MAX_SUBSCRIPTIONS_PER_ACCOUNT - 1).unwrap()) .execute(&pool) .await .unwrap(); let error = store .subscribe(quiet, &meta("turn-over.xml", "Over")) .await .unwrap_err(); assert_eq!( error.downcast_ref::(), Some(&PodcastStoreError::SubscriptionLimit( MAX_SUBSCRIPTIONS_PER_ACCOUNT )) ); // Subscribing again to a podcast already subscribed is not new. assert!( store .subscribe(quiet, &meta("turn-q.xml", "Quiet")) .await .unwrap() .already_subscribed ); cleanup(&pool, [busy, quiet]).await; } /// Due groups of the given accounts only (other tests share the database). async fn due_for(store: &PodcastStore, users: &[i64], now: i64) -> runner::RefreshGroups { let due = store .podcasts_due_for_refresh(now, 100_000, usize::MAX) .await .unwrap() .into_iter() .filter(|(user_id, _, _)| users.contains(user_id)) .collect(); runner::group_targets(due) } async fn run_due( store: &PodcastStore, fetcher: &FeedFetcher, users: &[i64], now: i64, ) -> runner::RunSummary { let groups = due_for(store, users, now).await; runner::refresh_groups(store, fetcher, groups, now, 4) .await .unwrap() } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_refresh_job_schedules_backs_off_and_moves_feeds() { use super::runner::{BACKOFF_BASE_MS, ManualRefreshError, TargetResult}; use super::test_server::{Route, TestServer}; let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let (a, b, c) = (random_user(), random_user(), random_user()); cleanup(&pool, [a, b, c]).await; let users = [a, b, c]; let server = TestServer::start(); server.set( "/show.xml", Route::new(rss(&refresh_item("one")), "application/rss+xml").etag("\"v1\""), ); let show = PodcastMeta { feed_url: FeedUrl::parse(&server.url("/show.xml")).unwrap(), title: "Local Show".into(), author: None, description: None, link: None, image_url: None, }; let podcast_id = store.subscribe(a, &show).await.unwrap().podcast_id; assert_eq!( store.subscribe(b, &show).await.unwrap().podcast_id, podcast_id ); let fetcher = local_fetcher(); // First run: one request serves both accounts. let t0 = super::now_ms(); let summary = run_due(&store, &fetcher, &users, t0).await; assert_eq!((summary.targets, summary.feeds), (2, 1), "{summary}"); assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}"); assert_eq!(server.requests("/show.xml").len(), 1); for user in [a, b] { let state = store.fetch_state(user, &podcast_id).await.unwrap(); assert_eq!(state.etag.as_deref(), Some("\"v1\"")); assert_eq!(state.consecutive_failures, 0); let next = state.next_check_at_ms.unwrap(); assert!((t0 + runner::REFRESH_INTERVAL_MS..t0 + 70 * 60 * 1000).contains(&next)); } let episodes = store .episodes(a, &EpisodeQuery::new(podcast_id.clone())) .await .unwrap(); assert_eq!(episodes.len(), 1); let episode_id = episodes[0].episode.episode_id.clone(); assert_eq!(episodes[0].episode.audio_url, server.url("/audio/one.mp3")); // The proxy and the source endpoint only find episodes of the account. assert_eq!( store.audio_source(a, &episode_id).await.unwrap(), Some((server.url("/audio/one.mp3"), Some("audio/mpeg".to_string()))) ); assert_eq!(store.audio_source(c, &episode_id).await.unwrap(), None); // Nothing is due before the interval; afterwards the request is // conditional and the catalog stays as it is. assert!(due_for(&store, &users, t0 + 1000).await.is_empty()); let t1 = t0 + 2 * runner::REFRESH_INTERVAL_MS; let summary = run_due(&store, &fetcher, &users, t1).await; assert_eq!(summary.not_modified, 2, "{summary}"); assert_eq!(server.requests("/show.xml")[1]["if-none-match"], "\"v1\""); // Failures back off exponentially and keep the catalog. server.set("/show.xml", Route::new("boom", "text/plain").status(500)); let t2 = t1 + 2 * runner::REFRESH_INTERVAL_MS; let summary = run_due(&store, &fetcher, &users, t2).await; assert_eq!(summary.failed, 2, "{summary}"); assert_eq!(summary.failures.len(), 2); let state = store.fetch_state(a, &podcast_id).await.unwrap(); assert_eq!(state.consecutive_failures, 1); assert_eq!(state.next_check_at_ms, Some(t2 + BACKOFF_BASE_MS)); assert!(state.last_error.as_deref().unwrap().contains("500")); assert_eq!(state.etag.as_deref(), Some("\"v1\""), "validators are kept"); assert!( due_for(&store, &users, t2 + BACKOFF_BASE_MS - 1) .await .is_empty() ); let t3 = t2 + BACKOFF_BASE_MS; let summary = run_due(&store, &fetcher, &users, t3).await; assert_eq!(summary.failed, 2, "{summary}"); let state = store.fetch_state(a, &podcast_id).await.unwrap(); assert_eq!(state.consecutive_failures, 2); assert_eq!(state.next_check_at_ms, Some(t3 + 2 * BACKOFF_BASE_MS)); assert_eq!( store .episodes(a, &EpisodeQuery::new(podcast_id.clone())) .await .unwrap() .len(), 1 ); // Recovery resets the backoff. Accounts with different validators are // fetched unconditionally. server.set( "/show.xml", Route::new( rss(&format!("{}{}", refresh_item("two"), refresh_item("one"))), "application/rss+xml", ) .etag("\"v2\""), ); store .record_fetch_success(b, &podcast_id, t3, Some("\"other\""), None, Some(t3)) .await .unwrap(); let t4 = t3 + 2 * BACKOFF_BASE_MS; let summary = run_due(&store, &fetcher, &users, t4).await; assert_eq!((summary.updated, summary.new_episodes), (2, 2), "{summary}"); let last = server.requests("/show.xml").pop().unwrap(); assert!(!last.contains_key("if-none-match"), "{last:?}"); let state = store.fetch_state(a, &podcast_id).await.unwrap(); assert_eq!((state.consecutive_failures, state.last_error), (0, None)); assert_eq!(state.etag.as_deref(), Some("\"v2\"")); // Manual refresh: not more often than every 30 s, only when subscribed. let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 10_000).await; assert_eq!( manual.unwrap_err().downcast_ref::(), Some(&ManualRefreshError::TooSoon { retry_after_ms: 20_000 }) ); let manual = runner::refresh_now(&store, &fetcher, a, &podcast_id, t4 + 31_000) .await .unwrap(); assert_eq!(manual, TargetResult::NotModified); let unknown = runner::refresh_now(&store, &fetcher, c, &podcast_id, t4 + 31_000).await; assert_eq!( unknown.unwrap_err().downcast_ref::(), Some(&ManualRefreshError::UnknownPodcast) ); store.unsubscribe(b, &podcast_id).await.unwrap(); let unsubscribed = runner::refresh_now(&store, &fetcher, b, &podcast_id, t4 + 31_000).await; assert_eq!( unsubscribed .unwrap_err() .downcast_ref::(), Some(&ManualRefreshError::NotSubscribed) ); let far = t4 + 10 * runner::REFRESH_INTERVAL_MS; let due = due_for(&store, &users, far).await; assert_eq!( due.values().flatten().count(), 1, "unsubscribing stops monitoring" ); // A permanent move becomes an alias (one sync operation) and the new // fetch URL; later refreshes go to the new URL without new operations. server.set("/show.xml", Route::redirect(301, "/moved.xml")); server.set( "/moved.xml", Route::new(rss(&refresh_item("one")), "application/rss+xml"), ); let moved = FeedUrl::parse(&server.url("/moved.xml")).unwrap(); let result = runner::refresh_now(&store, &fetcher, a, &podcast_id, far) .await .unwrap(); let TargetResult::Updated(outcome) = result else { panic!("{result:?}"); }; assert_eq!(outcome.feed_url.as_ref(), Some(&moved)); assert_eq!(outcome.ops.len(), 1); assert_eq!( store .podcast(a, &podcast_id) .await .unwrap() .unwrap() .meta .feed_url, moved ); let due = due_for(&store, &users, far + 10 * runner::REFRESH_INTERVAL_MS).await; assert_eq!(due.keys().collect::>(), vec![&moved]); let summary = run_due( &store, &fetcher, &users, far + 10 * runner::REFRESH_INTERVAL_MS, ) .await; assert_eq!(summary.updated, 1, "{summary}"); assert_eq!(server.requests("/moved.xml").len(), 2); assert_eq!(server.requests("/show.xml").len(), 7); let TargetResult::Updated(again) = runner::refresh_now( &store, &fetcher, a, &podcast_id, far + 20 * runner::REFRESH_INTERVAL_MS, ) .await .unwrap() else { panic!("expected a document"); }; assert!(again.ops.is_empty()); // The episode keeps its id; its audio now resolves against the new URL // (same host here). let after_move = store.episode(a, &episode_id).await.unwrap().unwrap(); assert_eq!(after_move.episode.audio_url, server.url("/audio/one.mp3")); cleanup(&pool, [a, b, c]).await; } // --------------------------------------------------------------------------- // HTTP API of the podcast UI and the web player (`api`, behind `http`) // --------------------------------------------------------------------------- async fn logged_kinds(pool: &sqlx::PgPool, user_id: i64) -> Vec { sqlx::query_scalar("SELECT kind FROM furumusic__fed_sync_ops WHERE user_id = $1 ORDER BY seq") .bind(user_id) .fetch_all(pool) .await .unwrap() } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_api_subscribes_plays_and_keeps_progress() { use super::api::{self, EpisodeAction, EpisodeListParams}; use super::test_server::{Route, TestServer}; use serde_json::json; let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let (user, other) = (random_user(), random_user()); cleanup(&pool, [user, other]).await; let server = TestServer::start(); let items = r#" Pilots1e11 1600 Mon, 01 Jan 2024 10:00:00 GMTFirst interview Bonusbonus120 Tue, 02 Jan 2024 10:00:00 GMT "#; server.set("/show.xml", Route::new(rss(items), "application/rss+xml")); server.set("/page.html", Route::new("", "text/html")); server.set("/gone.xml", Route::new("gone", "text/plain").status(404)); let fetcher = local_fetcher(); let now = super::now_ms(); // "Add RSS": invalid input and unusable feeds subscribe nothing. for (body, status) in [ (json!({}), 400), (json!({"url": "ftp://example.org/feed"}), 400), (json!({"url": server.url("/page.html")}), 422), (json!({"url": server.url("/gone.xml")}), 422), ] { let error = api::subscribe_url(&store, &fetcher, user, body.clone(), now) .await .unwrap_err(); assert_eq!(error.status, status, "{body}: {}", error.message); } // The production policy never reaches the local server. let refused = api::subscribe_url( &store, &FeedFetcher::new().unwrap(), user, json!({"url": server.url("/show.xml")}), now, ) .await .unwrap_err(); assert_eq!(refused.status, 422, "{}", refused.message); assert_eq!( refused.message, "the feed could not be loaded: only public http and https addresses on ports 80, 443, \ 8080 and 8443 are allowed" ); assert!(logged_kinds(&pool, user).await.is_empty()); assert_eq!(server.requests("/show.xml").len(), 0); let added = api::subscribe_url( &store, &fetcher, user, json!({"url": format!(" {} ", server.url("/show.xml"))}), now, ) .await .unwrap(); assert!(!added.already_subscribed); assert_eq!(added.new_episodes, 2); let podcast = added.podcast; assert!(podcast.subscribed); assert!(!podcast.shared, "a new subscription is private"); assert_eq!((podcast.episode_count, podcast.unplayed_count), (2, 2)); assert_eq!(podcast.title, "Refresh Show"); assert!(podcast.fetch.last_success_at_ms.is_some()); assert!(podcast.fetch.next_check_at_ms.unwrap() > now); let podcast_id = PodcastId::parse(&podcast.podcast_id).unwrap(); assert_eq!( logged_kinds(&pool, user).await, ["podcast_subscription_set"] ); // Adding it again writes nothing. let again = api::subscribe_url( &store, &fetcher, user, json!({"url": server.url("/show.xml")}), now, ) .await .unwrap(); assert!(again.already_subscribed); assert_eq!(again.podcast.podcast_id, podcast.podcast_id); assert_eq!(logged_kinds(&pool, user).await.len(), 1); // Lists and details are per account. let list = api::list_podcasts(&store, user, false).await.unwrap(); assert_eq!(list.podcasts.len(), 1); assert!( api::list_podcasts(&store, other, true) .await .unwrap() .podcasts .is_empty() ); assert_eq!( api::podcast_detail(&store, other, &podcast_id) .await .unwrap_err() .status, 404 ); let detail = api::podcast_detail(&store, user, &podcast_id) .await .unwrap(); let seasons: Vec> = detail.seasons.iter().map(|season| season.season).collect(); assert_eq!(seasons, [Some(1), None]); assert!(detail.history.is_empty()); // Episodes: newest first, filters, search, paging, queue items. let page = api::episodes(&store, user, &podcast_id, &EpisodeListParams::default()) .await .unwrap(); let titles: Vec<&str> = page.episodes.iter().map(|e| e.title.as_str()).collect(); assert_eq!(titles, ["Bonus", "Pilot"]); assert!(!page.has_more); let pilot = &page.episodes[1]; assert_eq!((pilot.season, pilot.episode_number), (Some(1), Some(1))); assert_eq!(pilot.status, "not_started"); assert_eq!(pilot.track["source"]["kind"], "podcast_episode"); assert_eq!( pilot.track["source"]["episode_id"], pilot.episode_id.as_str() ); assert_eq!(pilot.track["unavailable"], false); assert_eq!(pilot.track["title"], "Pilot"); let first = api::episodes( &store, user, &podcast_id, &EpisodeListParams { limit: Some(1), ..Default::default() }, ) .await .unwrap(); assert_eq!((first.episodes.len(), first.has_more), (1, true)); let season = api::episodes( &store, user, &podcast_id, &EpisodeListParams { season: Some("none".into()), ..Default::default() }, ) .await .unwrap(); assert_eq!(season.episodes.len(), 1); assert_eq!(season.episodes[0].title, "Bonus"); let found = api::episodes( &store, user, &podcast_id, &EpisodeListParams { q: Some("INTERVIEW".into()), ..Default::default() }, ) .await .unwrap(); assert_eq!(found.episodes.len(), 1, "description search, any case"); let episode_id = EpisodeId::parse(&pilot.episode_id).unwrap(); // Playback: the source and the resume position, without writing. let prepared = api::playback(&store, user, &episode_id, json!({}), false) .await .unwrap(); assert_eq!(prepared.direct_url, server.url("/audio/s1e1.mp3")); assert_eq!(prepared.proxy_url, None); assert_eq!((prepared.position_ms, prepared.generation), (0, None)); assert_eq!(prepared.duration_ms, Some(600_000)); let proxied = api::playback(&store, user, &episode_id, json!({}), true) .await .unwrap(); assert_eq!( proxied.proxy_url.as_deref(), Some(format!("/api/podcasts/episodes/{episode_id}/audio").as_str()) ); assert_eq!(logged_kinds(&pool, user).await.len(), 1); assert_eq!( api::playback(&store, user, &episode_id, json!({"start": true}), false) .await .unwrap_err() .status, 400, "starting needs a session id" ); // Play starts a session (explicit generation 1). let started = api::playback( &store, user, &episode_id, json!({"start": true, "session_id": "web-s1"}), false, ) .await .unwrap(); assert_eq!(started.session_id.as_deref(), Some("web-s1")); assert_eq!((started.generation, started.position_ms), (Some(1), 0)); let report = |body: serde_json::Value| { let store = store.clone(); let episode_id = episode_id.clone(); async move { api::progress(&store, user, &episode_id, body, super::now_ms()).await } }; let t0 = super::now_ms() - 120_000; let saved = report(json!({ "session_id": "web-s1", "generation": 1, "event": "checkpoint", "position_ms": 60_000, "duration_ms": 600_000, })) .await .unwrap(); assert_eq!( (saved.outcome, saved.generation, saved.status), ("written", 1, "in_progress") ); assert!( !saved.session_recorded, "checkpoints do not record the session" ); // Pause records the session (at least 30 s listened). let paused = report(json!({ "session_id": "web-s1", "generation": 1, "event": "pause", "position_ms": 90_000, "listened_ms": 90_000, "started_at_ms": t0, "start_position_ms": 0, })) .await .unwrap(); assert!(paused.session_recorded); // Seeking back is an explicit action; a late checkpoint at the old // generation is superseded. let sought = report(json!({ "session_id": "web-s1", "generation": 1, "event": "seek", "position_ms": 30_000, })) .await .unwrap(); assert_eq!( (sought.outcome, sought.generation, sought.position_ms), ("written", 2, 30_000) ); let late = report(json!({ "session_id": "web-s1", "generation": 1, "event": "checkpoint", "position_ms": 95_000, })) .await .unwrap(); assert_eq!( (late.outcome, late.generation, late.position_ms), ("superseded", 2, 30_000) ); // Ordinary Play resumes from the register. let resumed = api::playback(&store, user, &episode_id, json!({}), false) .await .unwrap(); assert_eq!( (resumed.position_ms, resumed.status), (30_000, "in_progress") ); // Reaching the completion tail completes once, with the session. let near_end = report(json!({ "session_id": "web-s1", "generation": 2, "event": "checkpoint", "position_ms": 590_000, })) .await .unwrap(); assert_eq!( (near_end.outcome, near_end.status, near_end.completed), ("written", "completed", true) ); assert_eq!(near_end.generation, 3); let ended = report(json!({ "session_id": "web-s1", "generation": 3, "event": "ended", "position_ms": 600_000, "listened_ms": 400_000, "started_at_ms": t0, "start_position_ms": 0, })) .await .unwrap(); assert_eq!(ended.outcome, "already_completed"); assert!( ended.session_recorded, "the session grew and is now completed" ); let item = store.episode(user, &episode_id).await.unwrap().unwrap(); assert_eq!(item.completed_count, 1); // A completed episode plays from the start. let replay = api::playback(&store, user, &episode_id, json!({}), false) .await .unwrap(); assert_eq!((replay.position_ms, replay.status), (0, "completed")); let history = api::podcast_detail(&store, user, &podcast_id) .await .unwrap() .history; assert_eq!(history.len(), 1); assert!(history[0].completed); assert_eq!(history[0].episode_title.as_deref(), Some("Pilot")); // Explicit list actions. let restarted = api::episode_action(&store, user, &episode_id, EpisodeAction::Restart) .await .unwrap(); assert_eq!( (restarted.status, restarted.position_ms), ("in_progress", 0) ); let played = api::episode_action(&store, user, &episode_id, EpisodeAction::MarkPlayed) .await .unwrap(); assert_eq!(played.status, "completed"); assert_eq!( played.completed_count, 1, "marking played invents no session" ); let unplayed = api::episode_action(&store, user, &episode_id, EpisodeAction::MarkUnplayed) .await .unwrap(); assert_eq!((unplayed.status, unplayed.position_ms), ("not_started", 0)); assert_eq!( api::episode_action(&store, other, &episode_id, EpisodeAction::MarkPlayed) .await .unwrap_err() .status, 404 ); // A handed-off queue item the catalog does not know yet: the reference // is recorded and the session starts above the carried generation. let mut handed = test_episode(); handed.progress_generation = Some(9); let handed_id = handed.episode_id.clone(); assert_eq!( api::playback(&store, user, &handed_id, json!({}), false) .await .unwrap_err() .status, 404 ); let mut wrong = handed.clone(); wrong.episode_id = episode_id.clone(); assert_eq!( api::playback(&store, user, &handed_id, json!({"episode": wrong}), false) .await .unwrap_err() .status, 400 ); let taken_over = api::playback( &store, user, &handed_id, json!({ "episode": handed, "start": true, "session_id": "web-s2", "position_ms": 1_200_000, "carried_generation": 9, }), false, ) .await .unwrap(); assert_eq!( (taken_over.generation, taken_over.position_ms), (Some(10), 1_200_000) ); assert_eq!(taken_over.direct_url, "https://cdn.example/ep42.mp3"); let catalog = store.episode(user, &handed_id).await.unwrap().unwrap(); assert!(!catalog.in_feed); // A handoff away writes a final checkpoint at the session generation. let away = api::progress( &store, user, &handed_id, json!({ "session_id": "web-s2", "generation": 10, "event": "handoff", "position_ms": 1_260_000, "listened_ms": 10_000, "started_at_ms": super::now_ms() - 60_000, "start_position_ms": 1_200_000, }), super::now_ms(), ) .await .unwrap(); assert_eq!( (away.outcome, away.generation, away.position_ms), ("written", 10, 1_260_000) ); assert!( !away.session_recorded, "10 s is below the history threshold" ); // Unsubscribing keeps progress and history; subscribing again continues. let left = api::unsubscribe(&store, user, &podcast_id).await.unwrap(); assert!(!left.subscribed); assert!( api::list_podcasts(&store, user, false) .await .unwrap() .podcasts .is_empty() ); let all = api::list_podcasts(&store, user, true).await.unwrap(); assert!( all.podcasts .iter() .any(|p| p.podcast_id == podcast.podcast_id) ); let kept = store.episode(user, &episode_id).await.unwrap().unwrap(); assert_eq!(kept.completed_count, 1); let back = api::resubscribe(&store, user, &podcast_id).await.unwrap(); assert!(back.subscribed); assert_eq!( api::unsubscribe(&store, other, &podcast_id) .await .unwrap_err() .status, 404 ); // Every write went through the sync log, with its own operation kinds. let kinds = logged_kinds(&pool, user).await; let count = |kind: &str| kinds.iter().filter(|k| k.as_str() == kind).count(); assert_eq!(count("podcast_subscription_set"), 3); assert_eq!( count("podcast_share_set"), 1, "unsubscribing revokes sharing" ); assert_eq!(count("podcast_listen_recorded"), 2); // start, checkpoint, pause, seek, completion, restart, played, // unplayed, handed-off start, handoff checkpoint assert_eq!(count("podcast_progress_set"), 10); cleanup(&pool, [user, other]).await; } /// Ordinary Play continues from the progress register, not from a position /// the browser saved earlier: after a reload the web player prepares the /// episode without a position and gets what other devices wrote meanwhile. #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_playback_resumes_from_the_register() { use serde_json::json; let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user = random_user(); let episode = PodcastEpisodeRef { progress_generation: None, ..test_episode() }; let episode_id = episode.episode_id.clone(); let started = api::playback( &store, user, &episode_id, json!({"episode": episode, "start": true, "session_id": "web-s1", "position_ms": 600_000}), false, ) .await .unwrap(); assert_eq!( (started.generation, started.position_ms), (Some(1), 600_000) ); assert_eq!(started.register_generation, Some(1)); assert_eq!(started.register_session_id.as_deref(), Some("web-s1")); let paused = api::progress( &store, user, &episode_id, json!({"session_id": "web-s1", "generation": 1, "event": "pause", "position_ms": 620_000}), super::now_ms(), ) .await .unwrap(); assert_eq!(paused.outcome, "written"); let prepare = || api::playback(&store, user, &episode_id, json!({}), false); let mine = prepare().await.unwrap(); assert_eq!(mine.position_ms, 620_000); assert_eq!(mine.register_session_id.as_deref(), Some("web-s1")); // The phone listened further offline; its progress arrives later. store .apply_remote_op(user, &progress_op("phone", 1, 50, &episode, 2, 2_400_000)) .await .unwrap(); let moved = prepare().await.unwrap(); assert_eq!(moved.position_ms, 2_400_000, "the register, not 620 000"); assert_eq!(moved.register_generation, Some(2)); assert_eq!(moved.register_session_id.as_deref(), Some("phone-g2")); assert_eq!(moved.generation, None, "preparing writes nothing"); let resumed = api::playback( &store, user, &episode_id, json!({"start": true, "session_id": "web-s2"}), false, ) .await .unwrap(); assert_eq!( (resumed.generation, resumed.position_ms), (Some(3), 2_400_000) ); assert_eq!(resumed.register_session_id.as_deref(), Some("web-s2")); // Marked played elsewhere: Play starts over. let reference = store.episode_ref(user, &episode_id).await.unwrap().unwrap(); store.mark_played(user, &reference).await.unwrap(); let played = prepare().await.unwrap(); assert_eq!((played.status, played.position_ms), ("completed", 0)); cleanup(&pool, [user]).await; }