//! Podcast store against PostgreSQL. //! //! Ignored by default: set `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a //! disposable database (for example a throwaway `postgres:16-alpine` //! container). The tests migrate it with the production migrations, use //! random account ids and remove their rows again; the migration test //! creates and drops its own database next to it. use std::collections::BTreeSet; use cot::db::migrations::{MigrationEngine, SyncDynMigration}; use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot, random_hex}; use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; use music_dht::podcast::{ EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, }; use super::*; use crate::player::queue_items::test_episode; const DATABASE_URL: &str = "FURU_DEVICE_SYNC_TEST_DATABASE_URL"; /// Every production migration (the list of `admin::App::migrations`). fn all_migrations() -> Vec<&'static SyncDynMigration> { [ crate::config::db_migrations::MIGRATIONS, crate::user::db_migrations::MIGRATIONS, crate::music::db_migrations::MIGRATIONS, crate::scheduler::db_migrations::MIGRATIONS, crate::auth::db_migrations::MIGRATIONS, crate::youtube::db_migrations::MIGRATIONS, ] .into_iter() .flatten() .copied() .collect() } async fn migrate(url: &str, migrations: Vec<&'static SyncDynMigration>) { let database = cot::db::Database::new(url).await.unwrap(); MigrationEngine::new(migrations) .unwrap() .run(&database) .await .unwrap(); database.close().await.unwrap(); } /// Pool on the test database, migrated once per test binary with the /// production migrations (also used by the device-sync Postgres tests). pub(crate) async fn test_pool() -> sqlx::PgPool { static MIGRATED: tokio::sync::Mutex = tokio::sync::Mutex::const_new(false); let url = std::env::var(DATABASE_URL).unwrap(); let mut migrated = MIGRATED.lock().await; if !*migrated { migrate(&url, all_migrations()).await; *migrated = true; } sqlx::PgPool::connect(&url).await.unwrap() } fn random_user() -> i64 { let bytes = blake3::hash(random_hex(8).as_bytes()); 9_000_000_000 + i64::from(u32::from_le_bytes(bytes.as_bytes()[..4].try_into().unwrap()) % 1_000_000_000) } const ACCOUNT_TABLES: &[&str] = &[ "furumusic__podcast_fetch_state", "furumusic__podcast_listen_session", "furumusic__podcast_progress", "furumusic__podcast_share", "furumusic__podcast_subscription", "furumusic__podcast_episode", "furumusic__podcast_feed_alias", "furumusic__podcast", "furumusic__fed_podcast_relay", "furumusic__fed_device_identity", "furumusic__fed_device", "furumusic__fed_sync_ops", "furumusic__fed_sync_vector", "furumusic__fed_state_like", "furumusic__fed_state_playlist", "furumusic__fed_state_playlist_item", ]; async fn cleanup(pool: &sqlx::PgPool, users: impl IntoIterator) { let users: Vec = users.into_iter().collect(); for table in ACCOUNT_TABLES { sqlx::query(&format!("DELETE FROM {table} WHERE user_id = ANY($1)")) .bind(&users) .execute(pool) .await .unwrap(); } } fn feed_url(path: &str) -> FeedUrl { FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap() } fn meta(path: &str, title: &str) -> PodcastMeta { PodcastMeta { feed_url: feed_url(path), title: title.into(), author: Some("Host".into()), description: Some("Channel description".into()), link: None, image_url: None, } } fn op(origin: &str, seq: i64, hlc_ms: i64, payload: SyncOpPayload) -> SyncOpWire { SyncOpWire { op_id: format!("{origin}:{seq}"), origin_device_id: origin.into(), seq, hlc_ms, payload, } } fn session( id: &str, device: &str, episode: &PodcastEpisodeRef, listened_ms: i64, completed: bool, ) -> ListenSession { ListenSession { session_id: id.into(), episode_id: episode.episode_id.clone(), podcast_id: episode.podcast_id.clone(), device_id: device.into(), started_at_ms: 1_000, ended_at_ms: 1_000 + listened_ms, start_position_ms: 0, end_position_ms: listened_ms, listened_ms, completed, } } fn progress_op( origin: &str, seq: i64, hlc_ms: i64, episode: &PodcastEpisodeRef, generation: u64, position_ms: i64, ) -> SyncOpWire { op( origin, seq, hlc_ms, SyncOpPayload::PodcastProgressSet { episode_id: episode.episode_id.clone(), podcast_id: episode.podcast_id.clone(), position_ms, status: ProgressStatus::InProgress, generation, session_id: format!("{origin}-g{generation}"), episode: Some(PodcastEpisodeRef { progress_generation: None, ..episode.clone() }), }, ) } /// Every permutation of `0..n` (Heap's algorithm). fn permutations(n: usize) -> Vec> { fn heap(k: usize, items: &mut Vec, out: &mut Vec>) { if k <= 1 { out.push(items.clone()); return; } for i in 0..k { heap(k - 1, items, out); let j = if k.is_multiple_of(2) { i } else { 0 }; items.swap(j, k - 1); } } let mut out = Vec::new(); heap(n, &mut (0..n).collect(), &mut out); out } async fn apply_all(store: &PodcastStore, user_id: i64, ops: &[SyncOpWire]) { for op in ops { let outcome = store.apply_remote_op(user_id, op).await.unwrap(); assert!( matches!(outcome, ApplyOutcome::Changed | ApplyOutcome::Unchanged), "{outcome:?}" ); } } /// The registers and sessions of one podcast and episode from three /// devices, with a duplicate, a late checkpoint of an older session, a /// concurrent offline action and a growing session. fn convergence_ops() -> Vec { let episode = test_episode(); let podcast_id = episode.podcast_id.clone(); let podcast = PodcastMeta { feed_url: episode.feed_url.clone(), title: episode.podcast_title.clone(), author: None, description: Some("dropped from snapshots".into()), link: None, image_url: None, }; let subscribe = op( "tui_a", 1, 100, SyncOpPayload::PodcastSubscriptionSet { podcast_id: podcast_id.clone(), subscribed: true, podcast: podcast.clone(), feed_aliases: vec![], generation: 1, }, ); vec![ subscribe.clone(), // B saw g1 and unsubscribes with a slow clock: g2 still wins. op( "tui_b", 1, 50, SyncOpPayload::PodcastSubscriptionSet { podcast_id: podcast_id.clone(), subscribed: false, podcast, feed_aliases: vec![], generation: 2, }, ), // A shared concurrently at g2: stored, but not effective. op( "tui_a", 2, 300, SyncOpPayload::PodcastShareSet { podcast_id: podcast_id.clone(), shared: true, generation: 2, }, ), op( "tui_c", 1, 400, SyncOpPayload::PodcastFeedAliasesAdded { podcast_id, feed_aliases: vec![feed_url("moved.xml")], }, ), subscribe, // A late g3 checkpoint of an old session with a fast clock. progress_op("tui_a", 3, 9_000, &episode, 3, 600_000), // B and C both seek at g4 with the same clock: device id decides. progress_op("tui_b", 2, 1_000, &episode, 4, 10_000), progress_op("tui_c", 2, 1_000, &episode, 4, 20_000), op( "tui_a", 4, 9_100, SyncOpPayload::PodcastListenRecorded { session: session("s1", "tui_a", &episode, 40_000, false), episode: None, }, ), op( "tui_a", 5, 9_200, SyncOpPayload::PodcastListenRecorded { session: session("s1", "tui_a", &episode, 90_000, true), episode: None, }, ), op( "tui_b", 3, 1_100, SyncOpPayload::PodcastListenRecorded { session: session("s2", "tui_b", &episode, 31_000, false), episode: Some(PodcastEpisodeRef { progress_generation: None, ..episode }), }, ), ] } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_ops_converge_in_any_order_and_match_snapshot_import() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let ops = convergence_ops(); let reference_user = random_user(); let mut users = vec![reference_user]; apply_all(&store, reference_user, &ops).await; let reference = store.export_snapshot(reference_user).await.unwrap(); // The contract outcome, independently of the order. let episode = test_episode(); assert_eq!(reference.podcast_subscriptions.len(), 1); let subscription = &reference.podcast_subscriptions[0]; assert!(!subscription.subscribed); assert_eq!(subscription.generation, 2); assert!(subscription.podcast.description.is_none()); assert_eq!( subscription.feed_urls(), vec![episode.feed_url.clone(), feed_url("moved.xml")] ); assert!(reference.podcast_shares[0].shared); let summary = store .podcast(reference_user, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(!summary.subscribed && !summary.shared); assert_eq!(reference.podcast_progress.len(), 1); assert_eq!(reference.podcast_progress[0].position_ms, 20_000); assert_eq!(reference.podcast_progress[0].device_id, "tui_c"); let sessions: BTreeSet<_> = reference .podcast_listen_sessions .iter() .map(|session| { ( session.session_id.as_str(), session.listened_ms, session.completed, ) }) .collect(); assert_eq!( sessions, BTreeSet::from([("s1", 90_000, true), ("s2", 31_000, false)]) ); assert_eq!( store .completed_listen_count(reference_user, &episode.episode_id) .await .unwrap(), 1 ); // The episode reference created a catalog row (never touching progress). let item = store .episode(reference_user, &episode.episode_id) .await .unwrap() .unwrap(); assert!(!item.in_feed); assert_eq!(item.resume_position_ms(), 20_000); // Register operations: all 120 orders of the first five operations // (subscription, unsubscription, share, aliases, duplicate) with the // rest appended; progress and session operations: all 720 orders of // the last six with the first five prepended. let (registers, personal) = ops.split_at(5); for order in permutations(registers.len()) { let user_id = random_user(); users.push(user_id); let mut shuffled: Vec = order.iter().map(|&i| registers[i].clone()).collect(); shuffled.extend_from_slice(personal); apply_all(&store, user_id, &shuffled).await; assert_eq!( store.export_snapshot(user_id).await.unwrap(), reference, "{order:?}" ); } for order in permutations(personal.len()) { let user_id = random_user(); users.push(user_id); let mut shuffled = registers.to_vec(); shuffled.extend(order.iter().map(|&i| personal[i].clone())); apply_all(&store, user_id, &shuffled).await; assert_eq!( store.export_snapshot(user_id).await.unwrap(), reference, "{order:?}" ); } // A new device recovering from the snapshot reaches the same state, and // the operations delivered afterwards change nothing. (Snapshots omit // the channel description, so the winning subscription operation only // adds it back to the stored register: the contract's tie-break by // metadata. Every other operation is already reflected.) let recovered = random_user(); users.push(recovered); let stats = store.apply_snapshot(recovered, &reference).await.unwrap(); assert_eq!(stats.rejected, 0); assert!(!stats.ignored_over_limits); assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); for op in ops.iter().rev() { let expected = if op.op_id == "tui_b:1" { ApplyOutcome::Changed } else { ApplyOutcome::Unchanged }; assert_eq!( store.apply_remote_op(recovered, op).await.unwrap(), expected, "{}", op.op_id ); } assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); // A stale snapshot never replaces newer state. let stale = SyncSnapshot { podcast_progress: vec![ProgressRecord { position_ms: 1, generation: 3, hlc_ms: i64::MAX, ..reference.podcast_progress[0].clone() }], ..SyncSnapshot::default() }; let stats = store.apply_snapshot(recovered, &stale).await.unwrap(); assert_eq!((stats.changed, stats.unchanged), (0, 1)); assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); // Invalid operations are rejected without touching state. let mut invalid = progress_op("tui_a", 9, 1, &episode, 9, 1); if let SyncOpPayload::PodcastProgressSet { session_id, .. } = &mut invalid.payload { session_id.clear(); } assert!(matches!( store.apply_remote_op(recovered, &invalid).await.unwrap(), ApplyOutcome::Rejected(_) )); let mut huge = progress_op("tui_a", 10, 1, &episode, u64::MAX, 1); huge.op_id = "tui_a:10".into(); assert!(matches!( store.apply_remote_op(recovered, &huge).await.unwrap(), ApplyOutcome::Rejected(_) )); assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference); cleanup(&pool, users).await; } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_local_actions_keep_progress_through_unsubscribe() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user_id = random_user(); let device = crate::federation::devices::local_device_id(&pool, user_id) .await .unwrap(); let episode = test_episode(); let show = PodcastMeta { feed_url: episode.feed_url.clone(), ..meta("unused.xml", &episode.podcast_title) }; let subscribed = store.subscribe(user_id, &show).await.unwrap(); assert_eq!(subscribed.podcast_id, episode.podcast_id); assert!(!subscribed.already_subscribed); let [first] = subscribed.ops.as_slice() else { panic!("one operation"); }; assert_eq!(first.origin_device_id, device); assert_eq!(first.op_id, format!("{device}:{}", first.seq)); // The operation is in the log with exactly this stamp; applying it again // is a no-op. let logged: serde_json::Value = sqlx::query_scalar( "SELECT payload_json FROM furumusic__fed_sync_ops WHERE user_id = $1 AND op_id = $2 AND hlc_ms = $3 AND kind = 'podcast_subscription_set'", ) .bind(user_id) .bind(&first.op_id) .bind(first.hlc_ms) .fetch_one(&pool) .await .unwrap(); assert_eq!(logged, serde_json::to_value(&first.payload).unwrap()); assert_eq!( store.apply_remote_op(user_id, first).await.unwrap(), ApplyOutcome::Unchanged ); assert!( store .subscribe(user_id, &show) .await .unwrap() .already_subscribed ); assert_eq!( store .set_shared(user_id, &episode.podcast_id, true) .await .unwrap() .len(), 1 ); assert!( store .set_shared(user_id, &episode.podcast_id, true) .await .unwrap() .is_empty() ); assert!( store .podcast(user_id, &episode.podcast_id) .await .unwrap() .unwrap() .shared ); // Listening: an explicit start, checkpoints keep the generation. let start = store .start_session(user_id, &episode, "web-s1", 0, episode.progress_generation) .await .unwrap(); assert_eq!(start.record.generation, 8, "max(local, carried 7) + 1"); let CheckpointOutcome::Written(checkpoint) = store .checkpoint(user_id, &start.session, 30_000) .await .unwrap() else { panic!("checkpoint written"); }; assert_eq!(checkpoint.record.generation, 8); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 30_000 ); // A seek is explicit: later checkpoints use its generation. let seek = store .seek(user_id, &checkpoint.session, 12_000) .await .unwrap(); assert_eq!(seek.record.generation, 9); assert_eq!(seek.session.generation, 9); let start = seek; // Another device seeks back at a greater generation with a slow clock: // later checkpoints of the old session are refused. store .apply_remote_op(user_id, &progress_op("tui_b", 1, 1, &episode, 10, 5_000)) .await .unwrap(); assert!(matches!( store .checkpoint(user_id, &start.session, 900_000) .await .unwrap(), CheckpointOutcome::Superseded { .. } )); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 5_000 ); let listened = ListenSession { device_id: device.clone(), ..session("web-s1", &device, &episode, 45_000, false) }; assert!( store .record_listen_session(user_id, &listened, None) .await .unwrap() .is_some() ); assert!( store .record_listen_session(user_id, &listened, None) .await .unwrap() .is_none() ); assert!( store .record_listen_session( user_id, &session("x", "tui_b", &episode, 45_000, false), None ) .await .is_err() ); // Unsubscribing keeps progress and history and revokes the share. let ops = store .unsubscribe(user_id, &episode.podcast_id) .await .unwrap(); assert_eq!(ops.len(), 2); assert!( store .unsubscribe(user_id, &episode.podcast_id) .await .unwrap() .is_empty() ); let summary = store .podcast(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(!summary.subscribed && !summary.shared); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 5_000 ); assert_eq!( store.listen_history(user_id, None, 10).await.unwrap().len(), 1 ); assert!( store .list_podcasts(user_id, PodcastListFilter::Subscribed) .await .unwrap() .is_empty() ); assert_eq!( store .list_podcasts(user_id, PodcastListFilter::All) .await .unwrap() .len(), 1 ); // Subscribing again resolves to the same podcast, continues the // progress and is private again. let again = store.subscribe(user_id, &show).await.unwrap(); assert_eq!(again.podcast_id, episode.podcast_id); let register = store .subscription(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); let share = store .share(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(register.subscribed); assert!(register.generation > share.generation); let summary = store .podcast(user_id, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(summary.subscribed && !summary.shared); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 5_000 ); // Mark played writes progress only; restart and unplayed reset it. let played = store.mark_played(user_id, &episode).await.unwrap(); assert_eq!(played.record.status, ProgressStatus::Completed); assert_eq!(played.record.generation, 11); assert_eq!( store.listen_history(user_id, None, 10).await.unwrap().len(), 1 ); assert_eq!( store .resume_position_ms(user_id, &episode.episode_id) .await .unwrap(), 0 ); let restarted = store.restart(user_id, &episode, "web-s2").await.unwrap(); assert_eq!( ( restarted.record.position_ms, restarted.record.status, restarted.record.generation ), (0, ProgressStatus::InProgress, 12) ); let finished = store .complete_session(user_id, &restarted.session, 1_790_000) .await .unwrap(); assert_eq!( (finished.record.status, finished.record.generation), (ProgressStatus::Completed, 13) ); assert!(matches!( store .checkpoint(user_id, &finished.session, 1_795_000) .await .unwrap(), CheckpointOutcome::AlreadyCompleted { .. } )); let mut played_filter = EpisodeQuery::new(episode.podcast_id.clone()); played_filter.played = PlayedFilter::Played; assert_eq!( store.episodes(user_id, &played_filter).await.unwrap().len(), 1 ); played_filter.played = PlayedFilter::Unplayed; assert!( store .episodes(user_id, &played_filter) .await .unwrap() .is_empty() ); assert_eq!( store .mark_unplayed(user_id, &episode) .await .unwrap() .record .status, ProgressStatus::NotStarted ); played_filter.played = PlayedFilter::NotStarted; let items = store.episodes(user_id, &played_filter).await.unwrap(); assert_eq!(items[0].status(), ProgressStatus::NotStarted); // Local stamps: consecutive sequence numbers, strictly growing clock. let stamps: Vec<(i64, i64)> = sqlx::query_as( "SELECT seq, hlc_ms FROM furumusic__fed_sync_ops WHERE user_id = $1 AND origin_device_id = $2 ORDER BY seq", ) .bind(user_id) .bind(&device) .fetch_all(&pool) .await .unwrap(); assert!( stamps .windows(2) .all(|pair| pair[1].0 == pair[0].0 + 1 && pair[1].1 > pair[0].1) ); // Re-applying the whole local log changes nothing. let before = store.export_snapshot(user_id).await.unwrap(); let payloads: Vec<(String, i64, i64, serde_json::Value)> = sqlx::query_as( "SELECT op_id, seq, hlc_ms, payload_json FROM furumusic__fed_sync_ops WHERE user_id = $1 AND kind LIKE 'podcast\\_%' ORDER BY hlc_ms", ) .bind(user_id) .fetch_all(&pool) .await .unwrap(); for (op_id, seq, hlc_ms, payload) in payloads { let origin = op_id.rsplit_once(':').unwrap().0.to_string(); let op = SyncOpWire { op_id, origin_device_id: origin, seq, hlc_ms, payload: serde_json::from_value(payload).unwrap(), }; assert_eq!( store.apply_remote_op(user_id, &op).await.unwrap(), ApplyOutcome::Unchanged ); } assert_eq!(store.export_snapshot(user_id).await.unwrap(), before); // Sharing needs a subscription. let unknown = PodcastId::from_feed_url(&feed_url("unknown.xml")); let error = store.set_shared(user_id, &unknown, true).await.unwrap_err(); assert!(matches!( error.downcast_ref::(), Some(PodcastStoreError::NotSubscribed(_)) )); cleanup(&pool, [user_id]).await; } fn rss(items: &str) -> String { format!( r#" Refresh Show https://show.example/ Channel text Host {items} "# ) } fn parse(xml: &str, url: &FeedUrl) -> ParsedFeed { parse_feed_with_base(xml.as_bytes(), Some(url.as_str())).unwrap() } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_feed_refresh_keeps_ids_and_progress() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user_id = random_user(); let url = feed_url("refresh.xml"); let podcast_id = PodcastId::from_feed_url(&url); let v1 = parse( &rss(r#" g-1One1 1Mon, 01 Jan 2024 10:00:00 GMT Twohttps://show.example/ep2 12 Mon, 08 Jan 2024 10:00:00 GMT Talk about Ünïcode ThreeMon, 15 Jan 2024 10:00:00 GMT "#), &url, ); // What every device derives for this document. let ids: Vec = v1 .resolve_episode_ids(&podcast_id, &[]) .into_iter() .map(Option::unwrap) .collect(); // Another device played the GUID-less "Two" before this account ever // fetched the feed: its reference arrives with the progress. let two = PodcastEpisodeRef { podcast_id: podcast_id.clone(), episode_id: ids[1].clone(), feed_url: url.clone(), podcast_title: "Refresh Show".into(), episode_title: "Two".into(), audio_url: "https://cdn.example/a2.mp3".into(), mime: None, duration_ms: None, published_at_ms: None, season: None, episode_number: None, image_url: None, progress_generation: None, }; store .apply_remote_op(user_id, &progress_op("tui_a", 1, 10, &two, 3, 61_000)) .await .unwrap(); let subscribed = store .subscribe(user_id, &v1.to_podcast_meta(url.clone())) .await .unwrap(); assert_eq!(subscribed.podcast_id, podcast_id); let first = store .apply_feed( user_id, &FeedRefresh { podcast_id: &podcast_id, feed: &v1, fetched_from: &url, moved_permanently: false, }, ) .await .unwrap(); assert_eq!((first.new_episodes, first.updated_episodes), (2, 1)); assert!(first.ops.is_empty()); let stored: BTreeSet = store .episodes(user_id, &EpisodeQuery::new(podcast_id.clone())) .await .unwrap() .into_iter() .map(|item| item.episode.episode_id) .collect(); assert_eq!(stored, ids.iter().cloned().collect()); let progress_before = store.progress(user_id, &ids[1]).await.unwrap().unwrap(); // The publisher renames episodes, moves their audio, drops "Three", // adds "Four" and moves the feed permanently. let moved = feed_url("moved-refresh.xml"); let v2 = parse( &rss(r#" g-1One (remastered)1 1Mon, 01 Jan 2024 10:00:00 GMT Two, revisedhttps://show.example/ep2 12 Mon, 08 Jan 2024 10:00:00 GMT Talk about Ünïcode g-4Four2 1Mon, 22 Jan 2024 10:00:00 GMT "#), &moved, ); let refresh = FeedRefresh { podcast_id: &podcast_id, feed: &v2, fetched_from: &moved, moved_permanently: true, }; let second = store.apply_feed(user_id, &refresh).await.unwrap(); assert_eq!( ( second.new_episodes, second.updated_episodes, second.unchanged_episodes ), (1, 2, 0) ); assert_eq!(second.ops.len(), 1, "aliases changed once"); assert_eq!(second.feed_url.as_ref(), Some(&moved)); let again = store.apply_feed(user_id, &refresh).await.unwrap(); assert!(again.ops.is_empty()); assert_eq!(again.unchanged_episodes, 3); assert_eq!( store.resolve_podcast_id(user_id, &moved).await.unwrap(), podcast_id ); assert_eq!( store.resolve_podcast_id(user_id, &url).await.unwrap(), podcast_id ); // Ids and progress survive; the dropped episode stays out of the feed. let one = store.episode(user_id, &ids[0]).await.unwrap().unwrap(); assert_eq!(one.episode.title, "One (remastered)"); assert_eq!(one.episode.audio_url, "https://cdn2.example/a1.mp3"); let two_item = store.episode(user_id, &ids[1]).await.unwrap().unwrap(); assert_eq!(two_item.episode.audio_url, "https://cdn2.example/a2.mp3"); assert_eq!(two_item.progress.as_ref(), Some(&progress_before)); assert_eq!(two_item.resume_position_ms(), 61_000); let three = store.episode(user_id, &ids[2]).await.unwrap().unwrap(); assert!(!three.in_feed); // The subscription register is untouched by refreshes. let register = store .subscription(user_id, &podcast_id) .await .unwrap() .unwrap(); assert_eq!(register.op_id, subscribed.ops[0].op_id); assert!(register.feed_urls().contains(&moved)); let reference = store.episode_ref(user_id, &ids[1]).await.unwrap().unwrap(); assert_eq!(reference.progress_generation, Some(3)); assert_eq!(reference.audio_url, "https://cdn2.example/a2.mp3"); // UI queries: counters, seasons, filters, search and sorting. let summary = store.podcast(user_id, &podcast_id).await.unwrap().unwrap(); assert_eq!( ( summary.episode_count, summary.unplayed_count, summary.in_progress_count, summary.season_count ), (4, 4, 1, 2) ); assert_eq!(summary.meta.feed_url, moved); let seasons = store.seasons(user_id, &podcast_id).await.unwrap(); assert_eq!( seasons .iter() .map(|season| (season.season, season.episode_count)) .collect::>(), vec![(Some(1), 2), (Some(2), 1), (None, 1)] ); let query = |configure: &dyn Fn(&mut EpisodeQuery)| { let mut query = EpisodeQuery::new(podcast_id.clone()); configure(&mut query); query }; let titles = |items: Vec| { items .into_iter() .map(|item| item.episode.title) .collect::>() }; assert_eq!( titles(store.episodes(user_id, &query(&|_| {})).await.unwrap()), ["Four", "Three", "Two, revised", "One (remastered)"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.sort = EpisodeSort::EpisodeNumber)) .await .unwrap() ), ["One (remastered)", "Two, revised", "Four", "Three"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.season = SeasonFilter::Season(1))) .await .unwrap() ), ["Two, revised", "One (remastered)"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.season = SeasonFilter::NoSeason)) .await .unwrap() ), ["Three"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.played = PlayedFilter::InProgress)) .await .unwrap() ), ["Two, revised"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.search = Some("ÜNÏCODE".into()))) .await .unwrap() ), ["Two, revised"] ); assert_eq!( titles( store .episodes(user_id, &query(&|q| q.search = Some("remaster".into()))) .await .unwrap() ), ["One (remastered)"] ); assert_eq!( titles( store .episodes( user_id, &query(&|q| { q.limit = Some(1); q.offset = 1; }) ) .await .unwrap() ), ["Three"] ); assert!( store .episodes(user_id, &query(&|q| q.search = Some("one two".into()))) .await .unwrap() .is_empty() ); // A refresh of a podcast this account does not know is refused. let unknown = PodcastId::from_feed_url(&feed_url("never.xml")); let error = store .apply_feed( user_id, &FeedRefresh { podcast_id: &unknown, ..refresh }, ) .await .unwrap_err(); assert!(matches!( error.downcast_ref::(), Some(PodcastStoreError::UnknownPodcast(_)) )); cleanup(&pool, [user_id]).await; } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_fetch_state_is_local_only() { let pool = test_pool().await; let store = PodcastStore::new(pool.clone()); let user_id = random_user(); let due_show = meta("due.xml", "Due"); let later_show = meta("later.xml", "Later"); let gone_show = meta("gone.xml", "Gone"); let mut ids = Vec::new(); for show in [&due_show, &later_show, &gone_show] { ids.push(store.subscribe(user_id, show).await.unwrap().podcast_id); } store.unsubscribe(user_id, &ids[2]).await.unwrap(); let before = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap(); let now = 1_800_000_000_000; store .record_fetch_success( user_id, &ids[1], now, Some("\"secret-etag\""), Some("Tue"), Some(now + 60_000), ) .await .unwrap(); assert_eq!( store .record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1)) .await .unwrap(), 1 ); assert_eq!( store .record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1)) .await .unwrap(), 2 ); store .set_cover_cache( user_id, &ids[0], Some("https://img.example/c.jpg"), Some("/var/cache/cover-secret.jpg"), ) .await .unwrap(); let state = store.fetch_state(user_id, &ids[0]).await.unwrap(); assert_eq!(state.consecutive_failures, 2); assert_eq!(state.last_error.as_deref(), Some("boom: 503")); assert_eq!( state.cover_path.as_deref(), Some("/var/cache/cover-secret.jpg") ); store .record_fetch_not_modified(user_id, &ids[1], now + 1, Some(now + 60_000)) .await .unwrap(); let later = store.fetch_state(user_id, &ids[1]).await.unwrap(); assert_eq!(later.etag.as_deref(), Some("\"secret-etag\"")); assert_eq!(later.last_success_at_ms, Some(now + 1)); // Fetch state changes nothing in the synced state. let after = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap(); assert_eq!(after, before); for secret in ["secret-etag", "boom", "cover-secret", "Channel description"] { assert!(!after.contains(secret), "{secret}"); } let ops: Vec = sqlx::query_scalar( "SELECT payload_json::text FROM furumusic__fed_sync_ops WHERE user_id = $1", ) .bind(user_id) .fetch_all(&pool) .await .unwrap(); assert!( ops.iter() .all(|op| !op.contains("secret-etag") && !op.contains("cover-secret")) ); // Scheduling: overdue first, future and unsubscribed podcasts skipped. let due: Vec = store .podcasts_due_for_refresh(now, 10_000) .await .unwrap() .into_iter() .filter(|(user, ..)| *user == user_id) .map(|(_, id, _)| id) .collect(); assert_eq!(due, vec![ids[0].clone()]); let unknown = PodcastId::from_feed_url(&feed_url("unknown-fetch.xml")); assert!( store .record_fetch_failure(user_id, &unknown, now, "x", None) .await .is_err() ); cleanup(&pool, [user_id]).await; } /// Relay rows exactly as 0.13.0 stored them (`podcast_relay`): the contract /// records as JSON, keyed by section and entity. async fn insert_relay( pool: &sqlx::PgPool, user_id: i64, section: &str, key: &str, value: serde_json::Value, ) { sqlx::query( "INSERT INTO furumusic__fed_podcast_relay (user_id, section, record_key, record_json, updated_at_ms) VALUES ($1, $2, $3, $4, 1700000000000)", ) .bind(user_id) .bind(section) .bind(key) .bind(value) .execute(pool) .await .unwrap(); } async fn insert_logged_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire, kind: &str) { sqlx::query( "INSERT INTO furumusic__fed_sync_ops (user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms, received_at_ms, tombstone) VALUES ($1, $2, $3, $4, $5, $6, $7, 1700000000000, FALSE)", ) .bind(user_id) .bind(&op.op_id) .bind(&op.origin_device_id) .bind(op.seq) .bind(kind) .bind(serde_json::to_value(&op.payload).unwrap()) .bind(op.hlc_ms) .execute(pool) .await .unwrap(); } #[tokio::test] #[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"] async fn postgres_podcast_migrations_import_the_relay_state() { let admin_url = std::env::var(DATABASE_URL).unwrap(); let admin = sqlx::PgPool::connect(&admin_url).await.unwrap(); let name = format!("furu_podcast_migration_{}", random_hex(6)); sqlx::query(&format!("CREATE DATABASE {name}")) .execute(&admin) .await .unwrap(); let (base, _) = admin_url.rsplit_once('/').unwrap(); let url = format!("{base}/{name}"); // A 0.13.0 database: every migration up to the relay table. let new_migrations = ["m_0056_create_podcast_store", "m_0057_import_podcast_relay"]; let old: Vec<_> = all_migrations() .into_iter() .filter(|migration| { !(migration.app_name() == "furumusic" && new_migrations.contains(&migration.name())) }) .collect(); migrate(&url, old).await; let pool = sqlx::PgPool::connect(&url).await.unwrap(); // State relayed by 0.13.0 for two accounts. let episode = test_episode(); let user_a = 1; let user_b = 2; let subscription = SubscriptionRecord { podcast_id: episode.podcast_id.clone(), subscribed: true, podcast: PodcastMeta { feed_url: episode.feed_url.clone(), title: episode.podcast_title.clone(), author: Some("Host".into()), description: None, link: None, image_url: Some("https://img.example/show.jpg".into()), }, feed_aliases: vec![episode.feed_url.clone(), feed_url("old.xml")], generation: 3, hlc_ms: 30, op_id: "tui_a:3".into(), }; let share = ShareRecord { podcast_id: episode.podcast_id.clone(), shared: false, generation: 2, hlc_ms: 20, op_id: "tui_a:2".into(), }; let progress = ProgressRecord { episode_id: episode.episode_id.clone(), podcast_id: episode.podcast_id.clone(), position_ms: 1_200_000, status: ProgressStatus::InProgress, generation: 9, session_id: "tui_a-g9".into(), hlc_ms: 90, device_id: "tui_a".into(), op_id: "tui_a:9".into(), }; let listened = session("s9", "tui_a", &episode, 120_000, true); let pending_podcast = PodcastId::from_feed_url(&feed_url("pending.xml")); insert_relay( &pool, user_a, "subscription", subscription.podcast_id.as_str(), serde_json::to_value(&subscription).unwrap(), ) .await; insert_relay( &pool, user_a, "share", share.podcast_id.as_str(), serde_json::to_value(&share).unwrap(), ) .await; insert_relay( &pool, user_a, "progress", progress.episode_id.as_str(), serde_json::to_value(&progress).unwrap(), ) .await; insert_relay( &pool, user_a, "session", "s9", serde_json::to_value(&listened).unwrap(), ) .await; insert_relay( &pool, user_a, "aliases", pending_podcast.as_str(), serde_json::to_value(vec![feed_url("pending.xml")]).unwrap(), ) .await; // An out-of-range generation (no conforming client) is skipped. let huge = ProgressRecord { episode_id: EpisodeId::from_guid(&episode.podcast_id, "huge").unwrap(), generation: u64::MAX, ..progress.clone() }; insert_relay( &pool, user_a, "progress", huge.episode_id.as_str(), serde_json::to_value(&huge).unwrap(), ) .await; insert_relay( &pool, user_b, "progress", progress.episode_id.as_str(), serde_json::to_value(&ProgressRecord { position_ms: 5, ..progress.clone() }) .unwrap(), ) .await; // The relayed operations kept the episode references (two versions). let old_ref = PodcastEpisodeRef { episode_title: "Old title".into(), progress_generation: None, ..episode.clone() }; insert_logged_op( &pool, user_a, &progress_op("tui_a", 8, 80, &old_ref, 8, 1_000), "podcast_progress_set", ) .await; insert_logged_op( &pool, user_a, &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000), "podcast_progress_set", ) .await; // Upgrade. migrate(&url, all_migrations()).await; let store = PodcastStore::new(pool.clone()); let mut expected = SyncSnapshot { podcast_subscriptions: vec![subscription.clone()], podcast_shares: vec![share], podcast_progress: vec![progress.clone()], podcast_listen_sessions: vec![listened], ..SyncSnapshot::default() }; expected.bound_podcast_state(); assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected); assert_eq!( store .export_snapshot(user_b) .await .unwrap() .podcast_progress[0] .position_ms, 5 ); assert_eq!( store.feed_aliases(user_a, &pending_podcast).await.unwrap(), vec![feed_url("pending.xml")] ); // Catalog rows: the podcast from its register, the episode from the // newest reference in the log. let summary = store .podcast(user_a, &episode.podcast_id) .await .unwrap() .unwrap(); assert!(summary.subscribed); assert_eq!( summary.meta.image_url.as_deref(), Some("https://img.example/show.jpg") ); let item = store .episode(user_a, &episode.episode_id) .await .unwrap() .unwrap(); assert_eq!(item.episode.title, episode.episode_title); assert_eq!(item.episode.season, episode.season); assert_eq!(item.resume_position_ms(), 1_200_000); assert_eq!(item.completed_count, 1); // Imported rows are searchable once their search text is computed. let mut search = EpisodeQuery::new(episode.podcast_id.clone()); search.search = Some("episode 42".into()); assert_eq!(store.episodes(user_a, &search).await.unwrap().len(), 1); // The store keeps working on imported rows, and the relay table stays. assert_eq!( store .apply_remote_op(user_a, &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000)) .await .unwrap(), ApplyOutcome::Unchanged ); let relay_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM furumusic__fed_podcast_relay") .fetch_one(&pool) .await .unwrap(); assert_eq!(relay_rows, 7); // Running the import again (or the migrations) changes nothing. for sql in schema::import_statements("TRUE") { sqlx::query(&sql).execute(&pool).await.unwrap(); } migrate(&url, all_migrations()).await; assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected); pool.close().await; sqlx::query(&format!("DROP DATABASE {name} WITH (FORCE)")) .execute(&admin) .await .unwrap(); }