Files
furumi-web/src/podcasts/postgres_tests.rs
T

1467 lines
46 KiB
Rust
Raw Normal View History

//! Podcast store against PostgreSQL.
//!
//! Ignored by default: set `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a
//! disposable database (for example a throwaway `postgres:16-alpine`
//! container). The tests migrate it with the production migrations, use
//! random account ids and remove their rows again; the migration test
//! creates and drops its own database next to it.
use std::collections::BTreeSet;
use cot::db::migrations::{MigrationEngine, SyncDynMigration};
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot, random_hex};
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use music_dht::podcast::{
EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord,
ProgressStatus, ShareRecord, SubscriptionRecord,
};
use super::*;
use crate::player::queue_items::test_episode;
const DATABASE_URL: &str = "FURU_DEVICE_SYNC_TEST_DATABASE_URL";
/// Every production migration (the list of `admin::App::migrations`).
fn all_migrations() -> Vec<&'static SyncDynMigration> {
[
crate::config::db_migrations::MIGRATIONS,
crate::user::db_migrations::MIGRATIONS,
crate::music::db_migrations::MIGRATIONS,
crate::scheduler::db_migrations::MIGRATIONS,
crate::auth::db_migrations::MIGRATIONS,
crate::youtube::db_migrations::MIGRATIONS,
]
.into_iter()
.flatten()
.copied()
.collect()
}
async fn migrate(url: &str, migrations: Vec<&'static SyncDynMigration>) {
let database = cot::db::Database::new(url).await.unwrap();
MigrationEngine::new(migrations)
.unwrap()
.run(&database)
.await
.unwrap();
database.close().await.unwrap();
}
/// Pool on the test database, migrated once per test binary with the
/// production migrations (also used by the device-sync Postgres tests).
pub(crate) async fn test_pool() -> sqlx::PgPool {
static MIGRATED: tokio::sync::Mutex<bool> = tokio::sync::Mutex::const_new(false);
let url = std::env::var(DATABASE_URL).unwrap();
let mut migrated = MIGRATED.lock().await;
if !*migrated {
migrate(&url, all_migrations()).await;
*migrated = true;
}
sqlx::PgPool::connect(&url).await.unwrap()
}
fn random_user() -> i64 {
let bytes = blake3::hash(random_hex(8).as_bytes());
9_000_000_000
+ i64::from(u32::from_le_bytes(bytes.as_bytes()[..4].try_into().unwrap()) % 1_000_000_000)
}
const ACCOUNT_TABLES: &[&str] = &[
"furumusic__podcast_fetch_state",
"furumusic__podcast_listen_session",
"furumusic__podcast_progress",
"furumusic__podcast_share",
"furumusic__podcast_subscription",
"furumusic__podcast_episode",
"furumusic__podcast_feed_alias",
"furumusic__podcast",
"furumusic__fed_podcast_relay",
"furumusic__fed_device_identity",
"furumusic__fed_device",
"furumusic__fed_sync_ops",
"furumusic__fed_sync_vector",
"furumusic__fed_state_like",
"furumusic__fed_state_playlist",
"furumusic__fed_state_playlist_item",
];
async fn cleanup(pool: &sqlx::PgPool, users: impl IntoIterator<Item = i64>) {
let users: Vec<i64> = users.into_iter().collect();
for table in ACCOUNT_TABLES {
sqlx::query(&format!("DELETE FROM {table} WHERE user_id = ANY($1)"))
.bind(&users)
.execute(pool)
.await
.unwrap();
}
}
fn feed_url(path: &str) -> FeedUrl {
FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap()
}
fn meta(path: &str, title: &str) -> PodcastMeta {
PodcastMeta {
feed_url: feed_url(path),
title: title.into(),
author: Some("Host".into()),
description: Some("Channel description".into()),
link: None,
image_url: None,
}
}
fn op(origin: &str, seq: i64, hlc_ms: i64, payload: SyncOpPayload) -> SyncOpWire {
SyncOpWire {
op_id: format!("{origin}:{seq}"),
origin_device_id: origin.into(),
seq,
hlc_ms,
payload,
}
}
fn session(
id: &str,
device: &str,
episode: &PodcastEpisodeRef,
listened_ms: i64,
completed: bool,
) -> ListenSession {
ListenSession {
session_id: id.into(),
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
device_id: device.into(),
started_at_ms: 1_000,
ended_at_ms: 1_000 + listened_ms,
start_position_ms: 0,
end_position_ms: listened_ms,
listened_ms,
completed,
}
}
fn progress_op(
origin: &str,
seq: i64,
hlc_ms: i64,
episode: &PodcastEpisodeRef,
generation: u64,
position_ms: i64,
) -> SyncOpWire {
op(
origin,
seq,
hlc_ms,
SyncOpPayload::PodcastProgressSet {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms,
status: ProgressStatus::InProgress,
generation,
session_id: format!("{origin}-g{generation}"),
episode: Some(PodcastEpisodeRef {
progress_generation: None,
..episode.clone()
}),
},
)
}
/// Every permutation of `0..n` (Heap's algorithm).
fn permutations(n: usize) -> Vec<Vec<usize>> {
fn heap(k: usize, items: &mut Vec<usize>, out: &mut Vec<Vec<usize>>) {
if k <= 1 {
out.push(items.clone());
return;
}
for i in 0..k {
heap(k - 1, items, out);
let j = if k.is_multiple_of(2) { i } else { 0 };
items.swap(j, k - 1);
}
}
let mut out = Vec::new();
heap(n, &mut (0..n).collect(), &mut out);
out
}
async fn apply_all(store: &PodcastStore, user_id: i64, ops: &[SyncOpWire]) {
for op in ops {
let outcome = store.apply_remote_op(user_id, op).await.unwrap();
assert!(
matches!(outcome, ApplyOutcome::Changed | ApplyOutcome::Unchanged),
"{outcome:?}"
);
}
}
/// The registers and sessions of one podcast and episode from three
/// devices, with a duplicate, a late checkpoint of an older session, a
/// concurrent offline action and a growing session.
fn convergence_ops() -> Vec<SyncOpWire> {
let episode = test_episode();
let podcast_id = episode.podcast_id.clone();
let podcast = PodcastMeta {
feed_url: episode.feed_url.clone(),
title: episode.podcast_title.clone(),
author: None,
description: Some("dropped from snapshots".into()),
link: None,
image_url: None,
};
let subscribe = op(
"tui_a",
1,
100,
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast_id.clone(),
subscribed: true,
podcast: podcast.clone(),
feed_aliases: vec![],
generation: 1,
},
);
vec![
subscribe.clone(),
// B saw g1 and unsubscribes with a slow clock: g2 still wins.
op(
"tui_b",
1,
50,
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast_id.clone(),
subscribed: false,
podcast,
feed_aliases: vec![],
generation: 2,
},
),
// A shared concurrently at g2: stored, but not effective.
op(
"tui_a",
2,
300,
SyncOpPayload::PodcastShareSet {
podcast_id: podcast_id.clone(),
shared: true,
generation: 2,
},
),
op(
"tui_c",
1,
400,
SyncOpPayload::PodcastFeedAliasesAdded {
podcast_id,
feed_aliases: vec![feed_url("moved.xml")],
},
),
subscribe,
// A late g3 checkpoint of an old session with a fast clock.
progress_op("tui_a", 3, 9_000, &episode, 3, 600_000),
// B and C both seek at g4 with the same clock: device id decides.
progress_op("tui_b", 2, 1_000, &episode, 4, 10_000),
progress_op("tui_c", 2, 1_000, &episode, 4, 20_000),
op(
"tui_a",
4,
9_100,
SyncOpPayload::PodcastListenRecorded {
session: session("s1", "tui_a", &episode, 40_000, false),
episode: None,
},
),
op(
"tui_a",
5,
9_200,
SyncOpPayload::PodcastListenRecorded {
session: session("s1", "tui_a", &episode, 90_000, true),
episode: None,
},
),
op(
"tui_b",
3,
1_100,
SyncOpPayload::PodcastListenRecorded {
session: session("s2", "tui_b", &episode, 31_000, false),
episode: Some(PodcastEpisodeRef {
progress_generation: None,
..episode
}),
},
),
]
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_ops_converge_in_any_order_and_match_snapshot_import() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let ops = convergence_ops();
let reference_user = random_user();
let mut users = vec![reference_user];
apply_all(&store, reference_user, &ops).await;
let reference = store.export_snapshot(reference_user).await.unwrap();
// The contract outcome, independently of the order.
let episode = test_episode();
assert_eq!(reference.podcast_subscriptions.len(), 1);
let subscription = &reference.podcast_subscriptions[0];
assert!(!subscription.subscribed);
assert_eq!(subscription.generation, 2);
assert!(subscription.podcast.description.is_none());
assert_eq!(
subscription.feed_urls(),
vec![episode.feed_url.clone(), feed_url("moved.xml")]
);
assert!(reference.podcast_shares[0].shared);
let summary = store
.podcast(reference_user, &episode.podcast_id)
.await
.unwrap()
.unwrap();
assert!(!summary.subscribed && !summary.shared);
assert_eq!(reference.podcast_progress.len(), 1);
assert_eq!(reference.podcast_progress[0].position_ms, 20_000);
assert_eq!(reference.podcast_progress[0].device_id, "tui_c");
let sessions: BTreeSet<_> = reference
.podcast_listen_sessions
.iter()
.map(|session| {
(
session.session_id.as_str(),
session.listened_ms,
session.completed,
)
})
.collect();
assert_eq!(
sessions,
BTreeSet::from([("s1", 90_000, true), ("s2", 31_000, false)])
);
assert_eq!(
store
.completed_listen_count(reference_user, &episode.episode_id)
.await
.unwrap(),
1
);
// The episode reference created a catalog row (never touching progress).
let item = store
.episode(reference_user, &episode.episode_id)
.await
.unwrap()
.unwrap();
assert!(!item.in_feed);
assert_eq!(item.resume_position_ms(), 20_000);
// Register operations: all 120 orders of the first five operations
// (subscription, unsubscription, share, aliases, duplicate) with the
// rest appended; progress and session operations: all 720 orders of
// the last six with the first five prepended.
let (registers, personal) = ops.split_at(5);
for order in permutations(registers.len()) {
let user_id = random_user();
users.push(user_id);
let mut shuffled: Vec<SyncOpWire> = order.iter().map(|&i| registers[i].clone()).collect();
shuffled.extend_from_slice(personal);
apply_all(&store, user_id, &shuffled).await;
assert_eq!(
store.export_snapshot(user_id).await.unwrap(),
reference,
"{order:?}"
);
}
for order in permutations(personal.len()) {
let user_id = random_user();
users.push(user_id);
let mut shuffled = registers.to_vec();
shuffled.extend(order.iter().map(|&i| personal[i].clone()));
apply_all(&store, user_id, &shuffled).await;
assert_eq!(
store.export_snapshot(user_id).await.unwrap(),
reference,
"{order:?}"
);
}
// A new device recovering from the snapshot reaches the same state, and
// the operations delivered afterwards change nothing. (Snapshots omit
// the channel description, so the winning subscription operation only
// adds it back to the stored register: the contract's tie-break by
// metadata. Every other operation is already reflected.)
let recovered = random_user();
users.push(recovered);
let stats = store.apply_snapshot(recovered, &reference).await.unwrap();
assert_eq!(stats.rejected, 0);
assert!(!stats.ignored_over_limits);
assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference);
for op in ops.iter().rev() {
let expected = if op.op_id == "tui_b:1" {
ApplyOutcome::Changed
} else {
ApplyOutcome::Unchanged
};
assert_eq!(
store.apply_remote_op(recovered, op).await.unwrap(),
expected,
"{}",
op.op_id
);
}
assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference);
// A stale snapshot never replaces newer state.
let stale = SyncSnapshot {
podcast_progress: vec![ProgressRecord {
position_ms: 1,
generation: 3,
hlc_ms: i64::MAX,
..reference.podcast_progress[0].clone()
}],
..SyncSnapshot::default()
};
let stats = store.apply_snapshot(recovered, &stale).await.unwrap();
assert_eq!((stats.changed, stats.unchanged), (0, 1));
assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference);
// Invalid operations are rejected without touching state.
let mut invalid = progress_op("tui_a", 9, 1, &episode, 9, 1);
if let SyncOpPayload::PodcastProgressSet { session_id, .. } = &mut invalid.payload {
session_id.clear();
}
assert!(matches!(
store.apply_remote_op(recovered, &invalid).await.unwrap(),
ApplyOutcome::Rejected(_)
));
let mut huge = progress_op("tui_a", 10, 1, &episode, u64::MAX, 1);
huge.op_id = "tui_a:10".into();
assert!(matches!(
store.apply_remote_op(recovered, &huge).await.unwrap(),
ApplyOutcome::Rejected(_)
));
assert_eq!(store.export_snapshot(recovered).await.unwrap(), reference);
cleanup(&pool, users).await;
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_local_actions_keep_progress_through_unsubscribe() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let user_id = random_user();
let device = crate::federation::devices::local_device_id(&pool, user_id)
.await
.unwrap();
let episode = test_episode();
let show = PodcastMeta {
feed_url: episode.feed_url.clone(),
..meta("unused.xml", &episode.podcast_title)
};
let subscribed = store.subscribe(user_id, &show).await.unwrap();
assert_eq!(subscribed.podcast_id, episode.podcast_id);
assert!(!subscribed.already_subscribed);
let [first] = subscribed.ops.as_slice() else {
panic!("one operation");
};
assert_eq!(first.origin_device_id, device);
assert_eq!(first.op_id, format!("{device}:{}", first.seq));
// The operation is in the log with exactly this stamp; applying it again
// is a no-op.
let logged: serde_json::Value = sqlx::query_scalar(
"SELECT payload_json FROM furumusic__fed_sync_ops
WHERE user_id = $1 AND op_id = $2 AND hlc_ms = $3 AND kind = 'podcast_subscription_set'",
)
.bind(user_id)
.bind(&first.op_id)
.bind(first.hlc_ms)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(logged, serde_json::to_value(&first.payload).unwrap());
assert_eq!(
store.apply_remote_op(user_id, first).await.unwrap(),
ApplyOutcome::Unchanged
);
assert!(
store
.subscribe(user_id, &show)
.await
.unwrap()
.already_subscribed
);
assert_eq!(
store
.set_shared(user_id, &episode.podcast_id, true)
.await
.unwrap()
.len(),
1
);
assert!(
store
.set_shared(user_id, &episode.podcast_id, true)
.await
.unwrap()
.is_empty()
);
assert!(
store
.podcast(user_id, &episode.podcast_id)
.await
.unwrap()
.unwrap()
.shared
);
// Listening: an explicit start, checkpoints keep the generation.
let start = store
.start_session(user_id, &episode, "web-s1", 0, episode.progress_generation)
.await
.unwrap();
assert_eq!(start.record.generation, 8, "max(local, carried 7) + 1");
let CheckpointOutcome::Written(checkpoint) = store
.checkpoint(user_id, &start.session, 30_000)
.await
.unwrap()
else {
panic!("checkpoint written");
};
assert_eq!(checkpoint.record.generation, 8);
assert_eq!(
store
.resume_position_ms(user_id, &episode.episode_id)
.await
.unwrap(),
30_000
);
// A seek is explicit: later checkpoints use its generation.
let seek = store
.seek(user_id, &checkpoint.session, 12_000)
.await
.unwrap();
assert_eq!(seek.record.generation, 9);
assert_eq!(seek.session.generation, 9);
let start = seek;
// Another device seeks back at a greater generation with a slow clock:
// later checkpoints of the old session are refused.
store
.apply_remote_op(user_id, &progress_op("tui_b", 1, 1, &episode, 10, 5_000))
.await
.unwrap();
assert!(matches!(
store
.checkpoint(user_id, &start.session, 900_000)
.await
.unwrap(),
CheckpointOutcome::Superseded { .. }
));
assert_eq!(
store
.resume_position_ms(user_id, &episode.episode_id)
.await
.unwrap(),
5_000
);
let listened = ListenSession {
device_id: device.clone(),
..session("web-s1", &device, &episode, 45_000, false)
};
assert!(
store
.record_listen_session(user_id, &listened, None)
.await
.unwrap()
.is_some()
);
assert!(
store
.record_listen_session(user_id, &listened, None)
.await
.unwrap()
.is_none()
);
assert!(
store
.record_listen_session(
user_id,
&session("x", "tui_b", &episode, 45_000, false),
None
)
.await
.is_err()
);
// Unsubscribing keeps progress and history and revokes the share.
let ops = store
.unsubscribe(user_id, &episode.podcast_id)
.await
.unwrap();
assert_eq!(ops.len(), 2);
assert!(
store
.unsubscribe(user_id, &episode.podcast_id)
.await
.unwrap()
.is_empty()
);
let summary = store
.podcast(user_id, &episode.podcast_id)
.await
.unwrap()
.unwrap();
assert!(!summary.subscribed && !summary.shared);
assert_eq!(
store
.resume_position_ms(user_id, &episode.episode_id)
.await
.unwrap(),
5_000
);
assert_eq!(
store.listen_history(user_id, None, 10).await.unwrap().len(),
1
);
assert!(
store
.list_podcasts(user_id, PodcastListFilter::Subscribed)
.await
.unwrap()
.is_empty()
);
assert_eq!(
store
.list_podcasts(user_id, PodcastListFilter::All)
.await
.unwrap()
.len(),
1
);
// Subscribing again resolves to the same podcast, continues the
// progress and is private again.
let again = store.subscribe(user_id, &show).await.unwrap();
assert_eq!(again.podcast_id, episode.podcast_id);
let register = store
.subscription(user_id, &episode.podcast_id)
.await
.unwrap()
.unwrap();
let share = store
.share(user_id, &episode.podcast_id)
.await
.unwrap()
.unwrap();
assert!(register.subscribed);
assert!(register.generation > share.generation);
let summary = store
.podcast(user_id, &episode.podcast_id)
.await
.unwrap()
.unwrap();
assert!(summary.subscribed && !summary.shared);
assert_eq!(
store
.resume_position_ms(user_id, &episode.episode_id)
.await
.unwrap(),
5_000
);
// Mark played writes progress only; restart and unplayed reset it.
let played = store.mark_played(user_id, &episode).await.unwrap();
assert_eq!(played.record.status, ProgressStatus::Completed);
assert_eq!(played.record.generation, 11);
assert_eq!(
store.listen_history(user_id, None, 10).await.unwrap().len(),
1
);
assert_eq!(
store
.resume_position_ms(user_id, &episode.episode_id)
.await
.unwrap(),
0
);
let restarted = store.restart(user_id, &episode, "web-s2").await.unwrap();
assert_eq!(
(
restarted.record.position_ms,
restarted.record.status,
restarted.record.generation
),
(0, ProgressStatus::InProgress, 12)
);
let finished = store
.complete_session(user_id, &restarted.session, 1_790_000)
.await
.unwrap();
assert_eq!(
(finished.record.status, finished.record.generation),
(ProgressStatus::Completed, 13)
);
assert!(matches!(
store
.checkpoint(user_id, &finished.session, 1_795_000)
.await
.unwrap(),
CheckpointOutcome::AlreadyCompleted { .. }
));
let mut played_filter = EpisodeQuery::new(episode.podcast_id.clone());
played_filter.played = PlayedFilter::Played;
assert_eq!(
store.episodes(user_id, &played_filter).await.unwrap().len(),
1
);
played_filter.played = PlayedFilter::Unplayed;
assert!(
store
.episodes(user_id, &played_filter)
.await
.unwrap()
.is_empty()
);
assert_eq!(
store
.mark_unplayed(user_id, &episode)
.await
.unwrap()
.record
.status,
ProgressStatus::NotStarted
);
played_filter.played = PlayedFilter::NotStarted;
let items = store.episodes(user_id, &played_filter).await.unwrap();
assert_eq!(items[0].status(), ProgressStatus::NotStarted);
// Local stamps: consecutive sequence numbers, strictly growing clock.
let stamps: Vec<(i64, i64)> = sqlx::query_as(
"SELECT seq, hlc_ms FROM furumusic__fed_sync_ops
WHERE user_id = $1 AND origin_device_id = $2 ORDER BY seq",
)
.bind(user_id)
.bind(&device)
.fetch_all(&pool)
.await
.unwrap();
assert!(
stamps
.windows(2)
.all(|pair| pair[1].0 == pair[0].0 + 1 && pair[1].1 > pair[0].1)
);
// Re-applying the whole local log changes nothing.
let before = store.export_snapshot(user_id).await.unwrap();
let payloads: Vec<(String, i64, i64, serde_json::Value)> = sqlx::query_as(
"SELECT op_id, seq, hlc_ms, payload_json FROM furumusic__fed_sync_ops
WHERE user_id = $1 AND kind LIKE 'podcast\\_%' ORDER BY hlc_ms",
)
.bind(user_id)
.fetch_all(&pool)
.await
.unwrap();
for (op_id, seq, hlc_ms, payload) in payloads {
let origin = op_id.rsplit_once(':').unwrap().0.to_string();
let op = SyncOpWire {
op_id,
origin_device_id: origin,
seq,
hlc_ms,
payload: serde_json::from_value(payload).unwrap(),
};
assert_eq!(
store.apply_remote_op(user_id, &op).await.unwrap(),
ApplyOutcome::Unchanged
);
}
assert_eq!(store.export_snapshot(user_id).await.unwrap(), before);
// Sharing needs a subscription.
let unknown = PodcastId::from_feed_url(&feed_url("unknown.xml"));
let error = store.set_shared(user_id, &unknown, true).await.unwrap_err();
assert!(matches!(
error.downcast_ref::<PodcastStoreError>(),
Some(PodcastStoreError::NotSubscribed(_))
));
cleanup(&pool, [user_id]).await;
}
fn rss(items: &str) -> String {
format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>Refresh Show</title>
<link>https://show.example/</link>
<description>Channel text</description>
<itunes:author>Host</itunes:author>
{items}
</channel>
</rss>"#
)
}
fn parse(xml: &str, url: &FeedUrl) -> ParsedFeed {
parse_feed_with_base(xml.as_bytes(), Some(url.as_str())).unwrap()
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_feed_refresh_keeps_ids_and_progress() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let user_id = random_user();
let url = feed_url("refresh.xml");
let podcast_id = PodcastId::from_feed_url(&url);
let v1 = parse(
&rss(r#"
<item><guid>g-1</guid><title>One</title><itunes:season>1</itunes:season>
<itunes:episode>1</itunes:episode><pubDate>Mon, 01 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="https://cdn.example/a1.mp3" type="audio/mpeg" length="10"/></item>
<item><title>Two</title><link>https://show.example/ep2</link>
<itunes:season>1</itunes:season><itunes:episode>2</itunes:episode>
<pubDate>Mon, 08 Jan 2024 10:00:00 GMT</pubDate>
<description>Talk about Ünïcode</description>
<enclosure url="https://cdn.example/a2.mp3" type="audio/mpeg"/></item>
<item><title>Three</title><pubDate>Mon, 15 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="https://cdn.example/a3.mp3" type="audio/mpeg"/></item>"#),
&url,
);
// What every device derives for this document.
let ids: Vec<EpisodeId> = v1
.resolve_episode_ids(&podcast_id, &[])
.into_iter()
.map(Option::unwrap)
.collect();
// Another device played the GUID-less "Two" before this account ever
// fetched the feed: its reference arrives with the progress.
let two = PodcastEpisodeRef {
podcast_id: podcast_id.clone(),
episode_id: ids[1].clone(),
feed_url: url.clone(),
podcast_title: "Refresh Show".into(),
episode_title: "Two".into(),
audio_url: "https://cdn.example/a2.mp3".into(),
mime: None,
duration_ms: None,
published_at_ms: None,
season: None,
episode_number: None,
image_url: None,
progress_generation: None,
};
store
.apply_remote_op(user_id, &progress_op("tui_a", 1, 10, &two, 3, 61_000))
.await
.unwrap();
let subscribed = store
.subscribe(user_id, &v1.to_podcast_meta(url.clone()))
.await
.unwrap();
assert_eq!(subscribed.podcast_id, podcast_id);
let first = store
.apply_feed(
user_id,
&FeedRefresh {
podcast_id: &podcast_id,
feed: &v1,
fetched_from: &url,
moved_permanently: false,
},
)
.await
.unwrap();
assert_eq!((first.new_episodes, first.updated_episodes), (2, 1));
assert!(first.ops.is_empty());
let stored: BTreeSet<EpisodeId> = store
.episodes(user_id, &EpisodeQuery::new(podcast_id.clone()))
.await
.unwrap()
.into_iter()
.map(|item| item.episode.episode_id)
.collect();
assert_eq!(stored, ids.iter().cloned().collect());
let progress_before = store.progress(user_id, &ids[1]).await.unwrap().unwrap();
// The publisher renames episodes, moves their audio, drops "Three",
// adds "Four" and moves the feed permanently.
let moved = feed_url("moved-refresh.xml");
let v2 = parse(
&rss(r#"
<item><guid>g-1</guid><title>One (remastered)</title><itunes:season>1</itunes:season>
<itunes:episode>1</itunes:episode><pubDate>Mon, 01 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="https://cdn2.example/a1.mp3" type="audio/mpeg"/></item>
<item><title>Two, revised</title><link>https://show.example/ep2</link>
<itunes:season>1</itunes:season><itunes:episode>2</itunes:episode>
<pubDate>Mon, 08 Jan 2024 10:00:00 GMT</pubDate>
<description>Talk about Ünïcode</description>
<enclosure url="https://cdn2.example/a2.mp3" type="audio/mpeg"/></item>
<item><guid>g-4</guid><title>Four</title><itunes:season>2</itunes:season>
<itunes:episode>1</itunes:episode><pubDate>Mon, 22 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="https://cdn2.example/a4.mp3" type="audio/mpeg"/></item>"#),
&moved,
);
let refresh = FeedRefresh {
podcast_id: &podcast_id,
feed: &v2,
fetched_from: &moved,
moved_permanently: true,
};
let second = store.apply_feed(user_id, &refresh).await.unwrap();
assert_eq!(
(
second.new_episodes,
second.updated_episodes,
second.unchanged_episodes
),
(1, 2, 0)
);
assert_eq!(second.ops.len(), 1, "aliases changed once");
assert_eq!(second.feed_url.as_ref(), Some(&moved));
let again = store.apply_feed(user_id, &refresh).await.unwrap();
assert!(again.ops.is_empty());
assert_eq!(again.unchanged_episodes, 3);
assert_eq!(
store.resolve_podcast_id(user_id, &moved).await.unwrap(),
podcast_id
);
assert_eq!(
store.resolve_podcast_id(user_id, &url).await.unwrap(),
podcast_id
);
// Ids and progress survive; the dropped episode stays out of the feed.
let one = store.episode(user_id, &ids[0]).await.unwrap().unwrap();
assert_eq!(one.episode.title, "One (remastered)");
assert_eq!(one.episode.audio_url, "https://cdn2.example/a1.mp3");
let two_item = store.episode(user_id, &ids[1]).await.unwrap().unwrap();
assert_eq!(two_item.episode.audio_url, "https://cdn2.example/a2.mp3");
assert_eq!(two_item.progress.as_ref(), Some(&progress_before));
assert_eq!(two_item.resume_position_ms(), 61_000);
let three = store.episode(user_id, &ids[2]).await.unwrap().unwrap();
assert!(!three.in_feed);
// The subscription register is untouched by refreshes.
let register = store
.subscription(user_id, &podcast_id)
.await
.unwrap()
.unwrap();
assert_eq!(register.op_id, subscribed.ops[0].op_id);
assert!(register.feed_urls().contains(&moved));
let reference = store.episode_ref(user_id, &ids[1]).await.unwrap().unwrap();
assert_eq!(reference.progress_generation, Some(3));
assert_eq!(reference.audio_url, "https://cdn2.example/a2.mp3");
// UI queries: counters, seasons, filters, search and sorting.
let summary = store.podcast(user_id, &podcast_id).await.unwrap().unwrap();
assert_eq!(
(
summary.episode_count,
summary.unplayed_count,
summary.in_progress_count,
summary.season_count
),
(4, 4, 1, 2)
);
assert_eq!(summary.meta.feed_url, moved);
let seasons = store.seasons(user_id, &podcast_id).await.unwrap();
assert_eq!(
seasons
.iter()
.map(|season| (season.season, season.episode_count))
.collect::<Vec<_>>(),
vec![(Some(1), 2), (Some(2), 1), (None, 1)]
);
let query = |configure: &dyn Fn(&mut EpisodeQuery)| {
let mut query = EpisodeQuery::new(podcast_id.clone());
configure(&mut query);
query
};
let titles = |items: Vec<EpisodeListItem>| {
items
.into_iter()
.map(|item| item.episode.title)
.collect::<Vec<_>>()
};
assert_eq!(
titles(store.episodes(user_id, &query(&|_| {})).await.unwrap()),
["Four", "Three", "Two, revised", "One (remastered)"]
);
assert_eq!(
titles(
store
.episodes(user_id, &query(&|q| q.sort = EpisodeSort::EpisodeNumber))
.await
.unwrap()
),
["One (remastered)", "Two, revised", "Four", "Three"]
);
assert_eq!(
titles(
store
.episodes(user_id, &query(&|q| q.season = SeasonFilter::Season(1)))
.await
.unwrap()
),
["Two, revised", "One (remastered)"]
);
assert_eq!(
titles(
store
.episodes(user_id, &query(&|q| q.season = SeasonFilter::NoSeason))
.await
.unwrap()
),
["Three"]
);
assert_eq!(
titles(
store
.episodes(user_id, &query(&|q| q.played = PlayedFilter::InProgress))
.await
.unwrap()
),
["Two, revised"]
);
assert_eq!(
titles(
store
.episodes(user_id, &query(&|q| q.search = Some("ÜNÏCODE".into())))
.await
.unwrap()
),
["Two, revised"]
);
assert_eq!(
titles(
store
.episodes(user_id, &query(&|q| q.search = Some("remaster".into())))
.await
.unwrap()
),
["One (remastered)"]
);
assert_eq!(
titles(
store
.episodes(
user_id,
&query(&|q| {
q.limit = Some(1);
q.offset = 1;
})
)
.await
.unwrap()
),
["Three"]
);
assert!(
store
.episodes(user_id, &query(&|q| q.search = Some("one two".into())))
.await
.unwrap()
.is_empty()
);
// A refresh of a podcast this account does not know is refused.
let unknown = PodcastId::from_feed_url(&feed_url("never.xml"));
let error = store
.apply_feed(
user_id,
&FeedRefresh {
podcast_id: &unknown,
..refresh
},
)
.await
.unwrap_err();
assert!(matches!(
error.downcast_ref::<PodcastStoreError>(),
Some(PodcastStoreError::UnknownPodcast(_))
));
cleanup(&pool, [user_id]).await;
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_fetch_state_is_local_only() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let user_id = random_user();
let due_show = meta("due.xml", "Due");
let later_show = meta("later.xml", "Later");
let gone_show = meta("gone.xml", "Gone");
let mut ids = Vec::new();
for show in [&due_show, &later_show, &gone_show] {
ids.push(store.subscribe(user_id, show).await.unwrap().podcast_id);
}
store.unsubscribe(user_id, &ids[2]).await.unwrap();
let before = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap();
let now = 1_800_000_000_000;
store
.record_fetch_success(
user_id,
&ids[1],
now,
Some("\"secret-etag\""),
Some("Tue"),
Some(now + 60_000),
)
.await
.unwrap();
assert_eq!(
store
.record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1))
.await
.unwrap(),
1
);
assert_eq!(
store
.record_fetch_failure(user_id, &ids[0], now, "boom: 503", Some(now - 1))
.await
.unwrap(),
2
);
store
.set_cover_cache(
user_id,
&ids[0],
Some("https://img.example/c.jpg"),
Some("/var/cache/cover-secret.jpg"),
)
.await
.unwrap();
let state = store.fetch_state(user_id, &ids[0]).await.unwrap();
assert_eq!(state.consecutive_failures, 2);
assert_eq!(state.last_error.as_deref(), Some("boom: 503"));
assert_eq!(
state.cover_path.as_deref(),
Some("/var/cache/cover-secret.jpg")
);
store
.record_fetch_not_modified(user_id, &ids[1], now + 1, Some(now + 60_000))
.await
.unwrap();
let later = store.fetch_state(user_id, &ids[1]).await.unwrap();
assert_eq!(later.etag.as_deref(), Some("\"secret-etag\""));
assert_eq!(later.last_success_at_ms, Some(now + 1));
// Fetch state changes nothing in the synced state.
let after = serde_json::to_string(&store.export_snapshot(user_id).await.unwrap()).unwrap();
assert_eq!(after, before);
for secret in ["secret-etag", "boom", "cover-secret", "Channel description"] {
assert!(!after.contains(secret), "{secret}");
}
let ops: Vec<String> = sqlx::query_scalar(
"SELECT payload_json::text FROM furumusic__fed_sync_ops WHERE user_id = $1",
)
.bind(user_id)
.fetch_all(&pool)
.await
.unwrap();
assert!(
ops.iter()
.all(|op| !op.contains("secret-etag") && !op.contains("cover-secret"))
);
// Scheduling: overdue first, future and unsubscribed podcasts skipped.
let due: Vec<PodcastId> = store
.podcasts_due_for_refresh(now, 10_000)
.await
.unwrap()
.into_iter()
.filter(|(user, ..)| *user == user_id)
.map(|(_, id, _)| id)
.collect();
assert_eq!(due, vec![ids[0].clone()]);
let unknown = PodcastId::from_feed_url(&feed_url("unknown-fetch.xml"));
assert!(
store
.record_fetch_failure(user_id, &unknown, now, "x", None)
.await
.is_err()
);
cleanup(&pool, [user_id]).await;
}
/// Relay rows exactly as 0.13.0 stored them (`podcast_relay`): the contract
/// records as JSON, keyed by section and entity.
async fn insert_relay(
pool: &sqlx::PgPool,
user_id: i64,
section: &str,
key: &str,
value: serde_json::Value,
) {
sqlx::query(
"INSERT INTO furumusic__fed_podcast_relay
(user_id, section, record_key, record_json, updated_at_ms)
VALUES ($1, $2, $3, $4, 1700000000000)",
)
.bind(user_id)
.bind(section)
.bind(key)
.bind(value)
.execute(pool)
.await
.unwrap();
}
async fn insert_logged_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire, kind: &str) {
sqlx::query(
"INSERT INTO furumusic__fed_sync_ops
(user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms,
received_at_ms, tombstone)
VALUES ($1, $2, $3, $4, $5, $6, $7, 1700000000000, FALSE)",
)
.bind(user_id)
.bind(&op.op_id)
.bind(&op.origin_device_id)
.bind(op.seq)
.bind(kind)
.bind(serde_json::to_value(&op.payload).unwrap())
.bind(op.hlc_ms)
.execute(pool)
.await
.unwrap();
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_migrations_import_the_relay_state() {
let admin_url = std::env::var(DATABASE_URL).unwrap();
let admin = sqlx::PgPool::connect(&admin_url).await.unwrap();
let name = format!("furu_podcast_migration_{}", random_hex(6));
sqlx::query(&format!("CREATE DATABASE {name}"))
.execute(&admin)
.await
.unwrap();
let (base, _) = admin_url.rsplit_once('/').unwrap();
let url = format!("{base}/{name}");
// A 0.13.0 database: every migration up to the relay table.
let new_migrations = ["m_0056_create_podcast_store", "m_0057_import_podcast_relay"];
let old: Vec<_> = all_migrations()
.into_iter()
.filter(|migration| {
!(migration.app_name() == "furumusic" && new_migrations.contains(&migration.name()))
})
.collect();
migrate(&url, old).await;
let pool = sqlx::PgPool::connect(&url).await.unwrap();
// State relayed by 0.13.0 for two accounts.
let episode = test_episode();
let user_a = 1;
let user_b = 2;
let subscription = SubscriptionRecord {
podcast_id: episode.podcast_id.clone(),
subscribed: true,
podcast: PodcastMeta {
feed_url: episode.feed_url.clone(),
title: episode.podcast_title.clone(),
author: Some("Host".into()),
description: None,
link: None,
image_url: Some("https://img.example/show.jpg".into()),
},
feed_aliases: vec![episode.feed_url.clone(), feed_url("old.xml")],
generation: 3,
hlc_ms: 30,
op_id: "tui_a:3".into(),
};
let share = ShareRecord {
podcast_id: episode.podcast_id.clone(),
shared: false,
generation: 2,
hlc_ms: 20,
op_id: "tui_a:2".into(),
};
let progress = ProgressRecord {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms: 1_200_000,
status: ProgressStatus::InProgress,
generation: 9,
session_id: "tui_a-g9".into(),
hlc_ms: 90,
device_id: "tui_a".into(),
op_id: "tui_a:9".into(),
};
let listened = session("s9", "tui_a", &episode, 120_000, true);
let pending_podcast = PodcastId::from_feed_url(&feed_url("pending.xml"));
insert_relay(
&pool,
user_a,
"subscription",
subscription.podcast_id.as_str(),
serde_json::to_value(&subscription).unwrap(),
)
.await;
insert_relay(
&pool,
user_a,
"share",
share.podcast_id.as_str(),
serde_json::to_value(&share).unwrap(),
)
.await;
insert_relay(
&pool,
user_a,
"progress",
progress.episode_id.as_str(),
serde_json::to_value(&progress).unwrap(),
)
.await;
insert_relay(
&pool,
user_a,
"session",
"s9",
serde_json::to_value(&listened).unwrap(),
)
.await;
insert_relay(
&pool,
user_a,
"aliases",
pending_podcast.as_str(),
serde_json::to_value(vec![feed_url("pending.xml")]).unwrap(),
)
.await;
// An out-of-range generation (no conforming client) is skipped.
let huge = ProgressRecord {
episode_id: EpisodeId::from_guid(&episode.podcast_id, "huge").unwrap(),
generation: u64::MAX,
..progress.clone()
};
insert_relay(
&pool,
user_a,
"progress",
huge.episode_id.as_str(),
serde_json::to_value(&huge).unwrap(),
)
.await;
insert_relay(
&pool,
user_b,
"progress",
progress.episode_id.as_str(),
serde_json::to_value(&ProgressRecord {
position_ms: 5,
..progress.clone()
})
.unwrap(),
)
.await;
// The relayed operations kept the episode references (two versions).
let old_ref = PodcastEpisodeRef {
episode_title: "Old title".into(),
progress_generation: None,
..episode.clone()
};
insert_logged_op(
&pool,
user_a,
&progress_op("tui_a", 8, 80, &old_ref, 8, 1_000),
"podcast_progress_set",
)
.await;
insert_logged_op(
&pool,
user_a,
&progress_op("tui_a", 9, 90, &episode, 9, 1_200_000),
"podcast_progress_set",
)
.await;
// Upgrade.
migrate(&url, all_migrations()).await;
let store = PodcastStore::new(pool.clone());
let mut expected = SyncSnapshot {
podcast_subscriptions: vec![subscription.clone()],
podcast_shares: vec![share],
podcast_progress: vec![progress.clone()],
podcast_listen_sessions: vec![listened],
..SyncSnapshot::default()
};
expected.bound_podcast_state();
assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected);
assert_eq!(
store
.export_snapshot(user_b)
.await
.unwrap()
.podcast_progress[0]
.position_ms,
5
);
assert_eq!(
store.feed_aliases(user_a, &pending_podcast).await.unwrap(),
vec![feed_url("pending.xml")]
);
// Catalog rows: the podcast from its register, the episode from the
// newest reference in the log.
let summary = store
.podcast(user_a, &episode.podcast_id)
.await
.unwrap()
.unwrap();
assert!(summary.subscribed);
assert_eq!(
summary.meta.image_url.as_deref(),
Some("https://img.example/show.jpg")
);
let item = store
.episode(user_a, &episode.episode_id)
.await
.unwrap()
.unwrap();
assert_eq!(item.episode.title, episode.episode_title);
assert_eq!(item.episode.season, episode.season);
assert_eq!(item.resume_position_ms(), 1_200_000);
assert_eq!(item.completed_count, 1);
// Imported rows are searchable once their search text is computed.
let mut search = EpisodeQuery::new(episode.podcast_id.clone());
search.search = Some("episode 42".into());
assert_eq!(store.episodes(user_a, &search).await.unwrap().len(), 1);
// The store keeps working on imported rows, and the relay table stays.
assert_eq!(
store
.apply_remote_op(user_a, &progress_op("tui_a", 9, 90, &episode, 9, 1_200_000))
.await
.unwrap(),
ApplyOutcome::Unchanged
);
let relay_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM furumusic__fed_podcast_relay")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(relay_rows, 7);
// Running the import again (or the migrations) changes nothing.
for sql in schema::import_statements("TRUE") {
sqlx::query(&sql).execute(&pool).await.unwrap();
}
migrate(&url, all_migrations()).await;
assert_eq!(store.export_snapshot(user_a).await.unwrap(), expected);
pool.close().await;
sqlx::query(&format!("DROP DATABASE {name} WITH (FORCE)"))
.execute(&admin)
.await
.unwrap();
}