Подкасты в веб-плеере: хранилище PostgreSQL, обновление RSS, прокси, UI и воспроизведение (MR-W2) #11
No files matched your search
+41
-23
@@ -6,10 +6,10 @@
|
||||
//! `furumi/sync/2` and maps operations into user-scoped Postgres state.
|
||||
//!
|
||||
//! The wire types are the shared definitions; there are no local copies.
|
||||
//! Device-sync protocol v4 (podcasts) is announced because this adapter
|
||||
//! keeps every v4 element it receives: podcast operations stay in the
|
||||
//! generic operation log, podcast snapshot sections are merged into
|
||||
//! [`super::podcast_relay`], and queue items keep their explicit
|
||||
//! Device-sync protocol v4 (podcasts) is announced: podcast operations stay
|
||||
//! in the generic operation log and are applied to the account's podcast
|
||||
//! store ([`crate::podcasts::PodcastStore`]), which also provides the
|
||||
//! podcast snapshot sections, and queue items keep their explicit
|
||||
//! `PlaybackTrack.source` through the browser bridge
|
||||
//! ([`crate::player::queue_items`]). Everything sent to a peer is gated by
|
||||
//! the version that peer reported itself.
|
||||
@@ -39,8 +39,8 @@ use sqlx::Row;
|
||||
use crate::player::PlayerDeviceHub;
|
||||
use crate::player::queue_items;
|
||||
|
||||
use super::podcast_relay::{self, RelayRecord};
|
||||
use super::{TransportStats, record_stream_transport};
|
||||
use crate::podcasts::{ApplyOutcome, PodcastStore};
|
||||
|
||||
pub const SYNC_ALPN: &[u8] = b"furumi/sync/2";
|
||||
|
||||
@@ -2089,12 +2089,19 @@ async fn apply_op(
|
||||
| SyncOpPayload::PodcastFeedAliasesAdded { .. }
|
||||
| SyncOpPayload::PodcastProgressSet { .. }
|
||||
| SyncOpPayload::PodcastListenRecorded { .. } => {
|
||||
// The web player has no podcast features yet: keep the registers
|
||||
// for this device's snapshots; the operation itself is relayed
|
||||
// from the log.
|
||||
let records: Vec<_> = RelayRecord::from_op(op).into_iter().collect();
|
||||
podcast_relay::merge_records(pool, user_id, &records).await?;
|
||||
Ok(false)
|
||||
// The operation itself is relayed from the log; the store keeps
|
||||
// the registers (and the catalog rows of its episode reference).
|
||||
match PodcastStore::new(pool.clone())
|
||||
.apply_remote_op(user_id, op)
|
||||
.await?
|
||||
{
|
||||
ApplyOutcome::Changed => Ok(true),
|
||||
ApplyOutcome::Rejected(reason) => {
|
||||
tracing::debug!(op_id = %op.op_id, "skipped invalid podcast operation: {reason}");
|
||||
Ok(false)
|
||||
}
|
||||
ApplyOutcome::Unchanged | ApplyOutcome::Ignored => Ok(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2461,13 +2468,17 @@ async fn apply_playlist_item_state(
|
||||
}
|
||||
|
||||
async fn apply_snapshot(pool: &sqlx::PgPool, user_id: i64, snapshot: SyncSnapshot) -> Result<()> {
|
||||
let podcast = SyncSnapshot {
|
||||
podcast_subscriptions: snapshot.podcast_subscriptions.clone(),
|
||||
podcast_shares: snapshot.podcast_shares.clone(),
|
||||
podcast_progress: snapshot.podcast_progress.clone(),
|
||||
podcast_listen_sessions: snapshot.podcast_listen_sessions.clone(),
|
||||
..SyncSnapshot::default()
|
||||
};
|
||||
let podcasts = PodcastStore::new(pool.clone())
|
||||
.apply_snapshot(user_id, &snapshot)
|
||||
.await?;
|
||||
if podcasts.ignored_over_limits {
|
||||
tracing::debug!("ignored podcast snapshot sections above the contract limits");
|
||||
} else if podcasts.rejected > 0 {
|
||||
tracing::debug!(
|
||||
rejected = podcasts.rejected,
|
||||
"skipped invalid podcast snapshot records"
|
||||
);
|
||||
}
|
||||
for like in snapshot.likes {
|
||||
apply_like_state(
|
||||
pool,
|
||||
@@ -2544,7 +2555,6 @@ async fn apply_snapshot(pool: &sqlx::PgPool, user_id: i64, snapshot: SyncSnapsho
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
podcast_relay::merge_records(pool, user_id, &RelayRecord::from_snapshot(&podcast)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3712,7 +3722,7 @@ fn playback_clock() -> u64 {
|
||||
///
|
||||
/// When the browser does not know the position of its current item
|
||||
/// ([`queue_items::POSITION_UNKNOWN_KEY`]) and that item is a podcast
|
||||
/// episode, the position comes from the relayed progress register: handing
|
||||
/// episode, the position comes from its synced progress register: handing
|
||||
/// 0 to the next device would be written there as an explicit action and
|
||||
/// restart the episode everywhere.
|
||||
async fn playback_state_from_browser_json(
|
||||
@@ -3728,8 +3738,9 @@ async fn playback_state_from_browser_json(
|
||||
.get(wire.queue_pos)
|
||||
.and_then(PlaybackTrack::podcast_episode_ref)
|
||||
{
|
||||
let record =
|
||||
podcast_relay::progress_record(pool, user_id, episode.episode_id.as_str()).await?;
|
||||
let record = PodcastStore::new(pool.clone())
|
||||
.progress(user_id, &episode.episode_id)
|
||||
.await?;
|
||||
apply_resume_position(&mut wire, record.as_ref());
|
||||
}
|
||||
Ok(wire)
|
||||
@@ -4644,7 +4655,14 @@ async fn snapshot(pool: &sqlx::PgPool, user_id: i64) -> Result<SyncSnapshot> {
|
||||
removed_playlist_items,
|
||||
..SyncSnapshot::default()
|
||||
};
|
||||
podcast_relay::fill_snapshot(pool, user_id, &mut snapshot).await?;
|
||||
// Bounded again per message by `fit_podcast_snapshot`.
|
||||
PodcastStore::new(pool.clone())
|
||||
.fill_snapshot(
|
||||
user_id,
|
||||
&mut snapshot,
|
||||
music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES,
|
||||
)
|
||||
.await?;
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
//! Device-sync v4 storage paths against PostgreSQL.
|
||||
//!
|
||||
//! Requires a migrated disposable database (start the server once against
|
||||
//! it); the test writes rows for a random user id and removes them again.
|
||||
//! Requires a disposable database in `FURU_DEVICE_SYNC_TEST_DATABASE_URL`;
|
||||
//! it is migrated with the production migrations on first use
|
||||
//! (`crate::podcasts::postgres_tests::test_pool`). The tests write rows for
|
||||
//! random user ids and remove them again.
|
||||
|
||||
use super::*;
|
||||
use crate::player::queue_items::test_episode;
|
||||
@@ -48,6 +50,14 @@ async fn cleanup(pool: &sqlx::PgPool, user_id: i64) {
|
||||
"furumusic__fed_sync_ops",
|
||||
"furumusic__fed_sync_vector",
|
||||
"furumusic__playback_state",
|
||||
"furumusic__podcast_fetch_state",
|
||||
"furumusic__podcast_listen_session",
|
||||
"furumusic__podcast_progress",
|
||||
"furumusic__podcast_share",
|
||||
"furumusic__podcast_subscription",
|
||||
"furumusic__podcast_episode",
|
||||
"furumusic__podcast_feed_alias",
|
||||
"furumusic__podcast",
|
||||
] {
|
||||
sqlx::query(&format!("DELETE FROM {table} WHERE user_id = $1"))
|
||||
.bind(user_id)
|
||||
@@ -58,11 +68,9 @@ async fn cleanup(pool: &sqlx::PgPool, user_id: i64) {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a migrated disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
async fn postgres_podcast_relay_gating_and_peer_versions() {
|
||||
let pool = sqlx::PgPool::connect(&std::env::var("FURU_DEVICE_SYNC_TEST_DATABASE_URL").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let pool = crate::podcasts::postgres_tests::test_pool().await;
|
||||
let user_id = 8_000_000_000 + i64::from(rand_suffix());
|
||||
cleanup(&pool, user_id).await;
|
||||
let new = format!("tui_new_{user_id}");
|
||||
@@ -320,11 +328,9 @@ async fn postgres_podcast_relay_gating_and_peer_versions() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a migrated disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
async fn postgres_unknown_episode_position_and_rollback_cleanup() {
|
||||
let pool = sqlx::PgPool::connect(&std::env::var("FURU_DEVICE_SYNC_TEST_DATABASE_URL").unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let pool = crate::podcasts::postgres_tests::test_pool().await;
|
||||
let user_id = 8_000_000_000 + i64::from(rand_suffix());
|
||||
cleanup(&pool, user_id).await;
|
||||
let tui = format!("tui_{user_id}");
|
||||
@@ -469,6 +475,92 @@ async fn postgres_unknown_episode_position_and_rollback_cleanup() {
|
||||
cleanup(&pool, user_id).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
|
||||
async fn postgres_local_podcast_actions_reach_v4_peers_only() {
|
||||
let pool = crate::podcasts::postgres_tests::test_pool().await;
|
||||
let user_id = 8_000_000_000 + i64::from(rand_suffix());
|
||||
cleanup(&pool, user_id).await;
|
||||
let identity = ensure_identity(&pool, user_id, "tester").await.unwrap();
|
||||
let old = format!("tui_old_{user_id}");
|
||||
let new = format!("tui_new_{user_id}");
|
||||
for (device, version) in [(&old, 3), (&new, 4)] {
|
||||
apply_device_profile(
|
||||
&pool,
|
||||
user_id,
|
||||
&profile(device, version),
|
||||
VersionReport::SelfReported(Some(version)),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// A local action lands in the log with this device's stamp and in the
|
||||
// store, in one transaction.
|
||||
let store = crate::podcasts::PodcastStore::new(pool.clone());
|
||||
let episode = test_episode();
|
||||
let meta: PodcastMeta = serde_json::from_value(serde_json::json!({
|
||||
"feed_url": episode.feed_url.as_str(), "title": episode.podcast_title,
|
||||
}))
|
||||
.unwrap();
|
||||
let subscribed = store.subscribe(user_id, &meta).await.unwrap();
|
||||
let played = store.mark_played(user_id, &episode).await.unwrap();
|
||||
assert!(
|
||||
subscribed
|
||||
.ops
|
||||
.iter()
|
||||
.chain([&played.op])
|
||||
.all(|op| op.origin_device_id == identity.device_id)
|
||||
);
|
||||
// A music operation recorded afterwards keeps the same clock and sequence.
|
||||
record_local_op(
|
||||
&pool,
|
||||
user_id,
|
||||
SyncOpPayload::TrackLikeSet {
|
||||
content_id: format!("b3:{}", "5".repeat(64)),
|
||||
liked: false,
|
||||
fed: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let own: Vec<SyncOpWire> = ops_for_peer(&pool, user_id, &new, 4)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter(|op| op.origin_device_id == identity.device_id)
|
||||
.collect();
|
||||
let kinds: Vec<_> = own.iter().map(|op| payload_kind(&op.payload)).collect();
|
||||
assert!(kinds.ends_with(&[
|
||||
"podcast_subscription_set",
|
||||
"podcast_progress_set",
|
||||
"track_like_set"
|
||||
]));
|
||||
assert!(
|
||||
own.windows(2)
|
||||
.all(|pair| pair[1].seq > pair[0].seq && pair[1].hlc_ms > pair[0].hlc_ms)
|
||||
);
|
||||
assert_eq!(own[own.len() - 3], subscribed.ops[0]);
|
||||
assert_eq!(own[own.len() - 2], played.op);
|
||||
// The v3 peer gets the music operation only.
|
||||
let to_old = ops_for_peer(&pool, user_id, &old, 3).await.unwrap();
|
||||
assert!(to_old.iter().all(|op| !op.payload.is_podcast()));
|
||||
assert!(
|
||||
to_old
|
||||
.iter()
|
||||
.any(|op| payload_kind(&op.payload) == "track_like_set")
|
||||
);
|
||||
// The snapshot carries the store's registers for v4 only.
|
||||
let full = snapshot(&pool, user_id).await.unwrap();
|
||||
assert_eq!(full.podcast_subscriptions.len(), 1);
|
||||
assert_eq!(full.podcast_progress.len(), 1);
|
||||
assert_eq!(full.podcast_progress[0].status, ProgressStatus::Completed);
|
||||
assert!(!full.for_peer(3).has_podcast_state());
|
||||
|
||||
cleanup(&pool, user_id).await;
|
||||
}
|
||||
|
||||
fn rand_suffix() -> u32 {
|
||||
u32::from_le_bytes(
|
||||
blake3::hash(random_hex(8).as_bytes()).as_bytes()[..4]
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
mod capabilities;
|
||||
pub mod client;
|
||||
pub mod devices;
|
||||
mod podcast_relay;
|
||||
mod receive;
|
||||
mod serve;
|
||||
mod similarity;
|
||||
|
||||
@@ -1,575 +0,0 @@
|
||||
//! Lossless relay of podcast sync state (device-sync protocol v4).
|
||||
//!
|
||||
//! The web server does not implement podcasts yet, but it announces protocol
|
||||
//! v4 and therefore must not lose podcast state that trusted devices send
|
||||
//! through it:
|
||||
//!
|
||||
//! * podcast operations are stored verbatim in the generic operation log
|
||||
//! (`furumusic__fed_sync_ops`) and forwarded to v4 peers only;
|
||||
//! * the podcast registers carried by those operations and by snapshot
|
||||
//! sections are kept here as opaque JSON documents, merged with the shared
|
||||
//! `music_dht::podcast` rules, and returned in this device's snapshots.
|
||||
//! Snapshots are the only recovery path for a peer that skipped podcast
|
||||
//! operations while it ran an older version.
|
||||
//!
|
||||
//! The rows hold the shared contract types (`SubscriptionRecord`,
|
||||
//! `ShareRecord`, `ProgressRecord`, `ListenSession`) as JSON and nothing
|
||||
//! web-specific; a later podcast store can import them.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::{SyncOpWire, SyncSnapshot};
|
||||
use music_dht::podcast::{
|
||||
FeedUrl, ListenSession, PodcastId, ProgressRecord, ShareRecord, SubscriptionRecord,
|
||||
merge_feed_aliases, merge_listen_session, merge_progress, merge_share, merge_subscription,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::Value;
|
||||
use sqlx::Row;
|
||||
|
||||
const SECTION_SUBSCRIPTION: &str = "subscription";
|
||||
const SECTION_SHARE: &str = "share";
|
||||
const SECTION_ALIASES: &str = "aliases";
|
||||
const SECTION_PROGRESS: &str = "progress";
|
||||
const SECTION_SESSION: &str = "session";
|
||||
|
||||
/// One podcast register update, keyed by its entity.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) enum RelayRecord {
|
||||
/// Subscription register of a podcast.
|
||||
Subscription(SubscriptionRecord),
|
||||
/// Share register of a podcast.
|
||||
Share(ShareRecord),
|
||||
/// Feed aliases learned before (or without) the subscription register.
|
||||
Aliases(PodcastId, Vec<FeedUrl>),
|
||||
/// Progress register of an episode.
|
||||
Progress(ProgressRecord),
|
||||
/// Listening session.
|
||||
Session(ListenSession),
|
||||
}
|
||||
|
||||
impl RelayRecord {
|
||||
fn section(&self) -> &'static str {
|
||||
match self {
|
||||
RelayRecord::Subscription(_) => SECTION_SUBSCRIPTION,
|
||||
RelayRecord::Share(_) => SECTION_SHARE,
|
||||
RelayRecord::Aliases(..) => SECTION_ALIASES,
|
||||
RelayRecord::Progress(_) => SECTION_PROGRESS,
|
||||
RelayRecord::Session(_) => SECTION_SESSION,
|
||||
}
|
||||
}
|
||||
|
||||
fn key(&self) -> &str {
|
||||
match self {
|
||||
RelayRecord::Subscription(record) => record.podcast_id.as_str(),
|
||||
RelayRecord::Share(record) => record.podcast_id.as_str(),
|
||||
RelayRecord::Aliases(podcast_id, _) => podcast_id.as_str(),
|
||||
RelayRecord::Progress(record) => record.episode_id.as_str(),
|
||||
RelayRecord::Session(session) => session.session_id.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register updates carried by a podcast operation.
|
||||
///
|
||||
/// Operations that fail `validate_podcast` must be dropped before this.
|
||||
pub(crate) fn from_op(op: &SyncOpWire) -> Option<Self> {
|
||||
if let Some(record) = op.podcast_subscription() {
|
||||
return Some(RelayRecord::Subscription(record));
|
||||
}
|
||||
if let Some(record) = op.podcast_share() {
|
||||
return Some(RelayRecord::Share(record));
|
||||
}
|
||||
if let Some((podcast_id, aliases)) = op.podcast_feed_aliases() {
|
||||
return Some(RelayRecord::Aliases(podcast_id.clone(), aliases.to_vec()));
|
||||
}
|
||||
if let Some(record) = op.podcast_progress() {
|
||||
return Some(RelayRecord::Progress(record));
|
||||
}
|
||||
op.podcast_listen_session()
|
||||
.map(|session| RelayRecord::Session(session.clone()))
|
||||
}
|
||||
|
||||
/// Valid register records of a snapshot. Podcast sections that exceed
|
||||
/// the contract counts are ignored as a whole; invalid records are
|
||||
/// skipped one by one.
|
||||
pub(crate) fn from_snapshot(snapshot: &SyncSnapshot) -> Vec<Self> {
|
||||
if !snapshot.podcast_state_within_limits() {
|
||||
tracing::debug!("ignored podcast snapshot sections above the contract limits");
|
||||
return Vec::new();
|
||||
}
|
||||
let subscriptions = snapshot
|
||||
.podcast_subscriptions
|
||||
.iter()
|
||||
.filter(|record| record.validate().is_ok())
|
||||
.cloned()
|
||||
.map(RelayRecord::Subscription);
|
||||
let shares = snapshot
|
||||
.podcast_shares
|
||||
.iter()
|
||||
.filter(|record| record.validate().is_ok())
|
||||
.cloned()
|
||||
.map(RelayRecord::Share);
|
||||
let progress = snapshot
|
||||
.podcast_progress
|
||||
.iter()
|
||||
.filter(|record| record.validate().is_ok())
|
||||
.cloned()
|
||||
.map(RelayRecord::Progress);
|
||||
let sessions = snapshot
|
||||
.podcast_listen_sessions
|
||||
.iter()
|
||||
.filter(|session| session.validate().is_ok())
|
||||
.cloned()
|
||||
.map(RelayRecord::Session);
|
||||
subscriptions
|
||||
.chain(shares)
|
||||
.chain(progress)
|
||||
.chain(sessions)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Document address: `(section, record_key)`.
|
||||
type DocKey = (&'static str, String);
|
||||
|
||||
/// Documents a record's merge reads.
|
||||
fn read_keys(record: &RelayRecord) -> Vec<DocKey> {
|
||||
let own = (record.section(), record.key().to_string());
|
||||
match record {
|
||||
RelayRecord::Aliases(podcast_id, _) => {
|
||||
vec![(SECTION_SUBSCRIPTION, podcast_id.as_str().to_string()), own]
|
||||
}
|
||||
RelayRecord::Subscription(_) => {
|
||||
vec![(SECTION_ALIASES, record.key().to_string()), own]
|
||||
}
|
||||
_ => vec![own],
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges `incoming` into the stored documents with the shared rules.
|
||||
///
|
||||
/// Returns the writes: `Some(value)` upserts, `None` deletes. Aliases
|
||||
/// learned before the subscription register are kept in their own document
|
||||
/// and folded into the register when it arrives.
|
||||
fn plan_merge(
|
||||
docs: &std::collections::BTreeMap<DocKey, Value>,
|
||||
incoming: &RelayRecord,
|
||||
) -> Vec<(DocKey, Option<Value>)> {
|
||||
let own = (incoming.section(), incoming.key().to_string());
|
||||
let current = docs.get(&own);
|
||||
let changed = |merged: Option<Value>| {
|
||||
merged
|
||||
.filter(|merged| current != Some(merged))
|
||||
.map(|merged| (own.clone(), Some(merged)))
|
||||
};
|
||||
match incoming {
|
||||
RelayRecord::Subscription(record) => {
|
||||
let alias_key = (SECTION_ALIASES, incoming.key().to_string());
|
||||
let pending = docs.get(&alias_key).and_then(parse::<Vec<FeedUrl>>);
|
||||
let merged = match current.and_then(parse::<SubscriptionRecord>) {
|
||||
Some(current) => merge_subscription(¤t, record).ok(),
|
||||
None => {
|
||||
let mut record = record.clone();
|
||||
record.normalize();
|
||||
Some(record)
|
||||
}
|
||||
}
|
||||
.map(|mut record| {
|
||||
if let Some(aliases) = &pending {
|
||||
record.add_feed_aliases(aliases);
|
||||
}
|
||||
record
|
||||
});
|
||||
let mut writes: Vec<_> = changed(merged.as_ref().and_then(to_value))
|
||||
.into_iter()
|
||||
.collect();
|
||||
if docs.contains_key(&alias_key) && merged.is_some() {
|
||||
writes.push((alias_key, None));
|
||||
}
|
||||
writes
|
||||
}
|
||||
RelayRecord::Aliases(podcast_id, aliases) => {
|
||||
let sub_key = (SECTION_SUBSCRIPTION, podcast_id.as_str().to_string());
|
||||
if let Some(subscription) = docs.get(&sub_key) {
|
||||
let Some(mut record) = parse::<SubscriptionRecord>(subscription) else {
|
||||
return Vec::new();
|
||||
};
|
||||
record.add_feed_aliases(aliases);
|
||||
return to_value(&record)
|
||||
.filter(|merged| merged != subscription)
|
||||
.map(|merged| vec![(sub_key, Some(merged))])
|
||||
.unwrap_or_default();
|
||||
}
|
||||
let current = current.and_then(parse::<Vec<FeedUrl>>).unwrap_or_default();
|
||||
changed(to_value(&merge_feed_aliases(podcast_id, ¤t, aliases)))
|
||||
.into_iter()
|
||||
.collect()
|
||||
}
|
||||
RelayRecord::Share(record) => changed(match current.and_then(parse::<ShareRecord>) {
|
||||
Some(current) => merge_share(¤t, record)
|
||||
.ok()
|
||||
.as_ref()
|
||||
.and_then(to_value),
|
||||
None => to_value(record),
|
||||
})
|
||||
.into_iter()
|
||||
.collect(),
|
||||
RelayRecord::Progress(record) => changed(match current.and_then(parse::<ProgressRecord>) {
|
||||
Some(current) => merge_progress(¤t, record)
|
||||
.ok()
|
||||
.as_ref()
|
||||
.and_then(to_value),
|
||||
None => to_value(record),
|
||||
})
|
||||
.into_iter()
|
||||
.collect(),
|
||||
RelayRecord::Session(session) => changed(match current.and_then(parse::<ListenSession>) {
|
||||
Some(current) => merge_listen_session(¤t, session)
|
||||
.ok()
|
||||
.as_ref()
|
||||
.and_then(to_value),
|
||||
None => to_value(session),
|
||||
})
|
||||
.into_iter()
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse<T: DeserializeOwned>(value: &Value) -> Option<T> {
|
||||
serde_json::from_value(value.clone()).ok()
|
||||
}
|
||||
|
||||
fn to_value<T: Serialize>(value: &T) -> Option<Value> {
|
||||
serde_json::to_value(value).ok()
|
||||
}
|
||||
|
||||
/// Merges register updates into the user's relay state.
|
||||
pub(crate) async fn merge_records(
|
||||
pool: &sqlx::PgPool,
|
||||
user_id: i64,
|
||||
records: &[RelayRecord],
|
||||
) -> Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut tx = pool.begin().await?;
|
||||
// One writer per account: a merge reads, combines and writes documents.
|
||||
sqlx::query(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcast-relay:' || $1::text, 0))",
|
||||
)
|
||||
.bind(user_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let mut docs = std::collections::BTreeMap::new();
|
||||
for record in records {
|
||||
for key in read_keys(record) {
|
||||
if !docs.contains_key(&key)
|
||||
&& let Some(value) = load(&mut tx, user_id, key.0, &key.1).await?
|
||||
{
|
||||
docs.insert(key, value);
|
||||
}
|
||||
}
|
||||
for (key, value) in plan_merge(&docs, record) {
|
||||
match &value {
|
||||
Some(value) => store(&mut tx, user_id, key.0, &key.1, value).await?,
|
||||
None => {
|
||||
sqlx::query(
|
||||
"DELETE FROM furumusic__fed_podcast_relay
|
||||
WHERE user_id = $1 AND section = $2 AND record_key = $3",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(key.0)
|
||||
.bind(&key.1)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
match value {
|
||||
Some(value) => docs.insert(key, value),
|
||||
None => docs.remove(&key),
|
||||
};
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
user_id: i64,
|
||||
section: &str,
|
||||
key: &str,
|
||||
) -> Result<Option<Value>> {
|
||||
Ok(sqlx::query_scalar(
|
||||
"SELECT record_json FROM furumusic__fed_podcast_relay
|
||||
WHERE user_id = $1 AND section = $2 AND record_key = $3",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(section)
|
||||
.bind(key)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn store(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
user_id: i64,
|
||||
section: &str,
|
||||
key: &str,
|
||||
value: &Value,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
"INSERT INTO furumusic__fed_podcast_relay
|
||||
(user_id, section, record_key, record_json, updated_at_ms)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (user_id, section, record_key) DO UPDATE SET
|
||||
record_json = EXCLUDED.record_json,
|
||||
updated_at_ms = EXCLUDED.updated_at_ms",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(section)
|
||||
.bind(key)
|
||||
.bind(value)
|
||||
.bind(chrono::Utc::now().timestamp_millis())
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Relayed progress register of an episode, if any.
|
||||
pub(crate) async fn progress_record(
|
||||
pool: &sqlx::PgPool,
|
||||
user_id: i64,
|
||||
episode_id: &str,
|
||||
) -> Result<Option<ProgressRecord>> {
|
||||
let value: Option<Value> = sqlx::query_scalar(
|
||||
"SELECT record_json FROM furumusic__fed_podcast_relay
|
||||
WHERE user_id = $1 AND section = $2 AND record_key = $3",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(SECTION_PROGRESS)
|
||||
.bind(episode_id)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(value.as_ref().and_then(parse))
|
||||
}
|
||||
|
||||
/// Adds the relayed podcast registers to `snapshot`.
|
||||
///
|
||||
/// The sections are bounded later, per message, with the contract budget
|
||||
/// (`SyncSnapshot::bound_podcast_state_to`) and stripped for peers below v4
|
||||
/// (`SyncSnapshot::for_peer`).
|
||||
pub(crate) async fn fill_snapshot(
|
||||
pool: &sqlx::PgPool,
|
||||
user_id: i64,
|
||||
snapshot: &mut SyncSnapshot,
|
||||
) -> Result<()> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT section, record_json FROM furumusic__fed_podcast_relay
|
||||
WHERE user_id = $1 AND section <> $2
|
||||
ORDER BY section, record_key",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(SECTION_ALIASES)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
for row in rows {
|
||||
let section: String = row.get("section");
|
||||
let value: Value = row.get("record_json");
|
||||
fill_section(snapshot, §ion, &value);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn fill_section(snapshot: &mut SyncSnapshot, section: &str, value: &Value) {
|
||||
match section {
|
||||
SECTION_SUBSCRIPTION => snapshot.podcast_subscriptions.extend(parse(value)),
|
||||
SECTION_SHARE => snapshot.podcast_shares.extend(parse(value)),
|
||||
SECTION_PROGRESS => snapshot.podcast_progress.extend(parse(value)),
|
||||
SECTION_SESSION => snapshot.podcast_listen_sessions.extend(parse(value)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
|
||||
use music_dht::podcast::{EpisodeId, PodcastMeta, ProgressStatus};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn feed(path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap()
|
||||
}
|
||||
|
||||
fn meta() -> PodcastMeta {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"feed_url": feed("feed.xml").as_str(),
|
||||
"title": "Example Cast",
|
||||
"description": "Long channel description",
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn podcast() -> PodcastId {
|
||||
PodcastId::from_feed_url(&feed("feed.xml"))
|
||||
}
|
||||
|
||||
fn op(seq: i64, hlc_ms: i64, payload: SyncOpPayload) -> SyncOpWire {
|
||||
SyncOpWire {
|
||||
op_id: format!("tui_a:{seq}"),
|
||||
origin_device_id: "tui_a".into(),
|
||||
seq,
|
||||
hlc_ms,
|
||||
payload,
|
||||
}
|
||||
}
|
||||
|
||||
fn progress(seq: i64, generation: u64, hlc_ms: i64, position_ms: i64) -> SyncOpWire {
|
||||
let podcast_id = podcast();
|
||||
op(
|
||||
seq,
|
||||
hlc_ms,
|
||||
SyncOpPayload::PodcastProgressSet {
|
||||
episode_id: EpisodeId::from_guid(&podcast_id, "ep-1").unwrap(),
|
||||
podcast_id,
|
||||
position_ms,
|
||||
status: ProgressStatus::InProgress,
|
||||
generation,
|
||||
session_id: format!("s{generation}"),
|
||||
episode: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Applies records to an in-memory table with the production plan.
|
||||
fn apply(state: &mut BTreeMap<DocKey, Value>, records: &[RelayRecord]) {
|
||||
for record in records {
|
||||
for (key, value) in plan_merge(state, record) {
|
||||
match value {
|
||||
Some(value) => state.insert(key, value),
|
||||
None => state.remove(&key),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_of(state: &BTreeMap<DocKey, Value>) -> SyncSnapshot {
|
||||
let mut snapshot = SyncSnapshot::default();
|
||||
for ((section, _), value) in state {
|
||||
fill_section(&mut snapshot, section, value);
|
||||
}
|
||||
snapshot
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_registers_follow_the_contract_in_any_order() {
|
||||
// A late checkpoint of an older generation never wins, whatever its clock.
|
||||
let ops = [
|
||||
progress(1, 3, 9_000, 600_000),
|
||||
progress(2, 4, 1_000, 10_000),
|
||||
];
|
||||
for order in [[0, 1], [1, 0]] {
|
||||
let mut state = BTreeMap::new();
|
||||
for index in order {
|
||||
apply(&mut state, &[RelayRecord::from_op(&ops[index]).unwrap()]);
|
||||
}
|
||||
let snapshot = snapshot_of(&state);
|
||||
assert_eq!(snapshot.podcast_progress.len(), 1);
|
||||
assert_eq!(snapshot.podcast_progress[0].position_ms, 10_000);
|
||||
assert_eq!(snapshot.podcast_progress[0].generation, 4);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_sections_survive_the_relay() {
|
||||
let subscribe = op(
|
||||
1,
|
||||
10,
|
||||
SyncOpPayload::PodcastSubscriptionSet {
|
||||
podcast_id: podcast(),
|
||||
subscribed: true,
|
||||
podcast: meta(),
|
||||
feed_aliases: vec![],
|
||||
generation: 1,
|
||||
},
|
||||
);
|
||||
let share = op(
|
||||
2,
|
||||
11,
|
||||
SyncOpPayload::PodcastShareSet {
|
||||
podcast_id: podcast(),
|
||||
shared: false,
|
||||
generation: 1,
|
||||
},
|
||||
);
|
||||
let mut source = BTreeMap::new();
|
||||
for op in [&subscribe, &share, &progress(3, 2, 12, 5_000)] {
|
||||
op.validate_podcast().unwrap();
|
||||
apply(&mut source, &[RelayRecord::from_op(op).unwrap()]);
|
||||
}
|
||||
let mut sent = snapshot_of(&source);
|
||||
sent.bound_podcast_state();
|
||||
// A second relay that only saw the snapshot re-emits the same state.
|
||||
let mut relay = BTreeMap::new();
|
||||
apply(&mut relay, &RelayRecord::from_snapshot(&sent));
|
||||
let mut relayed = snapshot_of(&relay);
|
||||
relayed.bound_podcast_state();
|
||||
assert_eq!(relayed, sent);
|
||||
assert!(!relayed.podcast_shares[0].shared);
|
||||
assert!(relayed.podcast_subscriptions[0].subscribed);
|
||||
// Peers below v4 never receive the sections.
|
||||
assert!(!relayed.for_peer(3).has_podcast_state());
|
||||
assert!(relayed.for_peer(4).has_podcast_state());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_learned_before_the_subscription_are_merged_into_it() {
|
||||
let aliases = op(
|
||||
1,
|
||||
10,
|
||||
SyncOpPayload::PodcastFeedAliasesAdded {
|
||||
podcast_id: podcast(),
|
||||
feed_aliases: vec![feed("moved.xml")],
|
||||
},
|
||||
);
|
||||
let subscribe = op(
|
||||
2,
|
||||
11,
|
||||
SyncOpPayload::PodcastSubscriptionSet {
|
||||
podcast_id: podcast(),
|
||||
subscribed: true,
|
||||
podcast: meta(),
|
||||
feed_aliases: vec![],
|
||||
generation: 1,
|
||||
},
|
||||
);
|
||||
let mut state = BTreeMap::new();
|
||||
apply(&mut state, &[RelayRecord::from_op(&aliases).unwrap()]);
|
||||
apply(&mut state, &[RelayRecord::from_op(&subscribe).unwrap()]);
|
||||
let snapshot = snapshot_of(&state);
|
||||
assert_eq!(
|
||||
snapshot.podcast_subscriptions[0].feed_urls(),
|
||||
vec![feed("feed.xml"), feed("moved.xml")]
|
||||
);
|
||||
assert!(!state.keys().any(|(section, _)| *section == SECTION_ALIASES));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_snapshot_sections_are_ignored() {
|
||||
let mut snapshot = SyncSnapshot::default();
|
||||
let record = RelayRecord::from_op(&progress(1, 1, 1, 1)).unwrap();
|
||||
let RelayRecord::Progress(record) = record else {
|
||||
unreachable!()
|
||||
};
|
||||
snapshot.podcast_progress = vec![record; music_dht::podcast::MAX_SNAPSHOT_PROGRESS + 1];
|
||||
assert!(RelayRecord::from_snapshot(&snapshot).is_empty());
|
||||
snapshot
|
||||
.podcast_progress
|
||||
.truncate(music_dht::podcast::MAX_SNAPSHOT_PROGRESS);
|
||||
assert_eq!(
|
||||
RelayRecord::from_snapshot(&snapshot).len(),
|
||||
music_dht::podcast::MAX_SNAPSHOT_PROGRESS
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user