//! Receiving side of music federation. //! //! User-facing identity is content-addressed. An `(owner, item_id)` pair is //! only a source locator and several locators may resolve the same track. use std::collections::HashMap; use anyhow::{Context, Result}; use music_dht::{ItemKind, LibraryItem, normalize_content_id}; use serde::Serialize; use serde_json::{Value, json}; use sqlx::PgPool; use sqlx::Row as _; use tokio::io::AsyncReadExt; use super::{Federation, now_iso}; const MAX_CATALOG_BYTES: u64 = 4 * 1024 * 1024; #[derive(Debug, Clone, Serialize)] pub struct TrackKeyDto { pub content_id: String, } #[derive(Debug, Clone, Serialize)] pub struct ArtistKeyDto { pub normalized_name: String, } #[derive(Debug, Clone, Serialize)] pub struct ArtistRefDto { pub key: ArtistKeyDto, pub name: String, pub local_id: Option, } #[derive(Debug, Clone, Serialize)] pub struct ReleaseKeyDto { pub normalized_title: String, pub primary_artists: Vec, pub release_type: Option, pub year: Option, } #[derive(Debug, Clone, Serialize)] pub struct ReleaseRefDto { pub key: ReleaseKeyDto, pub local_id: Option, pub title: String, } #[derive(Debug, Clone, Serialize)] pub struct FederationSourceDto { pub owner: String, pub item_id: String, } #[derive(Debug, Clone, Serialize)] pub struct LocalAvailabilityDto { pub track_id: i64, pub stream_url: String, } #[derive(Debug, Clone, Serialize)] pub struct TrackMetadataDto { pub title: String, pub artists: Vec, pub featured_artists: Vec, pub release: Option, pub year: Option, pub duration_seconds: Option, pub track_number: Option, pub disc_number: Option, pub cover_url: Option, } #[derive(Debug, Clone, Serialize)] pub struct TrackAvailabilityDto { pub state: &'static str, pub local: Option, pub federation: Vec, } #[derive(Debug, Clone, Serialize)] pub struct TrackDto { pub key: TrackKeyDto, pub metadata: TrackMetadataDto, pub availability: TrackAvailabilityDto, #[serde(skip_serializing_if = "Option::is_none")] pub similarity_score: Option, } #[derive(Debug, Clone, Serialize)] pub struct SearchEvent { pub search_id: String, pub sequence: u64, pub kind: &'static str, /// `music` or `podcast` (`music_dht::podcast_dht::FederatedSearchHit`). #[serde(rename = "type")] pub hit_type: &'static str, pub peer: Option, pub entity_key: Value, pub entity: Value, } impl Federation { pub async fn prepare_similarity_tracks( &self, tracks: Vec, ) -> Result> { let pool = self.pool().await?; let mut prepared = Vec::new(); for track in tracks { let Some(content_id) = track.content_id.as_deref().and_then(normalize_content_id) else { continue; }; let local = local_availability(&pool, &content_id).await?; // A local result is already present in the first result section. if local.is_some() { continue; } let owner = track.owner; let item_id = track.item_id; let dto = TrackDto { key: TrackKeyDto { content_id: content_id.clone(), }, metadata: TrackMetadataDto { title: track.title, artists: artist_refs(&track.artist_names), featured_artists: artist_refs(&track.featured_artist_names), release: track.release_title.map(|title| ReleaseRefDto { key: ReleaseKeyDto { normalized_title: music_dht::normalize_name(&title), primary_artists: track .artist_names .iter() .map(|artist| music_dht::normalize_name(artist)) .collect(), release_type: None, year: track.year, }, local_id: None, title, }), year: track.year, duration_seconds: track.duration_seconds.map(|value| value as f64), track_number: track.track_number, disc_number: track.disc_number, cover_url: Some(format!( "/api/player/federation/tracks/artwork?owner={owner}&item_id={item_id}" )), }, availability: TrackAvailabilityDto { state: "federated", local: None, federation: vec![FederationSourceDto { owner, item_id }], }, similarity_score: Some(track.similarity_score), }; persist_track_ref(&pool, &dto).await?; prepared.push(dto); } Ok(prepared) } pub fn stream_artist_catalogs( self: &std::sync::Arc, name: String, ) -> tokio::sync::mpsc::UnboundedReceiver> { let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); let federation = std::sync::Arc::clone(self); tokio::spawn(async move { let result = async { let service = federation.service().await?; let normalized = music_dht::normalize_name(&name); let outcome = service .search_network(&name) .await .map_err(|err| anyhow::anyhow!("federated artist search failed: {err}"))?; let owners: std::collections::HashSet<_> = outcome .network_results .iter() .filter(|item| { (item.kind == ItemKind::Artist && item.normalized_name == normalized) || item .artist_names .iter() .chain(&item.featured_artist_names) .any(|artist| music_dht::normalize_name(artist) == normalized) }) .map(|item| item.owner) .collect(); for owner in owners { let service = std::sync::Arc::clone(&service); let sender = sender.clone(); let name = name.clone(); tokio::spawn(async move { let result = tokio::time::timeout( std::time::Duration::from_secs(8), fetch_artist_catalog(&service, owner, &name), ) .await .map_err(|_| anyhow::anyhow!("catalog request timed out")) .and_then(|result| result) .map(|artist| (owner.to_string(), artist)); let _ = sender.send(result); }); } Ok::<(), anyhow::Error>(()) } .await; if let Err(err) = result { let _ = sender.send(Err(err)); } }); receiver } /// Performs one bounded DHT search and returns entity upserts. The HTTP /// layer streams each upsert independently; catalog fan-out can append /// events to the same contract without changing the browser model. /// /// Music only (`federation.track|artist|release`, marked `"type": /// "music"`). Public podcasts are searched next to it /// ([`Self::spawn_podcast_search`]) and follow as `federation.podcast` /// events ([`Self::podcast_search_events`]). pub async fn search_events(&self, search_id: &str, query: &str) -> Result { let query = query.trim(); anyhow::ensure!(!query.is_empty(), "search query is empty"); anyhow::ensure!(query.chars().count() <= 200, "search query is too long"); let started = std::time::Instant::now(); let service = self.service().await?; tracing::info!( search_id, query, connected_peers = service.connected_peers().len(), known_contacts = service.known_peers().len(), "federated search started" ); let own = service.endpoint_id(); let result = tokio::time::timeout(SEARCH_TIMEOUT, service.search_network(query)) .await .map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))? .map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?; tracing::info!( search_id, query, local_results = result.local_results.len(), network_results = result.network_results.len(), queried_nodes = result.queried_nodes, elapsed_ms = started.elapsed().as_millis() as u64, "federated DHT search finished" ); let items: Vec = result .local_results .into_iter() .chain(result.network_results) .collect(); let pool = self.pool().await?; let events = music_search_events(&pool, search_id, &items, own).await?; tracing::info!( search_id, query, events = events.len(), elapsed_ms = started.elapsed().as_millis() as u64, "federated search response ready" ); Ok(MusicSearch { events, items }) } /// Starts the public podcast search of `query` on the discovery /// overlay, if it runs, next to the music search. pub fn spawn_podcast_search(&self, query: &str) -> tokio::task::JoinHandle { let discovery = self.podcast_discovery(); let query = query.trim().to_owned(); tokio::spawn(async move { search_podcasts(discovery.as_deref(), &query).await }) } /// The `federation.podcast` events of a search for `user_id`, numbered /// after the music events (see [`podcast_events`]). pub async fn podcast_search_events( &self, search_id: &str, query: &str, user_id: i64, music: &MusicSearch, podcasts: PodcastSearch, ) -> Result> { let PodcastSearch::Found(hits) = podcasts else { return Ok(Vec::new()); }; let pool = self.pool().await?; let first_sequence = music.events.len() as u64 + 1; let events = podcast_events( &pool, user_id, search_id, query.trim(), &music.items, first_sequence, hits, ) .await?; tracing::info!( search_id, podcasts = events.len(), "public podcast hits ready" ); Ok(events) } } /// Deadline of the music DHT and the podcast overlay searches. const SEARCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); /// Music results of a federated search: the events to stream and the items /// (podcast hits are ranked among them). #[derive(Debug)] pub struct MusicSearch { pub events: Vec, pub items: Vec, } /// Outcome of the public podcast search of one federated search. #[derive(Debug)] pub enum PodcastSearch { /// Public discovery does not run on this server. Off, /// The overlay search failed or timed out (music results still count). Failed, /// Hits, possibly none. Found(Vec), } impl PodcastSearch { /// `running`, `off` or `failed` (reported in `search.completed`). pub fn state(&self) -> &'static str { match self { Self::Off => "off", Self::Failed => "failed", Self::Found(_) => "running", } } } async fn search_podcasts( discovery: Option<&music_dht::podcast_dht::PodcastDht>, query: &str, ) -> PodcastSearch { let Some(dht) = discovery else { return PodcastSearch::Off; }; let limit = crate::podcasts::search::PODCAST_SEARCH_LIMIT; match tokio::time::timeout(SEARCH_TIMEOUT, dht.search(query, limit)).await { Ok(Ok(hits)) => PodcastSearch::Found(hits), Ok(Err(error)) => { tracing::info!("public podcast search failed: {error}"); PodcastSearch::Failed } Err(_) => { tracing::info!("public podcast search timed out"); PodcastSearch::Failed } } } /// One `federation.podcast` event per public podcast hit, for `user_id` /// (`subscribed`, `local_podcast_id`), ordered by the hit's place in /// `merge_search_results` with the music items and numbered from /// `first_sequence`. pub(crate) async fn podcast_events( pool: &PgPool, user_id: i64, search_id: &str, query: &str, music: &[LibraryItem], first_sequence: u64, hits: Vec, ) -> Result> { let store = crate::podcasts::PodcastStore::new(pool.clone()); let views = crate::podcasts::search::hit_views(&store, user_id, query, music, hits).await?; views .into_iter() .zip(first_sequence..) .map(|(view, sequence)| { Ok(SearchEvent { search_id: search_id.to_owned(), sequence, kind: "federation.podcast", hit_type: "podcast", // Subscribing never contacts a publisher: no peer to name. peer: None, entity_key: json!({ "podcast_id": view.podcast_id }), entity: serde_json::to_value(view)?, }) }) .collect() } /// Music events of a search (tracks, then artists and releases). pub(crate) async fn music_search_events( pool: &PgPool, search_id: &str, all_items: &[LibraryItem], own: music_dht::EndpointId, ) -> Result> { let mut by_content: HashMap = HashMap::new(); for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) { let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else { // A globally usable track reference must be verifiable. continue; }; let local = local_availability(pool, &content_id).await?; let source = FederationSourceDto { owner: item.owner.to_string(), item_id: hex(item.id.as_bytes()), }; let entry = by_content .entry(content_id.clone()) .or_insert_with(|| track_from_item(content_id.clone(), item, local, item.owner == own)); if !entry .availability .federation .iter() .any(|candidate| candidate.owner == source.owner && candidate.item_id == source.item_id) { entry.availability.federation.push(source); } if entry.availability.local.is_some() { entry.availability.state = "local"; } } let mut tracks: Vec<_> = by_content.into_values().collect(); tracks.sort_by(|left, right| { left.metadata .title .to_lowercase() .cmp(&right.metadata.title.to_lowercase()) }); let mut events = Vec::with_capacity(all_items.len()); for (index, track) in tracks.into_iter().enumerate() { persist_track_ref(pool, &track).await?; let peer = track .availability .federation .first() .map(|source| source.owner.clone()); events.push(SearchEvent { search_id: search_id.to_owned(), sequence: index as u64 + 1, kind: "federation.track", hit_type: "music", peer, entity_key: serde_json::to_value(&track.key)?, entity: serde_json::to_value(track)?, }); } let mut artist_peers: HashMap)> = HashMap::new(); let mut releases: HashMap = HashMap::new(); for item in all_items { match item.kind { ItemKind::Artist => { let key = music_dht::normalize_name(&item.name); let entry = artist_peers .entry(key) .or_insert_with(|| (item.name.clone(), Vec::new())); let owner = item.owner.to_string(); if !entry.1.contains(&owner) { entry.1.push(owner); } } ItemKind::Release => { let artist_keys: Vec = item .artist_names .iter() .map(|name| music_dht::normalize_name(name)) .collect(); let normalized_title = music_dht::normalize_name(&item.name); let cover_url = all_items .iter() .find(|track| { track.kind == ItemKind::Track && track.release_title.as_deref().is_some_and(|title| { music_dht::normalize_name(title) == normalized_title }) && track.year == item.year }) .map(|track| { format!( "/api/player/federation/tracks/artwork?owner={}&item_id={}", track.owner, hex(track.id.as_bytes()) ) }); let key = format!( "{}|{}|{}", normalized_title, artist_keys.join(","), item.year.map_or_else(String::new, |year| year.to_string()) ); releases.entry(key.clone()).or_insert_with(|| { json!({ "key": { "normalized_title": music_dht::normalize_name(&item.name), "primary_artists": artist_keys, "release_type": null, "year": item.year, }, "title": item.name, "artists": item.artist_names, "year": item.year, "cover_url": cover_url, "sources": [{ "owner": item.owner.to_string(), "item_id": hex(item.id.as_bytes()), }], }) }); } ItemKind::Track => {} } } for (key, (name, peers)) in artist_peers { let sequence = events.len() as u64 + 1; events.push(SearchEvent { search_id: search_id.to_owned(), sequence, kind: "federation.artist", hit_type: "music", peer: peers.first().cloned(), entity_key: json!({ "normalized_name": key }), entity: json!({ "key": { "normalized_name": key }, "name": name, "image_url": null, "peers": peers, }), }); } for (key, release) in releases { let sequence = events.len() as u64 + 1; events.push(SearchEvent { search_id: search_id.to_owned(), sequence, kind: "federation.release", hit_type: "music", peer: None, entity_key: json!({ "composite": key }), entity: release, }); } Ok(events) } async fn fetch_artist_catalog( service: &music_dht::MusicDhtService, owner: music_dht::EndpointId, artist: &str, ) -> Result { let mut stream = service .open_stream(owner, super::CATALOG_ALPN) .await .map_err(|err| anyhow::anyhow!("cannot reach catalog peer: {err}"))?; let mut request = serde_json::to_vec(&music_dht::catalog::CatalogRequest { artist: artist.to_owned(), want: Some("catalog".to_owned()), ..Default::default() })?; request.push(b'\n'); stream.send.write_all(&request).await?; stream.send.finish()?; let mut payload = Vec::new(); stream .recv .take(MAX_CATALOG_BYTES + 1) .read_to_end(&mut payload) .await?; anyhow::ensure!( payload.len() as u64 <= MAX_CATALOG_BYTES, "catalog response is too large" ); let response: music_dht::catalog::CatalogResponse = serde_json::from_slice(&payload).context("invalid catalog response")?; anyhow::ensure!( response.ok, "peer refused catalog: {}", response.error.unwrap_or_else(|| "unknown error".to_owned()) ); response.artist.context("peer returned no artist catalog") } fn track_from_item( content_id: String, item: &LibraryItem, local: Option, own: bool, ) -> TrackDto { let owner = item.owner.to_string(); let item_id = hex(item.id.as_bytes()); let artists = artist_refs(&item.artist_names); let featured_artists = artist_refs(&item.featured_artist_names); let release = item.release_title.as_ref().map(|title| ReleaseRefDto { key: ReleaseKeyDto { normalized_title: music_dht::normalize_name(title), primary_artists: item .artist_names .iter() .map(|artist| music_dht::normalize_name(artist)) .collect(), release_type: None, year: item.year, }, local_id: None, title: title.clone(), }); let state = if local.is_some() || own { "local" } else { "federated" }; TrackDto { key: TrackKeyDto { content_id }, metadata: TrackMetadataDto { title: item.name.clone(), artists, featured_artists, release, year: item.year, duration_seconds: item.duration_seconds, track_number: item.track_number, disc_number: item.disc_number, cover_url: Some(format!( "/api/player/federation/tracks/artwork?owner={owner}&item_id={item_id}" )), }, availability: TrackAvailabilityDto { state, local, federation: vec![FederationSourceDto { owner, item_id }], }, similarity_score: None, } } fn artist_refs(names: &[String]) -> Vec { names .iter() .map(|name| ArtistRefDto { key: ArtistKeyDto { normalized_name: music_dht::normalize_name(name), }, name: name.clone(), local_id: None, }) .collect() } async fn local_availability( pool: &sqlx::PgPool, content_id: &str, ) -> Result> { let row = sqlx::query( "SELECT t.id FROM furumusic__federation_content_id_cache c JOIN furumusic__track t ON t.audio_file_id = c.media_file_id WHERE c.content_id = $1 AND t.is_hidden = false LIMIT 1", ) .bind(content_id) .fetch_optional(pool) .await?; Ok(row.map(|row| { let track_id: i64 = row.get(0); LocalAvailabilityDto { track_id, stream_url: format!("/api/player/stream/{track_id}"), } })) } async fn persist_track_ref(pool: &sqlx::PgPool, track: &TrackDto) -> Result<()> { let metadata = serde_json::to_value(&track.metadata)?; let local_id = track .availability .local .as_ref() .map(|local| local.track_id); let row = sqlx::query( "INSERT INTO furumusic__track_ref (content_id, local_track_id, title, release_title, year, duration_seconds, metadata_json, metadata_authority, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, 'federation', $8, $8) ON CONFLICT (content_id) DO UPDATE SET local_track_id = COALESCE(furumusic__track_ref.local_track_id, EXCLUDED.local_track_id), title = EXCLUDED.title, release_title = EXCLUDED.release_title, year = EXCLUDED.year, duration_seconds = EXCLUDED.duration_seconds, metadata_json = EXCLUDED.metadata_json, updated_at = EXCLUDED.updated_at RETURNING id", ) .bind(&track.key.content_id) .bind(local_id) .bind(&track.metadata.title) .bind( track .metadata .release .as_ref() .map(|release| &release.title), ) .bind(track.metadata.year) .bind(track.metadata.duration_seconds) .bind(metadata) .bind(now_iso()) .fetch_one(pool) .await .context("persisting content-addressed track reference failed")?; let track_ref_id: i64 = row.get(0); for source in &track.availability.federation { sqlx::query( "INSERT INTO furumusic__federation_track_source (track_ref_id, owner_peer_id, item_id, last_seen_ms, metadata_json) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (owner_peer_id, item_id) DO UPDATE SET track_ref_id = EXCLUDED.track_ref_id, last_seen_ms = EXCLUDED.last_seen_ms, metadata_json = EXCLUDED.metadata_json", ) .bind(track_ref_id) .bind(&source.owner) .bind(&source.item_id) .bind(chrono::Utc::now().timestamp_millis()) .bind(json!({ "track": track.metadata })) .execute(pool) .await?; } Ok(()) } fn hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() }