diff --git a/src/federation/client.rs b/src/federation/client.rs index 7c36185..cc93363 100644 --- a/src/federation/client.rs +++ b/src/federation/client.rs @@ -9,6 +9,7 @@ use anyhow::{Context, Result}; use music_dht::{ItemKind, LibraryItem, normalize_content_id}; use serde::Serialize; use serde_json::{Value, json}; +use sqlx::PgPool; use sqlx::Row as _; use tokio::io::AsyncReadExt; @@ -94,6 +95,9 @@ pub struct SearchEvent { pub search_id: String, pub sequence: u64, pub kind: &'static str, + /// `music` or `podcast` (`music_dht::podcast_dht::FederatedSearchHit`). + #[serde(rename = "type")] + pub hit_type: &'static str, pub peer: Option, pub entity_key: Value, pub entity: Value, @@ -218,7 +222,12 @@ impl Federation { /// Performs one bounded DHT search and returns entity upserts. The HTTP /// layer streams each upsert independently; catalog fan-out can append /// events to the same contract without changing the browser model. - pub async fn search_events(&self, search_id: &str, query: &str) -> Result> { + /// + /// Music only (`federation.track|artist|release`, marked `"type": + /// "music"`). Public podcasts are searched next to it + /// ([`Self::spawn_podcast_search`]) and follow as `federation.podcast` + /// events ([`Self::podcast_search_events`]). + pub async fn search_events(&self, search_id: &str, query: &str) -> Result { let query = query.trim(); anyhow::ensure!(!query.is_empty(), "search query is empty"); anyhow::ensure!(query.chars().count() <= 200, "search query is too long"); @@ -233,13 +242,10 @@ impl Federation { "federated search started" ); let own = service.endpoint_id(); - let result = tokio::time::timeout( - std::time::Duration::from_secs(20), - service.search_network(query), - ) - .await - .map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))? - .map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?; + let result = tokio::time::timeout(SEARCH_TIMEOUT, service.search_network(query)) + .await + .map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))? + .map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?; tracing::info!( search_id, query, @@ -249,154 +255,14 @@ impl Federation { elapsed_ms = started.elapsed().as_millis() as u64, "federated DHT search finished" ); - let all_items: Vec = result + let items: Vec = result .local_results .into_iter() .chain(result.network_results) .collect(); let pool = self.pool().await?; - let mut by_content: HashMap = HashMap::new(); - for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) { - let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else { - // A globally usable track reference must be verifiable. - continue; - }; - let local = local_availability(&pool, &content_id).await?; - let source = FederationSourceDto { - owner: item.owner.to_string(), - item_id: hex(item.id.as_bytes()), - }; - let entry = by_content.entry(content_id.clone()).or_insert_with(|| { - track_from_item(content_id.clone(), item, local, item.owner == own) - }); - if !entry.availability.federation.iter().any(|candidate| { - candidate.owner == source.owner && candidate.item_id == source.item_id - }) { - entry.availability.federation.push(source); - } - if entry.availability.local.is_some() { - entry.availability.state = "local"; - } - } - - let mut tracks: Vec<_> = by_content.into_values().collect(); - tracks.sort_by(|left, right| { - left.metadata - .title - .to_lowercase() - .cmp(&right.metadata.title.to_lowercase()) - }); - - let mut events = Vec::with_capacity(all_items.len()); - for (index, track) in tracks.into_iter().enumerate() { - persist_track_ref(&pool, &track).await?; - let peer = track - .availability - .federation - .first() - .map(|source| source.owner.clone()); - events.push(SearchEvent { - search_id: search_id.to_owned(), - sequence: index as u64 + 1, - kind: "federation.track", - peer, - entity_key: serde_json::to_value(&track.key)?, - entity: serde_json::to_value(track)?, - }); - } - let mut artist_peers: HashMap)> = HashMap::new(); - let mut releases: HashMap = HashMap::new(); - for item in &all_items { - match item.kind { - ItemKind::Artist => { - let key = music_dht::normalize_name(&item.name); - let entry = artist_peers - .entry(key) - .or_insert_with(|| (item.name.clone(), Vec::new())); - let owner = item.owner.to_string(); - if !entry.1.contains(&owner) { - entry.1.push(owner); - } - } - ItemKind::Release => { - let artist_keys: Vec = item - .artist_names - .iter() - .map(|name| music_dht::normalize_name(name)) - .collect(); - let normalized_title = music_dht::normalize_name(&item.name); - let cover_url = all_items - .iter() - .find(|track| { - track.kind == ItemKind::Track - && track.release_title.as_deref().is_some_and(|title| { - music_dht::normalize_name(title) == normalized_title - }) - && track.year == item.year - }) - .map(|track| { - format!( - "/api/player/federation/tracks/artwork?owner={}&item_id={}", - track.owner, - hex(track.id.as_bytes()) - ) - }); - let key = format!( - "{}|{}|{}", - normalized_title, - artist_keys.join(","), - item.year.map_or_else(String::new, |year| year.to_string()) - ); - releases.entry(key.clone()).or_insert_with(|| { - json!({ - "key": { - "normalized_title": music_dht::normalize_name(&item.name), - "primary_artists": artist_keys, - "release_type": null, - "year": item.year, - }, - "title": item.name, - "artists": item.artist_names, - "year": item.year, - "cover_url": cover_url, - "sources": [{ - "owner": item.owner.to_string(), - "item_id": hex(item.id.as_bytes()), - }], - }) - }); - } - ItemKind::Track => {} - } - } - for (key, (name, peers)) in artist_peers { - let sequence = events.len() as u64 + 1; - events.push(SearchEvent { - search_id: search_id.to_owned(), - sequence, - kind: "federation.artist", - peer: peers.first().cloned(), - entity_key: json!({ "normalized_name": key }), - entity: json!({ - "key": { "normalized_name": key }, - "name": name, - "image_url": null, - "peers": peers, - }), - }); - } - for (key, release) in releases { - let sequence = events.len() as u64 + 1; - events.push(SearchEvent { - search_id: search_id.to_owned(), - sequence, - kind: "federation.release", - peer: None, - entity_key: json!({ "composite": key }), - entity: release, - }); - } + let events = music_search_events(&pool, search_id, &items, own).await?; tracing::info!( search_id, query, @@ -404,10 +270,295 @@ impl Federation { elapsed_ms = started.elapsed().as_millis() as u64, "federated search response ready" ); + Ok(MusicSearch { events, items }) + } + + /// Starts the public podcast search of `query` on the discovery + /// overlay, if it runs, next to the music search. + pub fn spawn_podcast_search(&self, query: &str) -> tokio::task::JoinHandle { + let discovery = self.podcast_discovery(); + let query = query.trim().to_owned(); + tokio::spawn(async move { search_podcasts(discovery.as_deref(), &query).await }) + } + + /// The `federation.podcast` events of a search for `user_id`, numbered + /// after the music events (see [`podcast_events`]). + pub async fn podcast_search_events( + &self, + search_id: &str, + query: &str, + user_id: i64, + music: &MusicSearch, + podcasts: PodcastSearch, + ) -> Result> { + let PodcastSearch::Found(hits) = podcasts else { + return Ok(Vec::new()); + }; + let pool = self.pool().await?; + let first_sequence = music.events.len() as u64 + 1; + let events = podcast_events( + &pool, + user_id, + search_id, + query.trim(), + &music.items, + first_sequence, + hits, + ) + .await?; + tracing::info!( + search_id, + podcasts = events.len(), + "public podcast hits ready" + ); Ok(events) } } +/// Deadline of the music DHT and the podcast overlay searches. +const SEARCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); + +/// Music results of a federated search: the events to stream and the items +/// (podcast hits are ranked among them). +#[derive(Debug)] +pub struct MusicSearch { + pub events: Vec, + pub items: Vec, +} + +/// Outcome of the public podcast search of one federated search. +#[derive(Debug)] +pub enum PodcastSearch { + /// Public discovery does not run on this server. + Off, + /// The overlay search failed or timed out (music results still count). + Failed, + /// Hits, possibly none. + Found(Vec), +} + +impl PodcastSearch { + /// `running`, `off` or `failed` (reported in `search.completed`). + pub fn state(&self) -> &'static str { + match self { + Self::Off => "off", + Self::Failed => "failed", + Self::Found(_) => "running", + } + } +} + +async fn search_podcasts( + discovery: Option<&music_dht::podcast_dht::PodcastDht>, + query: &str, +) -> PodcastSearch { + let Some(dht) = discovery else { + return PodcastSearch::Off; + }; + let limit = crate::podcasts::search::PODCAST_SEARCH_LIMIT; + match tokio::time::timeout(SEARCH_TIMEOUT, dht.search(query, limit)).await { + Ok(Ok(hits)) => PodcastSearch::Found(hits), + Ok(Err(error)) => { + tracing::info!("public podcast search failed: {error}"); + PodcastSearch::Failed + } + Err(_) => { + tracing::info!("public podcast search timed out"); + PodcastSearch::Failed + } + } +} + +/// One `federation.podcast` event per public podcast hit, for `user_id` +/// (`subscribed`, `local_podcast_id`), ordered by the hit's place in +/// `merge_search_results` with the music items and numbered from +/// `first_sequence`. +pub(crate) async fn podcast_events( + pool: &PgPool, + user_id: i64, + search_id: &str, + query: &str, + music: &[LibraryItem], + first_sequence: u64, + hits: Vec, +) -> Result> { + let store = crate::podcasts::PodcastStore::new(pool.clone()); + let views = crate::podcasts::search::hit_views(&store, user_id, query, music, hits).await?; + views + .into_iter() + .zip(first_sequence..) + .map(|(view, sequence)| { + Ok(SearchEvent { + search_id: search_id.to_owned(), + sequence, + kind: "federation.podcast", + hit_type: "podcast", + // Subscribing never contacts a publisher: no peer to name. + peer: None, + entity_key: json!({ "podcast_id": view.podcast_id }), + entity: serde_json::to_value(view)?, + }) + }) + .collect() +} + +/// Music events of a search (tracks, then artists and releases). +pub(crate) async fn music_search_events( + pool: &PgPool, + search_id: &str, + all_items: &[LibraryItem], + own: music_dht::EndpointId, +) -> Result> { + let mut by_content: HashMap = HashMap::new(); + for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) { + let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else { + // A globally usable track reference must be verifiable. + continue; + }; + let local = local_availability(pool, &content_id).await?; + let source = FederationSourceDto { + owner: item.owner.to_string(), + item_id: hex(item.id.as_bytes()), + }; + let entry = by_content + .entry(content_id.clone()) + .or_insert_with(|| track_from_item(content_id.clone(), item, local, item.owner == own)); + if !entry + .availability + .federation + .iter() + .any(|candidate| candidate.owner == source.owner && candidate.item_id == source.item_id) + { + entry.availability.federation.push(source); + } + if entry.availability.local.is_some() { + entry.availability.state = "local"; + } + } + + let mut tracks: Vec<_> = by_content.into_values().collect(); + tracks.sort_by(|left, right| { + left.metadata + .title + .to_lowercase() + .cmp(&right.metadata.title.to_lowercase()) + }); + + let mut events = Vec::with_capacity(all_items.len()); + for (index, track) in tracks.into_iter().enumerate() { + persist_track_ref(pool, &track).await?; + let peer = track + .availability + .federation + .first() + .map(|source| source.owner.clone()); + events.push(SearchEvent { + search_id: search_id.to_owned(), + sequence: index as u64 + 1, + kind: "federation.track", + hit_type: "music", + peer, + entity_key: serde_json::to_value(&track.key)?, + entity: serde_json::to_value(track)?, + }); + } + let mut artist_peers: HashMap)> = HashMap::new(); + let mut releases: HashMap = HashMap::new(); + for item in all_items { + match item.kind { + ItemKind::Artist => { + let key = music_dht::normalize_name(&item.name); + let entry = artist_peers + .entry(key) + .or_insert_with(|| (item.name.clone(), Vec::new())); + let owner = item.owner.to_string(); + if !entry.1.contains(&owner) { + entry.1.push(owner); + } + } + ItemKind::Release => { + let artist_keys: Vec = item + .artist_names + .iter() + .map(|name| music_dht::normalize_name(name)) + .collect(); + let normalized_title = music_dht::normalize_name(&item.name); + let cover_url = all_items + .iter() + .find(|track| { + track.kind == ItemKind::Track + && track.release_title.as_deref().is_some_and(|title| { + music_dht::normalize_name(title) == normalized_title + }) + && track.year == item.year + }) + .map(|track| { + format!( + "/api/player/federation/tracks/artwork?owner={}&item_id={}", + track.owner, + hex(track.id.as_bytes()) + ) + }); + let key = format!( + "{}|{}|{}", + normalized_title, + artist_keys.join(","), + item.year.map_or_else(String::new, |year| year.to_string()) + ); + releases.entry(key.clone()).or_insert_with(|| { + json!({ + "key": { + "normalized_title": music_dht::normalize_name(&item.name), + "primary_artists": artist_keys, + "release_type": null, + "year": item.year, + }, + "title": item.name, + "artists": item.artist_names, + "year": item.year, + "cover_url": cover_url, + "sources": [{ + "owner": item.owner.to_string(), + "item_id": hex(item.id.as_bytes()), + }], + }) + }); + } + ItemKind::Track => {} + } + } + for (key, (name, peers)) in artist_peers { + let sequence = events.len() as u64 + 1; + events.push(SearchEvent { + search_id: search_id.to_owned(), + sequence, + kind: "federation.artist", + hit_type: "music", + peer: peers.first().cloned(), + entity_key: json!({ "normalized_name": key }), + entity: json!({ + "key": { "normalized_name": key }, + "name": name, + "image_url": null, + "peers": peers, + }), + }); + } + for (key, release) in releases { + let sequence = events.len() as u64 + 1; + events.push(SearchEvent { + search_id: search_id.to_owned(), + sequence, + kind: "federation.release", + hit_type: "music", + peer: None, + entity_key: json!({ "composite": key }), + entity: release, + }); + } + Ok(events) +} + async fn fetch_artist_catalog( service: &music_dht::MusicDhtService, owner: music_dht::EndpointId, diff --git a/src/player/mod.rs b/src/player/mod.rs index e4dbe71..b683e0c 100644 --- a/src/player/mod.rs +++ b/src/player/mod.rs @@ -7472,7 +7472,7 @@ async fn federation_search_events_handler( db: Database, query: cot::request::extractors::UrlQuery, ) -> cot::Result { - let Some(_user) = auth::get_request_user(&auth_ctx, &session, &db).await else { + let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else { return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated")); }; let q = query.0.q.trim().to_owned(); @@ -7482,6 +7482,7 @@ async fn federation_search_events_handler( let search_id = uuid::Uuid::new_v4().to_string(); let body_search_id = search_id.clone(); + let user_id = user.id; let body = Body::streaming(async_stream::try_stream! { let started = serde_json::json!({ "search_id": body_search_id, @@ -7492,25 +7493,40 @@ async fn federation_search_events_handler( }); yield Bytes::from(format!("event: search.started\ndata: {started}\n\n")); - match crate::federation::handle().search_events(&body_search_id, &q).await { - Ok(events) => { + let federation = crate::federation::handle(); + // Public podcasts are searched next to the music DHT; their events + // follow the music events. + let podcasts = federation.spawn_podcast_search(&q); + match federation.search_events(&body_search_id, &q).await { + Ok(music) => { let mut last_sequence = 0; - for event in events { + for event in &music.events { last_sequence = event.sequence; - let data = serde_json::to_string(&event) - .map_err(|err| cot::Error::internal(err.to_string()))?; - yield Bytes::from(format!("event: {}\ndata: {data}\n\n", event.kind)); + yield search_event_frame(event)?; + } + let podcasts = podcasts + .await + .unwrap_or(crate::federation::client::PodcastSearch::Failed); + let podcast_discovery = podcasts.state(); + let podcast_events = federation + .podcast_search_events(&body_search_id, &q, user_id, &music, podcasts) + .await + .map_err(|err| cot::Error::internal(err.to_string()))?; + for event in &podcast_events { + last_sequence = event.sequence; + yield search_event_frame(event)?; } let completed = serde_json::json!({ "search_id": body_search_id, "sequence": last_sequence + 1, "kind": "search.completed", "entity_key": null, - "entity": { "partial": false }, + "entity": { "partial": false, "podcast_discovery": podcast_discovery }, }); yield Bytes::from(format!("event: search.completed\ndata: {completed}\n\n")); } Err(err) => { + podcasts.abort(); let failed = serde_json::json!({ "search_id": body_search_id, "sequence": 1, @@ -7531,6 +7547,14 @@ async fn federation_search_events_handler( .map_err(|err| cot::Error::internal(err.to_string())) } +/// One server-sent event of a federated search: `event: ` and the +/// event as JSON. +fn search_event_frame(event: &crate::federation::client::SearchEvent) -> cot::Result { + let data = serde_json::to_string(event).map_err(|err| cot::Error::internal(err.to_string()))?; + let kind = event.kind; + Ok(Bytes::from(format!("event: {kind}\ndata: {data}\n\n"))) +} + async fn federation_artist_events_handler( auth_ctx: auth::AuthContext, session: Session,