Stream public podcast hits in the federated search

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:51 +02:00
1 parent d4b4543be1
commit 4c5ba3ff0d
2 files changed
+333 -158

No files matched your search

+301 -150
View File
@@ -9,6 +9,7 @@ use anyhow::{Context, Result};
use music_dht::{ItemKind, LibraryItem, normalize_content_id};
use serde::Serialize;
use serde_json::{Value, json};
use sqlx::PgPool;
use sqlx::Row as _;
use tokio::io::AsyncReadExt;
@@ -94,6 +95,9 @@ pub struct SearchEvent {
pub search_id: String,
pub sequence: u64,
pub kind: &'static str,
/// `music` or `podcast` (`music_dht::podcast_dht::FederatedSearchHit`).
#[serde(rename = "type")]
pub hit_type: &'static str,
pub peer: Option<String>,
pub entity_key: Value,
pub entity: Value,
@@ -218,7 +222,12 @@ impl Federation {
/// Performs one bounded DHT search and returns entity upserts. The HTTP
/// layer streams each upsert independently; catalog fan-out can append
/// events to the same contract without changing the browser model.
pub async fn search_events(&self, search_id: &str, query: &str) -> Result<Vec<SearchEvent>> {
///
/// Music only (`federation.track|artist|release`, marked `"type":
/// "music"`). Public podcasts are searched next to it
/// ([`Self::spawn_podcast_search`]) and follow as `federation.podcast`
/// events ([`Self::podcast_search_events`]).
pub async fn search_events(&self, search_id: &str, query: &str) -> Result<MusicSearch> {
let query = query.trim();
anyhow::ensure!(!query.is_empty(), "search query is empty");
anyhow::ensure!(query.chars().count() <= 200, "search query is too long");
@@ -233,13 +242,10 @@ impl Federation {
"federated search started"
);
let own = service.endpoint_id();
let result = tokio::time::timeout(
std::time::Duration::from_secs(20),
service.search_network(query),
)
.await
.map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))?
.map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?;
let result = tokio::time::timeout(SEARCH_TIMEOUT, service.search_network(query))
.await
.map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))?
.map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?;
tracing::info!(
search_id,
query,
@@ -249,154 +255,14 @@ impl Federation {
elapsed_ms = started.elapsed().as_millis() as u64,
"federated DHT search finished"
);
let all_items: Vec<LibraryItem> = result
let items: Vec<LibraryItem> = result
.local_results
.into_iter()
.chain(result.network_results)
.collect();
let pool = self.pool().await?;
let mut by_content: HashMap<String, TrackDto> = HashMap::new();
for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) {
let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else {
// A globally usable track reference must be verifiable.
continue;
};
let local = local_availability(&pool, &content_id).await?;
let source = FederationSourceDto {
owner: item.owner.to_string(),
item_id: hex(item.id.as_bytes()),
};
let entry = by_content.entry(content_id.clone()).or_insert_with(|| {
track_from_item(content_id.clone(), item, local, item.owner == own)
});
if !entry.availability.federation.iter().any(|candidate| {
candidate.owner == source.owner && candidate.item_id == source.item_id
}) {
entry.availability.federation.push(source);
}
if entry.availability.local.is_some() {
entry.availability.state = "local";
}
}
let mut tracks: Vec<_> = by_content.into_values().collect();
tracks.sort_by(|left, right| {
left.metadata
.title
.to_lowercase()
.cmp(&right.metadata.title.to_lowercase())
});
let mut events = Vec::with_capacity(all_items.len());
for (index, track) in tracks.into_iter().enumerate() {
persist_track_ref(&pool, &track).await?;
let peer = track
.availability
.federation
.first()
.map(|source| source.owner.clone());
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence: index as u64 + 1,
kind: "federation.track",
peer,
entity_key: serde_json::to_value(&track.key)?,
entity: serde_json::to_value(track)?,
});
}
let mut artist_peers: HashMap<String, (String, Vec<String>)> = HashMap::new();
let mut releases: HashMap<String, Value> = HashMap::new();
for item in &all_items {
match item.kind {
ItemKind::Artist => {
let key = music_dht::normalize_name(&item.name);
let entry = artist_peers
.entry(key)
.or_insert_with(|| (item.name.clone(), Vec::new()));
let owner = item.owner.to_string();
if !entry.1.contains(&owner) {
entry.1.push(owner);
}
}
ItemKind::Release => {
let artist_keys: Vec<String> = item
.artist_names
.iter()
.map(|name| music_dht::normalize_name(name))
.collect();
let normalized_title = music_dht::normalize_name(&item.name);
let cover_url = all_items
.iter()
.find(|track| {
track.kind == ItemKind::Track
&& track.release_title.as_deref().is_some_and(|title| {
music_dht::normalize_name(title) == normalized_title
})
&& track.year == item.year
})
.map(|track| {
format!(
"/api/player/federation/tracks/artwork?owner={}&item_id={}",
track.owner,
hex(track.id.as_bytes())
)
});
let key = format!(
"{}|{}|{}",
normalized_title,
artist_keys.join(","),
item.year.map_or_else(String::new, |year| year.to_string())
);
releases.entry(key.clone()).or_insert_with(|| {
json!({
"key": {
"normalized_title": music_dht::normalize_name(&item.name),
"primary_artists": artist_keys,
"release_type": null,
"year": item.year,
},
"title": item.name,
"artists": item.artist_names,
"year": item.year,
"cover_url": cover_url,
"sources": [{
"owner": item.owner.to_string(),
"item_id": hex(item.id.as_bytes()),
}],
})
});
}
ItemKind::Track => {}
}
}
for (key, (name, peers)) in artist_peers {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.artist",
peer: peers.first().cloned(),
entity_key: json!({ "normalized_name": key }),
entity: json!({
"key": { "normalized_name": key },
"name": name,
"image_url": null,
"peers": peers,
}),
});
}
for (key, release) in releases {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.release",
peer: None,
entity_key: json!({ "composite": key }),
entity: release,
});
}
let events = music_search_events(&pool, search_id, &items, own).await?;
tracing::info!(
search_id,
query,
@@ -404,10 +270,295 @@ impl Federation {
elapsed_ms = started.elapsed().as_millis() as u64,
"federated search response ready"
);
Ok(MusicSearch { events, items })
}
/// Starts the public podcast search of `query` on the discovery
/// overlay, if it runs, next to the music search.
pub fn spawn_podcast_search(&self, query: &str) -> tokio::task::JoinHandle<PodcastSearch> {
let discovery = self.podcast_discovery();
let query = query.trim().to_owned();
tokio::spawn(async move { search_podcasts(discovery.as_deref(), &query).await })
}
/// The `federation.podcast` events of a search for `user_id`, numbered
/// after the music events (see [`podcast_events`]).
pub async fn podcast_search_events(
&self,
search_id: &str,
query: &str,
user_id: i64,
music: &MusicSearch,
podcasts: PodcastSearch,
) -> Result<Vec<SearchEvent>> {
let PodcastSearch::Found(hits) = podcasts else {
return Ok(Vec::new());
};
let pool = self.pool().await?;
let first_sequence = music.events.len() as u64 + 1;
let events = podcast_events(
&pool,
user_id,
search_id,
query.trim(),
&music.items,
first_sequence,
hits,
)
.await?;
tracing::info!(
search_id,
podcasts = events.len(),
"public podcast hits ready"
);
Ok(events)
}
}
/// Deadline of the music DHT and the podcast overlay searches.
const SEARCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
/// Music results of a federated search: the events to stream and the items
/// (podcast hits are ranked among them).
#[derive(Debug)]
pub struct MusicSearch {
pub events: Vec<SearchEvent>,
pub items: Vec<LibraryItem>,
}
/// Outcome of the public podcast search of one federated search.
#[derive(Debug)]
pub enum PodcastSearch {
/// Public discovery does not run on this server.
Off,
/// The overlay search failed or timed out (music results still count).
Failed,
/// Hits, possibly none.
Found(Vec<music_dht::podcast_dht::PodcastSearchHit>),
}
impl PodcastSearch {
/// `running`, `off` or `failed` (reported in `search.completed`).
pub fn state(&self) -> &'static str {
match self {
Self::Off => "off",
Self::Failed => "failed",
Self::Found(_) => "running",
}
}
}
async fn search_podcasts(
discovery: Option<&music_dht::podcast_dht::PodcastDht>,
query: &str,
) -> PodcastSearch {
let Some(dht) = discovery else {
return PodcastSearch::Off;
};
let limit = crate::podcasts::search::PODCAST_SEARCH_LIMIT;
match tokio::time::timeout(SEARCH_TIMEOUT, dht.search(query, limit)).await {
Ok(Ok(hits)) => PodcastSearch::Found(hits),
Ok(Err(error)) => {
tracing::info!("public podcast search failed: {error}");
PodcastSearch::Failed
}
Err(_) => {
tracing::info!("public podcast search timed out");
PodcastSearch::Failed
}
}
}
/// One `federation.podcast` event per public podcast hit, for `user_id`
/// (`subscribed`, `local_podcast_id`), ordered by the hit's place in
/// `merge_search_results` with the music items and numbered from
/// `first_sequence`.
pub(crate) async fn podcast_events(
pool: &PgPool,
user_id: i64,
search_id: &str,
query: &str,
music: &[LibraryItem],
first_sequence: u64,
hits: Vec<music_dht::podcast_dht::PodcastSearchHit>,
) -> Result<Vec<SearchEvent>> {
let store = crate::podcasts::PodcastStore::new(pool.clone());
let views = crate::podcasts::search::hit_views(&store, user_id, query, music, hits).await?;
views
.into_iter()
.zip(first_sequence..)
.map(|(view, sequence)| {
Ok(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.podcast",
hit_type: "podcast",
// Subscribing never contacts a publisher: no peer to name.
peer: None,
entity_key: json!({ "podcast_id": view.podcast_id }),
entity: serde_json::to_value(view)?,
})
})
.collect()
}
/// Music events of a search (tracks, then artists and releases).
pub(crate) async fn music_search_events(
pool: &PgPool,
search_id: &str,
all_items: &[LibraryItem],
own: music_dht::EndpointId,
) -> Result<Vec<SearchEvent>> {
let mut by_content: HashMap<String, TrackDto> = HashMap::new();
for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) {
let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else {
// A globally usable track reference must be verifiable.
continue;
};
let local = local_availability(pool, &content_id).await?;
let source = FederationSourceDto {
owner: item.owner.to_string(),
item_id: hex(item.id.as_bytes()),
};
let entry = by_content
.entry(content_id.clone())
.or_insert_with(|| track_from_item(content_id.clone(), item, local, item.owner == own));
if !entry
.availability
.federation
.iter()
.any(|candidate| candidate.owner == source.owner && candidate.item_id == source.item_id)
{
entry.availability.federation.push(source);
}
if entry.availability.local.is_some() {
entry.availability.state = "local";
}
}
let mut tracks: Vec<_> = by_content.into_values().collect();
tracks.sort_by(|left, right| {
left.metadata
.title
.to_lowercase()
.cmp(&right.metadata.title.to_lowercase())
});
let mut events = Vec::with_capacity(all_items.len());
for (index, track) in tracks.into_iter().enumerate() {
persist_track_ref(pool, &track).await?;
let peer = track
.availability
.federation
.first()
.map(|source| source.owner.clone());
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence: index as u64 + 1,
kind: "federation.track",
hit_type: "music",
peer,
entity_key: serde_json::to_value(&track.key)?,
entity: serde_json::to_value(track)?,
});
}
let mut artist_peers: HashMap<String, (String, Vec<String>)> = HashMap::new();
let mut releases: HashMap<String, Value> = HashMap::new();
for item in all_items {
match item.kind {
ItemKind::Artist => {
let key = music_dht::normalize_name(&item.name);
let entry = artist_peers
.entry(key)
.or_insert_with(|| (item.name.clone(), Vec::new()));
let owner = item.owner.to_string();
if !entry.1.contains(&owner) {
entry.1.push(owner);
}
}
ItemKind::Release => {
let artist_keys: Vec<String> = item
.artist_names
.iter()
.map(|name| music_dht::normalize_name(name))
.collect();
let normalized_title = music_dht::normalize_name(&item.name);
let cover_url = all_items
.iter()
.find(|track| {
track.kind == ItemKind::Track
&& track.release_title.as_deref().is_some_and(|title| {
music_dht::normalize_name(title) == normalized_title
})
&& track.year == item.year
})
.map(|track| {
format!(
"/api/player/federation/tracks/artwork?owner={}&item_id={}",
track.owner,
hex(track.id.as_bytes())
)
});
let key = format!(
"{}|{}|{}",
normalized_title,
artist_keys.join(","),
item.year.map_or_else(String::new, |year| year.to_string())
);
releases.entry(key.clone()).or_insert_with(|| {
json!({
"key": {
"normalized_title": music_dht::normalize_name(&item.name),
"primary_artists": artist_keys,
"release_type": null,
"year": item.year,
},
"title": item.name,
"artists": item.artist_names,
"year": item.year,
"cover_url": cover_url,
"sources": [{
"owner": item.owner.to_string(),
"item_id": hex(item.id.as_bytes()),
}],
})
});
}
ItemKind::Track => {}
}
}
for (key, (name, peers)) in artist_peers {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.artist",
hit_type: "music",
peer: peers.first().cloned(),
entity_key: json!({ "normalized_name": key }),
entity: json!({
"key": { "normalized_name": key },
"name": name,
"image_url": null,
"peers": peers,
}),
});
}
for (key, release) in releases {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.release",
hit_type: "music",
peer: None,
entity_key: json!({ "composite": key }),
entity: release,
});
}
Ok(events)
}
async fn fetch_artist_catalog(
service: &music_dht::MusicDhtService,
owner: music_dht::EndpointId,
+32 -8
View File
@@ -7472,7 +7472,7 @@ async fn federation_search_events_handler(
db: Database,
query: cot::request::extractors::UrlQuery<SearchQuery>,
) -> cot::Result<cot::response::Response> {
let Some(_user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let q = query.0.q.trim().to_owned();
@@ -7482,6 +7482,7 @@ async fn federation_search_events_handler(
let search_id = uuid::Uuid::new_v4().to_string();
let body_search_id = search_id.clone();
let user_id = user.id;
let body = Body::streaming(async_stream::try_stream! {
let started = serde_json::json!({
"search_id": body_search_id,
@@ -7492,25 +7493,40 @@ async fn federation_search_events_handler(
});
yield Bytes::from(format!("event: search.started\ndata: {started}\n\n"));
match crate::federation::handle().search_events(&body_search_id, &q).await {
Ok(events) => {
let federation = crate::federation::handle();
// Public podcasts are searched next to the music DHT; their events
// follow the music events.
let podcasts = federation.spawn_podcast_search(&q);
match federation.search_events(&body_search_id, &q).await {
Ok(music) => {
let mut last_sequence = 0;
for event in events {
for event in &music.events {
last_sequence = event.sequence;
let data = serde_json::to_string(&event)
.map_err(|err| cot::Error::internal(err.to_string()))?;
yield Bytes::from(format!("event: {}\ndata: {data}\n\n", event.kind));
yield search_event_frame(event)?;
}
let podcasts = podcasts
.await
.unwrap_or(crate::federation::client::PodcastSearch::Failed);
let podcast_discovery = podcasts.state();
let podcast_events = federation
.podcast_search_events(&body_search_id, &q, user_id, &music, podcasts)
.await
.map_err(|err| cot::Error::internal(err.to_string()))?;
for event in &podcast_events {
last_sequence = event.sequence;
yield search_event_frame(event)?;
}
let completed = serde_json::json!({
"search_id": body_search_id,
"sequence": last_sequence + 1,
"kind": "search.completed",
"entity_key": null,
"entity": { "partial": false },
"entity": { "partial": false, "podcast_discovery": podcast_discovery },
});
yield Bytes::from(format!("event: search.completed\ndata: {completed}\n\n"));
}
Err(err) => {
podcasts.abort();
let failed = serde_json::json!({
"search_id": body_search_id,
"sequence": 1,
@@ -7531,6 +7547,14 @@ async fn federation_search_events_handler(
.map_err(|err| cot::Error::internal(err.to_string()))
}
/// One server-sent event of a federated search: `event: <kind>` and the
/// event as JSON.
fn search_event_frame(event: &crate::federation::client::SearchEvent) -> cot::Result<Bytes> {
let data = serde_json::to_string(event).map_err(|err| cot::Error::internal(err.to_string()))?;
let kind = event.kind;
Ok(Bytes::from(format!("event: {kind}\ndata: {data}\n\n")))
}
async fn federation_artist_events_handler(
auth_ctx: auth::AuthContext,
session: Session,