Rank public podcast hits for the searching account
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
27fc2c5354
commit
d4b4543be1
3 files changed
+395
-1
No files matched your search
@@ -767,7 +767,6 @@ impl PodcastStore {
|
||||
/// Resolves a feed URL to a podcast id: the existing podcast when the
|
||||
/// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise
|
||||
/// the id a first subscription through this URL would derive.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn resolve_podcast_id(
|
||||
&self,
|
||||
user_id: i64,
|
||||
|
||||
@@ -61,6 +61,9 @@
|
||||
//! publishes to `music_dht::podcast_dht` for the accounts that explicitly
|
||||
//! shared them, the local privacy confirmations and the reconciler that
|
||||
//! calls `PodcastDht::sync_shares`.
|
||||
//! * [`search`] — public podcasts found by the federated search, as seen by
|
||||
//! the searching account ("already subscribed"), and the preview of a
|
||||
//! found podcast's feed before subscribing.
|
||||
|
||||
mod actions;
|
||||
pub(crate) mod api;
|
||||
@@ -75,6 +78,7 @@ mod queries;
|
||||
mod refresh;
|
||||
pub(crate) mod runner;
|
||||
pub(crate) mod schema;
|
||||
pub(crate) mod search;
|
||||
|
||||
#[cfg(test)]
|
||||
mod discovery_tests;
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
//! Public podcasts found by the federated search, seen by one account.
|
||||
//!
|
||||
//! The federated search (`/api/player/federation/search/events`) asks the
|
||||
//! discovery overlay (`PodcastDht::search`) next to the music DHT and merges
|
||||
//! both with `merge_search_results`: exact title matches first, music before
|
||||
//! podcasts inside a tier. This module turns the podcast hits into
|
||||
//! [`PodcastHitView`]s for the account that searched:
|
||||
//!
|
||||
//! * only the hit's `feed_url` (the origin URL the id hashes from) is
|
||||
//! offered; `claimed_feed_urls` are unverified publisher claims and are
|
||||
//! not passed on;
|
||||
//! * every text and URL of a card is a claim of some publisher. Artwork and
|
||||
//! web-page links are kept only as absolute `http(s)` URLs; the browser
|
||||
//! renders all text as text;
|
||||
//! * `subscribed` / `local_podcast_id` mark a podcast the account already
|
||||
//! has, by id or through a known feed URL or alias (`resolve_podcast_id`).
|
||||
//!
|
||||
//! [`preview`] reads the found podcast's feed (through the SSRF-guarded
|
||||
//! fetcher, nothing stored) so the user can look at its seasons and
|
||||
//! episodes before subscribing; subscribing uses the origin URL, so the new
|
||||
//! private subscription gets the card's id (see `api::subscribe_found`).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::podcast::feed::ParsedFeed;
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
use music_dht::podcast_dht::{FederatedSearchHit, PodcastSearchHit, merge_search_results};
|
||||
use music_dht::{LibraryItem, normalize_name};
|
||||
use serde::Serialize;
|
||||
|
||||
use super::{PodcastStore, PodcastSummary};
|
||||
|
||||
/// Podcast hits asked from the overlay per search.
|
||||
pub(crate) const PODCAST_SEARCH_LIMIT: usize = 20;
|
||||
/// Episodes listed in a preview (newest first).
|
||||
pub(crate) const PREVIEW_EPISODES: usize = 50;
|
||||
/// Longest URL kept from a card or a feed.
|
||||
const MAX_URL_BYTES: usize = 2048;
|
||||
|
||||
/// One public podcast in the federated search results.
|
||||
///
|
||||
/// Serialized with `"type": "podcast"` (music hits are `"type": "music"`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct PodcastHitView {
|
||||
#[serde(rename = "type")]
|
||||
pub(crate) hit_type: &'static str,
|
||||
/// Always `podcast` (music hits: `artist`, `release`, `track`).
|
||||
pub(crate) kind: &'static str,
|
||||
pub(crate) podcast_id: String,
|
||||
/// The origin feed URL: the only URL to subscribe with.
|
||||
pub(crate) feed_url: String,
|
||||
pub(crate) title: String,
|
||||
pub(crate) author: Option<String>,
|
||||
pub(crate) description: Option<String>,
|
||||
/// Absolute `http(s)` artwork URL, if the card has a valid one.
|
||||
pub(crate) image_url: Option<String>,
|
||||
/// Absolute `http(s)` web page, if the card has a valid one.
|
||||
pub(crate) link: Option<String>,
|
||||
pub(crate) language: Option<String>,
|
||||
pub(crate) categories: Vec<String>,
|
||||
pub(crate) episode_count: Option<u32>,
|
||||
pub(crate) latest_episode_at_ms: Option<i64>,
|
||||
/// Distinct publisher devices (not subscribers, not a trust signal).
|
||||
pub(crate) publisher_count: usize,
|
||||
/// Position in the merged music and podcast results.
|
||||
pub(crate) rank: usize,
|
||||
/// The normalized title equals the normalized query.
|
||||
pub(crate) exact_match: bool,
|
||||
/// The account has an active subscription to this podcast.
|
||||
pub(crate) subscribed: bool,
|
||||
/// The account's podcast for this hit (its id, or the podcast a known
|
||||
/// feed URL or alias resolves to), when the account knows it.
|
||||
pub(crate) local_podcast_id: Option<String>,
|
||||
}
|
||||
|
||||
/// An absolute `http(s)` URL without credentials, at most 2 048 bytes.
|
||||
pub(crate) fn http_url(value: Option<&str>) -> Option<String> {
|
||||
let value = value?.trim();
|
||||
if value.is_empty() || value.len() > MAX_URL_BYTES {
|
||||
return None;
|
||||
}
|
||||
let url = reqwest::Url::parse(value).ok()?;
|
||||
(matches!(url.scheme(), "http" | "https")
|
||||
&& url.host_str().is_some()
|
||||
&& url.username().is_empty()
|
||||
&& url.password().is_none())
|
||||
.then(|| value.to_string())
|
||||
}
|
||||
|
||||
/// Podcast hits in their merged order with the music results:
|
||||
/// `(rank, exact_match, hit)`.
|
||||
pub(crate) fn ranked_hits(
|
||||
query: &str,
|
||||
music: &[LibraryItem],
|
||||
podcasts: Vec<PodcastSearchHit>,
|
||||
) -> Vec<(usize, bool, PodcastSearchHit)> {
|
||||
let normalized = normalize_name(query);
|
||||
merge_search_results(query, music.to_vec(), podcasts)
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(rank, hit)| match hit {
|
||||
FederatedSearchHit::Podcast(hit) => {
|
||||
let exact = !normalized.is_empty() && normalize_name(&hit.card.title) == normalized;
|
||||
Some((rank, exact, hit))
|
||||
}
|
||||
FederatedSearchHit::Music(_) => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The account's podcast for a found podcast: the podcast with that id, or
|
||||
/// the podcast its origin URL resolves to through the account's known feed
|
||||
/// URLs and aliases.
|
||||
pub(crate) async fn known_podcast(
|
||||
store: &PodcastStore,
|
||||
user_id: i64,
|
||||
podcast_id: &PodcastId,
|
||||
feed_url: &FeedUrl,
|
||||
) -> Result<Option<PodcastSummary>> {
|
||||
if let Some(summary) = store.podcast(user_id, podcast_id).await? {
|
||||
return Ok(Some(summary));
|
||||
}
|
||||
let resolved = store.resolve_podcast_id(user_id, feed_url).await?;
|
||||
if resolved == *podcast_id {
|
||||
return Ok(None);
|
||||
}
|
||||
store.podcast(user_id, &resolved).await
|
||||
}
|
||||
|
||||
/// Views of the podcast hits of one search for `user_id`, in rank order.
|
||||
pub(crate) async fn hit_views(
|
||||
store: &PodcastStore,
|
||||
user_id: i64,
|
||||
query: &str,
|
||||
music: &[LibraryItem],
|
||||
podcasts: Vec<PodcastSearchHit>,
|
||||
) -> Result<Vec<PodcastHitView>> {
|
||||
let mut views = Vec::new();
|
||||
for (rank, exact_match, hit) in ranked_hits(query, music, podcasts) {
|
||||
let known = known_podcast(store, user_id, &hit.podcast_id, &hit.feed_url).await?;
|
||||
let card = hit.card;
|
||||
views.push(PodcastHitView {
|
||||
hit_type: "podcast",
|
||||
kind: "podcast",
|
||||
podcast_id: hit.podcast_id.to_string(),
|
||||
feed_url: hit.feed_url.to_string(),
|
||||
title: card.title,
|
||||
author: card.author,
|
||||
description: card.description,
|
||||
image_url: http_url(card.image_url.as_deref()),
|
||||
link: http_url(card.link.as_deref()),
|
||||
language: card.language,
|
||||
categories: card.categories,
|
||||
episode_count: card.episode_count,
|
||||
latest_episode_at_ms: card.latest_episode_at_ms,
|
||||
publisher_count: hit.publishers.len(),
|
||||
rank,
|
||||
exact_match,
|
||||
subscribed: known.as_ref().is_some_and(|summary| summary.subscribed),
|
||||
local_podcast_id: known.map(|summary| summary.podcast_id.to_string()),
|
||||
});
|
||||
}
|
||||
Ok(views)
|
||||
}
|
||||
|
||||
/// One season group of a preview (`season: null`: episodes without one).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct PreviewSeason {
|
||||
pub(crate) season: Option<u32>,
|
||||
pub(crate) episode_count: usize,
|
||||
}
|
||||
|
||||
/// One episode of a preview.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct PreviewEpisode {
|
||||
pub(crate) title: String,
|
||||
pub(crate) published_at_ms: Option<i64>,
|
||||
pub(crate) duration_ms: Option<i64>,
|
||||
pub(crate) season: Option<u32>,
|
||||
pub(crate) episode_number: Option<u32>,
|
||||
}
|
||||
|
||||
/// A found podcast read from its feed: channel, seasons and the newest
|
||||
/// episodes. Nothing of it is stored.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct PodcastPreview {
|
||||
pub(crate) podcast_id: String,
|
||||
/// The URL that was read (the card's origin URL).
|
||||
pub(crate) feed_url: String,
|
||||
/// The feed answered with a permanent redirect chain to another URL.
|
||||
pub(crate) moved: bool,
|
||||
pub(crate) title: String,
|
||||
pub(crate) author: Option<String>,
|
||||
pub(crate) description: Option<String>,
|
||||
pub(crate) image_url: Option<String>,
|
||||
pub(crate) link: Option<String>,
|
||||
pub(crate) episode_count: usize,
|
||||
pub(crate) seasons: Vec<PreviewSeason>,
|
||||
/// Newest first, at most [`PREVIEW_EPISODES`].
|
||||
pub(crate) episodes: Vec<PreviewEpisode>,
|
||||
pub(crate) subscribed: bool,
|
||||
pub(crate) local_podcast_id: Option<String>,
|
||||
}
|
||||
|
||||
/// The preview of a parsed feed (`known`: the account's podcast for it).
|
||||
pub(crate) fn preview(
|
||||
podcast_id: &PodcastId,
|
||||
feed_url: &FeedUrl,
|
||||
moved: bool,
|
||||
feed: &ParsedFeed,
|
||||
known: Option<&PodcastSummary>,
|
||||
) -> PodcastPreview {
|
||||
let mut seasons: BTreeMap<Option<u32>, usize> = BTreeMap::new();
|
||||
for episode in &feed.episodes {
|
||||
*seasons.entry(episode.season).or_default() += 1;
|
||||
}
|
||||
let mut episodes: Vec<&_> = feed.episodes.iter().collect();
|
||||
// Newest first; undated episodes keep their document order at the end.
|
||||
episodes.sort_by_key(|episode| std::cmp::Reverse(episode.published_at_ms));
|
||||
PodcastPreview {
|
||||
podcast_id: podcast_id.to_string(),
|
||||
feed_url: feed_url.to_string(),
|
||||
moved,
|
||||
title: feed.title.clone(),
|
||||
author: feed.author.clone(),
|
||||
description: feed.description.clone(),
|
||||
image_url: http_url(feed.image_url.as_deref()),
|
||||
link: http_url(feed.link.as_deref()),
|
||||
episode_count: feed.episodes.len(),
|
||||
// Numbered seasons first, then the episodes without a season.
|
||||
seasons: seasons
|
||||
.iter()
|
||||
.filter(|(season, _)| season.is_some())
|
||||
.chain(seasons.iter().filter(|(season, _)| season.is_none()))
|
||||
.map(|(season, count)| PreviewSeason {
|
||||
season: *season,
|
||||
episode_count: *count,
|
||||
})
|
||||
.collect(),
|
||||
episodes: episodes
|
||||
.into_iter()
|
||||
.take(PREVIEW_EPISODES)
|
||||
.map(|episode| PreviewEpisode {
|
||||
title: episode.title.clone(),
|
||||
published_at_ms: episode.published_at_ms,
|
||||
duration_ms: episode.duration_ms,
|
||||
season: episode.season,
|
||||
episode_number: episode.episode_number,
|
||||
})
|
||||
.collect(),
|
||||
subscribed: known.is_some_and(|summary| summary.subscribed),
|
||||
local_podcast_id: known.map(|summary| summary.podcast_id.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use music_dht::podcast_dht::{PodcastCard, PodcastCardSpec};
|
||||
use music_dht::{ItemId, ItemKind, NetworkId, SecretKey};
|
||||
|
||||
fn hit(path: &str, title: &str) -> PodcastSearchHit {
|
||||
let feed_url = FeedUrl::parse(&format!("https://feeds.example/{path}")).unwrap();
|
||||
let podcast_id = PodcastId::from_feed_url(&feed_url);
|
||||
let publisher = SecretKey::from_bytes(&[7; 32]).public();
|
||||
let card = PodcastCard::from_spec(
|
||||
PodcastCardSpec {
|
||||
podcast_id: podcast_id.clone(),
|
||||
feed_url: feed_url.clone(),
|
||||
id_feed_url: None,
|
||||
title: title.into(),
|
||||
author: None,
|
||||
description: None,
|
||||
image_url: None,
|
||||
link: None,
|
||||
language: None,
|
||||
categories: Vec::new(),
|
||||
episode_count: None,
|
||||
latest_episode_at_ms: None,
|
||||
},
|
||||
NetworkId::from_name("search-test"),
|
||||
publisher,
|
||||
1,
|
||||
1_700_000_000_000,
|
||||
);
|
||||
PodcastSearchHit {
|
||||
podcast_id,
|
||||
feed_url,
|
||||
card,
|
||||
publishers: vec![publisher],
|
||||
claimed_feed_urls: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn music(name: &str) -> LibraryItem {
|
||||
LibraryItem {
|
||||
id: ItemId::from_bytes([1; 32]),
|
||||
owner: SecretKey::from_bytes(&[5; 32]).public(),
|
||||
kind: ItemKind::Track,
|
||||
name: name.into(),
|
||||
normalized_name: normalize_name(name),
|
||||
artist_names: Vec::new(),
|
||||
featured_artist_names: Vec::new(),
|
||||
year: None,
|
||||
release_type: None,
|
||||
release_title: None,
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
duration_seconds: None,
|
||||
content_id: None,
|
||||
revision: 1,
|
||||
deleted: false,
|
||||
updated_at_ms: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_hits_take_their_place_among_music_results() {
|
||||
let ranked = ranked_hits(
|
||||
"Night Radio",
|
||||
&[music("Night Radio Theme"), music("Night Radio")],
|
||||
vec![
|
||||
hit("a.xml", "Night Radio"),
|
||||
hit("b.xml", "Night Radio Talks"),
|
||||
],
|
||||
);
|
||||
let ranks: Vec<_> = ranked
|
||||
.iter()
|
||||
.map(|(rank, exact, hit)| (*rank, *exact, hit.card.title.as_str()))
|
||||
.collect();
|
||||
// Exact matches lead (music first inside a tier), then the rest.
|
||||
assert_eq!(
|
||||
ranks,
|
||||
vec![(1, true, "Night Radio"), (3, false, "Night Radio Talks")]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_absolute_http_urls_without_credentials_are_kept() {
|
||||
for (value, kept) in [
|
||||
("https://cdn.example/art.jpg", true),
|
||||
("http://example.org/show", true),
|
||||
("javascript:alert(1)", false),
|
||||
("data:image/png;base64,AAAA", false),
|
||||
("//cdn.example/art.jpg", false),
|
||||
("https://user:secret@cdn.example/art.jpg", false),
|
||||
("file:///etc/passwd", false),
|
||||
("", false),
|
||||
] {
|
||||
assert_eq!(http_url(Some(value)).is_some(), kept, "{value}");
|
||||
}
|
||||
let long = format!("https://cdn.example/{}", "a".repeat(MAX_URL_BYTES));
|
||||
assert_eq!(http_url(Some(&long)), None);
|
||||
assert_eq!(http_url(None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_preview_lists_seasons_and_the_newest_episodes() {
|
||||
let xml = r#"<?xml version="1.0"?>
|
||||
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"><channel>
|
||||
<title>Preview Show</title><link>javascript:alert(1)</link>
|
||||
<item><title>Old</title><guid>1</guid><itunes:season>1</itunes:season>
|
||||
<pubDate>Mon, 01 Jan 2024 10:00:00 GMT</pubDate>
|
||||
<enclosure url="https://cdn.example/1.mp3" type="audio/mpeg"/></item>
|
||||
<item><title>Bonus</title><guid>2</guid>
|
||||
<enclosure url="https://cdn.example/2.mp3" type="audio/mpeg"/></item>
|
||||
<item><title>New</title><guid>3</guid><itunes:season>2</itunes:season>
|
||||
<pubDate>Mon, 01 Jul 2024 10:00:00 GMT</pubDate>
|
||||
<enclosure url="https://cdn.example/3.mp3" type="audio/mpeg"/></item>
|
||||
</channel></rss>"#;
|
||||
let url = FeedUrl::parse("https://feeds.example/preview.xml").unwrap();
|
||||
let feed =
|
||||
music_dht::podcast::feed::parse_feed_with_base(xml.as_bytes(), Some(url.as_str()))
|
||||
.unwrap();
|
||||
let id = PodcastId::from_feed_url(&url);
|
||||
let preview = preview(&id, &url, false, &feed, None);
|
||||
assert_eq!(preview.title, "Preview Show");
|
||||
assert_eq!(preview.link, None, "only http(s) links");
|
||||
assert_eq!(preview.episode_count, 3);
|
||||
let seasons: Vec<_> = preview
|
||||
.seasons
|
||||
.iter()
|
||||
.map(|season| (season.season, season.episode_count))
|
||||
.collect();
|
||||
assert_eq!(seasons, vec![(Some(1), 1), (Some(2), 1), (None, 1)]);
|
||||
let titles: Vec<_> = preview.episodes.iter().map(|e| e.title.as_str()).collect();
|
||||
assert_eq!(titles, vec!["New", "Old", "Bonus"]);
|
||||
assert!(!preview.subscribed && preview.local_podcast_id.is_none());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user