Add signed public podcast discovery overlay
New music_dht::podcast_dht: a schema-independent Kademlia overlay on furumi-fd/podcast-dht/1 (version 1) with its own SQLite cache. Devices publish signed public podcast cards only where effective_share holds, retract them with signed tombstones of the next revision, and search by normalized title/author tokens or podcast id. Records are verified on every receive and cache read. Adds typed federated search hits, the advisory feed_url_privacy_warning helper, the podcast_dht capability and MusicDhtError::PodcastPublication. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d2530dca6c
commit
d27155b257
9 files changed
+4142
No files matched your search
@@ -35,6 +35,8 @@ pub const CATALOG_ID: &str = "catalog";
|
||||
pub const SIMILARITY_ID: &str = "similarity";
|
||||
/// Stable protocol identifier for DHT-routed similarity discovery.
|
||||
pub const SIMILARITY_DHT_ID: &str = "similarity_dht";
|
||||
/// Stable protocol identifier for the public podcast-discovery overlay.
|
||||
pub const PODCAST_DHT_ID: &str = "podcast_dht";
|
||||
/// Stable protocol identifier for personal-device synchronization.
|
||||
pub const DEVICE_SYNC_ID: &str = "device_sync";
|
||||
/// Stable protocol identifier for Jam playback control.
|
||||
@@ -79,6 +81,10 @@ impl CapabilityManifest {
|
||||
SIMILARITY_DHT_ID.to_string(),
|
||||
crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION,
|
||||
);
|
||||
protocols.insert(
|
||||
PODCAST_DHT_ID.to_string(),
|
||||
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
||||
);
|
||||
protocols.insert(
|
||||
DEVICE_SYNC_ID.to_string(),
|
||||
crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION,
|
||||
@@ -105,6 +111,18 @@ impl CapabilityManifest {
|
||||
self
|
||||
}
|
||||
|
||||
/// Version of protocol `id` advertised by the peer, if any.
|
||||
pub fn protocol_version(&self, id: &str) -> Option<u16> {
|
||||
self.protocols.get(id).copied()
|
||||
}
|
||||
|
||||
/// Whether the peer advertises the public podcast-discovery overlay at
|
||||
/// the version this crate speaks. Peers without it are never queried.
|
||||
pub fn supports_podcast_dht(&self) -> bool {
|
||||
self.protocol_version(PODCAST_DHT_ID)
|
||||
== Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
}
|
||||
|
||||
/// Validates bounds and identifiers received from a peer.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.manifest_version != CAPABILITIES_PROTOCOL_VERSION {
|
||||
@@ -219,6 +237,7 @@ mod tests {
|
||||
CATALOG_ID,
|
||||
SIMILARITY_ID,
|
||||
SIMILARITY_DHT_ID,
|
||||
PODCAST_DHT_ID,
|
||||
DEVICE_SYNC_ID,
|
||||
JAM_ID,
|
||||
] {
|
||||
@@ -227,6 +246,20 @@ mod tests {
|
||||
manifest.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_dht_capability_is_advertised_and_removable() {
|
||||
let manifest = CapabilityManifest::frid("furumi", "1.2.3");
|
||||
assert_eq!(
|
||||
manifest.protocol_version(PODCAST_DHT_ID),
|
||||
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
);
|
||||
assert!(manifest.supports_podcast_dht());
|
||||
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
|
||||
assert!(!without.supports_podcast_dht());
|
||||
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
|
||||
assert!(!other_version.supports_podcast_dht());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn application_protocol_is_additive() {
|
||||
let manifest = CapabilityManifest::frid("furumi", "1.2.3").with_protocol("audio", 1);
|
||||
|
||||
@@ -50,6 +50,11 @@ pub enum MusicDhtError {
|
||||
/// The service is shutting down and no longer accepts operations.
|
||||
#[error("service is shutting down")]
|
||||
ShuttingDown,
|
||||
|
||||
/// A podcast card cannot be published, for example because the user did
|
||||
/// not share the podcast (see `music_dht::podcast::effective_share`).
|
||||
#[error("podcast publication refused: {0}")]
|
||||
PodcastPublication(String),
|
||||
}
|
||||
|
||||
impl From<federation_net::NetworkError> for MusicDhtError {
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
//! podcast/episode ids, metadata DTOs and convergent merge rules), with an
|
||||
//! optional RSS/Atom parser behind the `podcast-feed` feature;
|
||||
//! [`device_sync`] protocol v4 carries it between trusted devices.
|
||||
//! * [`podcast_dht`] is a separate signed overlay for public podcast
|
||||
//! discovery: cards a user explicitly shared, found by title/author tokens
|
||||
//! or podcast id, retracted by signed tombstones.
|
||||
//! * Records are replicated to the `K` nodes whose ids are XOR-closest to
|
||||
//! each key. Publishers pick the targets from their routing table and send
|
||||
//! batched store requests (one pipeline per peer), so even a large library
|
||||
@@ -97,6 +100,7 @@ mod message;
|
||||
mod node;
|
||||
mod normalization;
|
||||
pub mod podcast;
|
||||
pub mod podcast_dht;
|
||||
mod record;
|
||||
mod request;
|
||||
mod routing;
|
||||
|
||||
@@ -0,0 +1,898 @@
|
||||
//! Signed public podcast cards, tombstones, DHT keys and their canonical
|
||||
//! signing encoding.
|
||||
//!
|
||||
//! Everything here is pure and deterministic: no clock, no I/O. Callers pass
|
||||
//! the current time to the validation functions.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
use federation_net::{EndpointId, NetworkId, Signature};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{MusicDhtError, Result};
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::{
|
||||
FeedUrl, MAX_URL_BYTES, PodcastId, SubscriptionRecord, bounded_text, is_http_url,
|
||||
};
|
||||
|
||||
use super::privacy::{FeedUrlPrivacyWarning, feed_url_privacy_warning};
|
||||
|
||||
/// Current card, tombstone and overlay wire version.
|
||||
pub const PODCAST_DHT_PROTOCOL_VERSION: u16 = 1;
|
||||
/// Signing domain of a [`PodcastCard`]. Changing it invalidates every card.
|
||||
pub const PODCAST_CARD_SIGNING_DOMAIN: &str = "furumi-podcast-card:v1";
|
||||
/// Signing domain of a [`PodcastTombstone`].
|
||||
pub const PODCAST_TOMBSTONE_SIGNING_DOMAIN: &str = "furumi-podcast-card-tombstone:v1";
|
||||
/// BLAKE3 derivation domain of [`PodcastDhtKey`].
|
||||
pub const PODCAST_DHT_KEY_DOMAIN: &str = "furumi-podcast-dht:key:v1";
|
||||
|
||||
/// Maximum lifetime of a card: the main DHT's active-record TTL.
|
||||
pub const PODCAST_CARD_TTL: Duration = crate::record::ACTIVE_RECORD_TTL;
|
||||
/// Maximum lifetime of a tombstone: the main DHT's tombstone TTL. It exceeds
|
||||
/// [`PODCAST_CARD_TTL`], so a card issued before a tombstone always expires
|
||||
/// before the tombstone does.
|
||||
pub const PODCAST_TOMBSTONE_TTL: Duration = crate::record::TOMBSTONE_TTL;
|
||||
/// Tolerated clock skew for `issued_at_ms` in the future.
|
||||
pub const PODCAST_DHT_MAX_CLOCK_SKEW: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// Maximum characters of a card title.
|
||||
pub const MAX_CARD_TITLE_CHARS: usize = 300;
|
||||
/// Maximum characters of a card author.
|
||||
pub const MAX_CARD_AUTHOR_CHARS: usize = 300;
|
||||
/// Maximum characters of a card description.
|
||||
pub const MAX_CARD_DESCRIPTION_CHARS: usize = 1_000;
|
||||
/// Maximum bytes of a card language tag (BCP 47 shape: ASCII letters,
|
||||
/// digits and `-`).
|
||||
pub const MAX_CARD_LANGUAGE_BYTES: usize = 35;
|
||||
/// Maximum categories on a card.
|
||||
pub const MAX_CARD_CATEGORIES: usize = 8;
|
||||
/// Maximum characters of one category.
|
||||
pub const MAX_CARD_CATEGORY_CHARS: usize = 64;
|
||||
/// Maximum indexed search tokens of a card (title tokens, then author tokens).
|
||||
pub const MAX_CARD_INDEX_TOKENS: usize = 32;
|
||||
/// Maximum bytes of one indexed or query token. Longer tokens are not indexed.
|
||||
pub const MAX_INDEX_TOKEN_BYTES: usize = 64;
|
||||
/// Maximum DHT keys a record is stored under: the podcast-id key plus one
|
||||
/// key per indexed token.
|
||||
pub const MAX_RECORD_DHT_KEYS: usize = MAX_CARD_INDEX_TOKENS + 1;
|
||||
/// Maximum postcard-encoded size of one signed record.
|
||||
pub const MAX_PODCAST_RECORD_BYTES: usize = 24 * 1024;
|
||||
|
||||
const KEY_KIND_PODCAST: u8 = 1;
|
||||
const KEY_KIND_TOKEN: u8 = 2;
|
||||
|
||||
/// A 256-bit point in the podcast-discovery key space.
|
||||
///
|
||||
/// `BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind || material)`
|
||||
/// with kind `0x01` and the podcast id string for the id key, or kind `0x02`
|
||||
/// and a normalized token for a text key.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub struct PodcastDhtKey([u8; 32]);
|
||||
|
||||
impl PodcastDhtKey {
|
||||
/// Key under which every card and tombstone of `podcast_id` is stored.
|
||||
pub fn podcast(network_id: &NetworkId, podcast_id: &PodcastId) -> Self {
|
||||
Self::derive(network_id, KEY_KIND_PODCAST, podcast_id.as_str())
|
||||
}
|
||||
|
||||
/// Key of one normalized search token (see [`card_index_tokens`]).
|
||||
pub fn token(network_id: &NetworkId, token: &str) -> Self {
|
||||
Self::derive(network_id, KEY_KIND_TOKEN, token)
|
||||
}
|
||||
|
||||
fn derive(network_id: &NetworkId, kind: u8, material: &str) -> Self {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update(PODCAST_DHT_KEY_DOMAIN.as_bytes());
|
||||
hasher.update(&[0]);
|
||||
hasher.update(network_id.as_bytes());
|
||||
hasher.update(&[kind]);
|
||||
hasher.update(material.as_bytes());
|
||||
Self(*hasher.finalize().as_bytes())
|
||||
}
|
||||
|
||||
/// Creates a key from raw bytes received on the wire.
|
||||
pub fn from_bytes(bytes: [u8; 32]) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
/// Raw key bytes.
|
||||
pub fn as_bytes(&self) -> &[u8; 32] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed search tokens of a card: distinct [`normalize_name`] tokens of the
|
||||
/// title, then of the author, in order, skipping tokens longer than
|
||||
/// [`MAX_INDEX_TOKEN_BYTES`], at most [`MAX_CARD_INDEX_TOKENS`].
|
||||
pub fn card_index_tokens(title: &str, author: Option<&str>) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut tokens = Vec::new();
|
||||
let sources = std::iter::once(title).chain(author);
|
||||
for source in sources {
|
||||
for token in tokenize(&normalize_name(source)) {
|
||||
if tokens.len() == MAX_CARD_INDEX_TOKENS {
|
||||
return tokens;
|
||||
}
|
||||
if token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()) {
|
||||
tokens.push(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
/// Unsigned public projection of a podcast prepared by its publisher.
|
||||
///
|
||||
/// It is built from the user's subscription ([`PodcastCardSpec::from_subscription`])
|
||||
/// and contains only publisher metadata: never progress, listening history,
|
||||
/// subscription times, operation ids, device names or episodes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastCardSpec {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// URL `podcast_id` was derived from, when it differs from `feed_url`.
|
||||
pub id_feed_url: Option<FeedUrl>,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description.
|
||||
pub description: Option<String>,
|
||||
/// Artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
/// Publisher web page.
|
||||
pub link: Option<String>,
|
||||
/// Feed language tag, for example `en` or `pt-BR`.
|
||||
pub language: Option<String>,
|
||||
/// Feed categories.
|
||||
pub categories: Vec<String>,
|
||||
/// Number of episodes in the feed, when known.
|
||||
pub episode_count: Option<u32>,
|
||||
/// Publication time of the newest episode in unix milliseconds.
|
||||
pub latest_episode_at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
impl PodcastCardSpec {
|
||||
/// Builds the bounded public projection of a subscription.
|
||||
///
|
||||
/// Only `podcast_id`, the feed URLs and the channel metadata are read.
|
||||
/// `id_feed_url` is the alias that hashes to `podcast_id`
|
||||
/// ([`crate::podcast::merge_feed_aliases`] always keeps it). Fails when no
|
||||
/// such URL is known. Callers may then set `language`, `categories`,
|
||||
/// `episode_count` and `latest_episode_at_ms` from their feed cache.
|
||||
pub fn from_subscription(subscription: &SubscriptionRecord) -> Result<Self> {
|
||||
let origin = subscription
|
||||
.feed_urls()
|
||||
.into_iter()
|
||||
.find(|url| PodcastId::from_feed_url(url) == subscription.podcast_id)
|
||||
.ok_or_else(|| {
|
||||
publication_error("the feed URL the podcast id was derived from is unknown")
|
||||
})?;
|
||||
let feed_url = subscription.podcast.feed_url.clone();
|
||||
let meta = &subscription.podcast;
|
||||
Ok(Self {
|
||||
podcast_id: subscription.podcast_id.clone(),
|
||||
id_feed_url: (origin != feed_url).then_some(origin),
|
||||
feed_url,
|
||||
title: meta.title.clone(),
|
||||
author: meta.author.clone(),
|
||||
description: meta.description.clone(),
|
||||
image_url: meta.image_url.clone(),
|
||||
link: meta.link.clone(),
|
||||
language: None,
|
||||
categories: Vec::new(),
|
||||
episode_count: None,
|
||||
latest_episode_at_ms: None,
|
||||
}
|
||||
.bounded())
|
||||
}
|
||||
|
||||
/// Returns a copy that fits the card bounds: text is truncated with
|
||||
/// [`bounded_text`], control characters are removed, invalid URLs and
|
||||
/// language tags are dropped, and categories are trimmed, deduplicated and
|
||||
/// bounded. An empty title becomes `"Untitled podcast"`.
|
||||
pub fn bounded(&self) -> Self {
|
||||
let title = clean_line(&self.title);
|
||||
let mut seen = HashSet::new();
|
||||
let categories = self
|
||||
.categories
|
||||
.iter()
|
||||
.map(|category| bounded_text(&clean_line(category), MAX_CARD_CATEGORY_CHARS))
|
||||
.filter(|category| !category.is_empty() && seen.insert(category.to_lowercase()))
|
||||
.take(MAX_CARD_CATEGORIES)
|
||||
.collect();
|
||||
Self {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
feed_url: self.feed_url.clone(),
|
||||
id_feed_url: self
|
||||
.id_feed_url
|
||||
.clone()
|
||||
.filter(|origin| *origin != self.feed_url),
|
||||
title: if title.is_empty() {
|
||||
"Untitled podcast".to_string()
|
||||
} else {
|
||||
bounded_text(&title, MAX_CARD_TITLE_CHARS)
|
||||
},
|
||||
author: bounded_line(self.author.as_deref(), MAX_CARD_AUTHOR_CHARS),
|
||||
description: self
|
||||
.description
|
||||
.as_deref()
|
||||
.map(|text| bounded_text(clean_block(text).trim(), MAX_CARD_DESCRIPTION_CHARS))
|
||||
.filter(|text| !text.is_empty()),
|
||||
image_url: self.image_url.clone().filter(|url| is_http_url(url)),
|
||||
link: self.link.clone().filter(|url| is_http_url(url)),
|
||||
language: self
|
||||
.language
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|language| valid_language(language))
|
||||
.map(str::to_string),
|
||||
categories,
|
||||
episode_count: self.episode_count,
|
||||
latest_episode_at_ms: self.latest_episode_at_ms.filter(|at| *at >= 0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the card bounds without bounding anything.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let id_url = self.id_feed_url.as_ref().unwrap_or(&self.feed_url);
|
||||
if PodcastId::from_feed_url(id_url) != self.podcast_id {
|
||||
return Err(invalid("podcast id does not match its feed URL"));
|
||||
}
|
||||
if self.id_feed_url.as_ref() == Some(&self.feed_url) {
|
||||
return Err(invalid("id_feed_url must differ from feed_url"));
|
||||
}
|
||||
check_line("title", &self.title, MAX_CARD_TITLE_CHARS, true)?;
|
||||
if let Some(author) = &self.author {
|
||||
check_line("author", author, MAX_CARD_AUTHOR_CHARS, false)?;
|
||||
}
|
||||
if let Some(description) = &self.description {
|
||||
check_chars("description", description, MAX_CARD_DESCRIPTION_CHARS)?;
|
||||
if description
|
||||
.chars()
|
||||
.any(|c| c.is_control() && c != '\n' && c != '\t')
|
||||
{
|
||||
return Err(invalid("description contains control characters"));
|
||||
}
|
||||
}
|
||||
for url in [&self.image_url, &self.link].into_iter().flatten() {
|
||||
if !is_http_url(url) {
|
||||
return Err(invalid("card URLs must be absolute http(s) URLs"));
|
||||
}
|
||||
}
|
||||
if self
|
||||
.language
|
||||
.as_deref()
|
||||
.is_some_and(|language| !valid_language(language))
|
||||
{
|
||||
return Err(invalid("invalid language tag"));
|
||||
}
|
||||
if self.categories.len() > MAX_CARD_CATEGORIES {
|
||||
return Err(invalid("too many categories"));
|
||||
}
|
||||
for category in &self.categories {
|
||||
check_line("category", category, MAX_CARD_CATEGORY_CHARS, true)?;
|
||||
}
|
||||
if self.latest_episode_at_ms.is_some_and(|at| at < 0) {
|
||||
return Err(invalid("negative latest_episode_at_ms"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Advisory privacy warnings for both feed URLs of the card (see
|
||||
/// [`feed_url_privacy_warning`]). The UI must ask before publishing a
|
||||
/// card with warnings.
|
||||
pub fn privacy_warnings(&self) -> Vec<FeedUrlPrivacyWarning> {
|
||||
std::iter::once(&self.feed_url)
|
||||
.chain(self.id_feed_url.as_ref())
|
||||
.filter_map(feed_url_privacy_warning)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Indexed search tokens of this card.
|
||||
pub fn index_tokens(&self) -> Vec<String> {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed public podcast card: one publisher device's projection of one
|
||||
/// podcast. See `PODCASTS.md`, "Public discovery (podcast_dht)".
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastCard {
|
||||
/// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
pub version: u16,
|
||||
/// Network scope; prevents replay into another federation.
|
||||
pub network_id: NetworkId,
|
||||
/// Publisher device (its persistent transport identity); signs the card.
|
||||
pub publisher: EndpointId,
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// URL `podcast_id` was derived from, when it differs from `feed_url`.
|
||||
pub id_feed_url: Option<FeedUrl>,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description.
|
||||
pub description: Option<String>,
|
||||
/// Artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
/// Publisher web page.
|
||||
pub link: Option<String>,
|
||||
/// Feed language tag.
|
||||
pub language: Option<String>,
|
||||
/// Feed categories.
|
||||
pub categories: Vec<String>,
|
||||
/// Number of episodes in the feed, when known.
|
||||
pub episode_count: Option<u32>,
|
||||
/// Publication time of the newest episode in unix milliseconds.
|
||||
pub latest_episode_at_ms: Option<i64>,
|
||||
/// Monotonic per `(podcast_id, publisher)` revision.
|
||||
pub revision: u64,
|
||||
/// Unix time at which this version was signed.
|
||||
pub issued_at_ms: u64,
|
||||
/// Unix time after which the card must be discarded.
|
||||
pub expires_at_ms: u64,
|
||||
}
|
||||
|
||||
impl PodcastCard {
|
||||
/// Builds a card from a spec. `expires_at_ms` is
|
||||
/// `issued_at_ms + PODCAST_CARD_TTL`.
|
||||
pub fn from_spec(
|
||||
spec: PodcastCardSpec,
|
||||
network_id: NetworkId,
|
||||
publisher: EndpointId,
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
network_id,
|
||||
publisher,
|
||||
podcast_id: spec.podcast_id,
|
||||
feed_url: spec.feed_url,
|
||||
id_feed_url: spec.id_feed_url,
|
||||
title: spec.title,
|
||||
author: spec.author,
|
||||
description: spec.description,
|
||||
image_url: spec.image_url,
|
||||
link: spec.link,
|
||||
language: spec.language,
|
||||
categories: spec.categories,
|
||||
episode_count: spec.episode_count,
|
||||
latest_episode_at_ms: spec.latest_episode_at_ms,
|
||||
revision,
|
||||
issued_at_ms,
|
||||
expires_at_ms: issued_at_ms.saturating_add(PODCAST_CARD_TTL.as_millis() as u64),
|
||||
}
|
||||
}
|
||||
|
||||
/// The publisher-provided part of the card.
|
||||
pub fn spec(&self) -> PodcastCardSpec {
|
||||
PodcastCardSpec {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
feed_url: self.feed_url.clone(),
|
||||
id_feed_url: self.id_feed_url.clone(),
|
||||
title: self.title.clone(),
|
||||
author: self.author.clone(),
|
||||
description: self.description.clone(),
|
||||
image_url: self.image_url.clone(),
|
||||
link: self.link.clone(),
|
||||
language: self.language.clone(),
|
||||
categories: self.categories.clone(),
|
||||
episode_count: self.episode_count,
|
||||
latest_episode_at_ms: self.latest_episode_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed search tokens.
|
||||
pub fn index_tokens(&self) -> Vec<String> {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
|
||||
out.u16(self.version);
|
||||
out.raw(self.network_id.as_bytes());
|
||||
out.raw(self.publisher.as_bytes());
|
||||
out.str(self.podcast_id.as_str());
|
||||
out.str(self.feed_url.as_str());
|
||||
out.opt_str(self.id_feed_url.as_ref().map(FeedUrl::as_str));
|
||||
out.str(&self.title);
|
||||
out.opt_str(self.author.as_deref());
|
||||
out.opt_str(self.description.as_deref());
|
||||
out.opt_str(self.image_url.as_deref());
|
||||
out.opt_str(self.link.as_deref());
|
||||
out.opt_str(self.language.as_deref());
|
||||
out.u32(self.categories.len() as u32);
|
||||
for category in &self.categories {
|
||||
out.str(category);
|
||||
}
|
||||
match self.episode_count {
|
||||
Some(count) => {
|
||||
out.u8(1);
|
||||
out.u32(count);
|
||||
}
|
||||
None => out.u8(0),
|
||||
}
|
||||
match self.latest_episode_at_ms {
|
||||
Some(at) => {
|
||||
out.u8(1);
|
||||
out.i64(at);
|
||||
}
|
||||
None => out.u8(0),
|
||||
}
|
||||
out.u64(self.revision);
|
||||
out.u64(self.issued_at_ms);
|
||||
out.u64(self.expires_at_ms);
|
||||
out.finish()
|
||||
}
|
||||
|
||||
fn validate(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
check_header(self.version, &self.network_id, expected_network)?;
|
||||
self.spec().validate()?;
|
||||
check_lifetime(
|
||||
self.revision,
|
||||
self.issued_at_ms,
|
||||
self.expires_at_ms,
|
||||
PODCAST_CARD_TTL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed retraction of one publisher's card of one podcast.
|
||||
///
|
||||
/// It supersedes every card of the same `(podcast_id, publisher)` with a
|
||||
/// revision lower than or equal to its own and is retained until it expires.
|
||||
/// `index_keys` lists the text keys of the retracted cards so the tombstone
|
||||
/// can be stored next to them; it carries hashes, not the tokens.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastTombstone {
|
||||
/// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
pub version: u16,
|
||||
/// Network scope.
|
||||
pub network_id: NetworkId,
|
||||
/// Publisher device; signs the tombstone.
|
||||
pub publisher: EndpointId,
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Revision; greater than that of every retracted card.
|
||||
pub revision: u64,
|
||||
/// Unix time at which the tombstone was signed.
|
||||
pub issued_at_ms: u64,
|
||||
/// Unix time after which the tombstone may be discarded.
|
||||
pub expires_at_ms: u64,
|
||||
/// Sorted, distinct text keys besides the podcast-id key, at most
|
||||
/// [`MAX_CARD_INDEX_TOKENS`].
|
||||
pub index_keys: Vec<PodcastDhtKey>,
|
||||
}
|
||||
|
||||
impl PodcastTombstone {
|
||||
/// Builds a tombstone. `expires_at_ms` is `issued_at_ms + PODCAST_TOMBSTONE_TTL`;
|
||||
/// `index_keys` is sorted, deduplicated and stripped of the podcast-id key.
|
||||
pub fn new(
|
||||
network_id: NetworkId,
|
||||
publisher: EndpointId,
|
||||
podcast_id: PodcastId,
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
index_keys: impl IntoIterator<Item = PodcastDhtKey>,
|
||||
) -> Self {
|
||||
let id_key = PodcastDhtKey::podcast(&network_id, &podcast_id);
|
||||
let mut index_keys: Vec<_> = index_keys
|
||||
.into_iter()
|
||||
.filter(|key| *key != id_key)
|
||||
.collect();
|
||||
index_keys.sort();
|
||||
index_keys.dedup();
|
||||
index_keys.truncate(MAX_CARD_INDEX_TOKENS);
|
||||
Self {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
network_id,
|
||||
publisher,
|
||||
podcast_id,
|
||||
revision,
|
||||
issued_at_ms,
|
||||
expires_at_ms: issued_at_ms.saturating_add(PODCAST_TOMBSTONE_TTL.as_millis() as u64),
|
||||
index_keys,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_TOMBSTONE_SIGNING_DOMAIN);
|
||||
out.u16(self.version);
|
||||
out.raw(self.network_id.as_bytes());
|
||||
out.raw(self.publisher.as_bytes());
|
||||
out.str(self.podcast_id.as_str());
|
||||
out.u64(self.revision);
|
||||
out.u64(self.issued_at_ms);
|
||||
out.u64(self.expires_at_ms);
|
||||
out.u32(self.index_keys.len() as u32);
|
||||
for key in &self.index_keys {
|
||||
out.raw(key.as_bytes());
|
||||
}
|
||||
out.finish()
|
||||
}
|
||||
|
||||
fn validate(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
check_header(self.version, &self.network_id, expected_network)?;
|
||||
if self.index_keys.len() > MAX_CARD_INDEX_TOKENS {
|
||||
return Err(invalid("tombstone has too many keys"));
|
||||
}
|
||||
let id_key = PodcastDhtKey::podcast(&self.network_id, &self.podcast_id);
|
||||
if self.index_keys.contains(&id_key)
|
||||
|| self.index_keys.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
{
|
||||
return Err(invalid("tombstone keys are not canonical"));
|
||||
}
|
||||
check_lifetime(
|
||||
self.revision,
|
||||
self.issued_at_ms,
|
||||
self.expires_at_ms,
|
||||
PODCAST_TOMBSTONE_TTL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Body of a signed podcast-discovery record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum PodcastRecordBody {
|
||||
/// An active public card.
|
||||
Card(Box<PodcastCard>),
|
||||
/// A retraction.
|
||||
Tombstone(PodcastTombstone),
|
||||
}
|
||||
|
||||
/// End-to-end signed podcast-discovery record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SignedPodcastRecord {
|
||||
/// Signed body.
|
||||
pub body: PodcastRecordBody,
|
||||
/// Ed25519 signature by `publisher` over [`Self::signing_bytes`].
|
||||
pub signature: Signature,
|
||||
}
|
||||
|
||||
impl SignedPodcastRecord {
|
||||
/// Validates and signs a card.
|
||||
pub fn sign_card(card: PodcastCard, signer: impl FnOnce(&[u8]) -> Signature) -> Result<Self> {
|
||||
card.validate(&card.network_id)?;
|
||||
let signature = signer(&card.signing_bytes());
|
||||
let record = Self {
|
||||
body: PodcastRecordBody::Card(Box::new(card)),
|
||||
signature,
|
||||
};
|
||||
record.check_size()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Validates and signs a tombstone.
|
||||
pub fn sign_tombstone(
|
||||
tombstone: PodcastTombstone,
|
||||
signer: impl FnOnce(&[u8]) -> Signature,
|
||||
) -> Result<Self> {
|
||||
tombstone.validate(&tombstone.network_id)?;
|
||||
let signature = signer(&tombstone.signing_bytes());
|
||||
let record = Self {
|
||||
body: PodcastRecordBody::Tombstone(tombstone),
|
||||
signature,
|
||||
};
|
||||
record.check_size()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Canonical signing input of the body.
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.signing_bytes(),
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.signing_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates bounds, network, lifetime shape, encoded size and the
|
||||
/// publisher's signature. Does not look at the clock; see
|
||||
/// [`Self::verify_at`].
|
||||
pub fn verify(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.validate(expected_network)?,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.validate(expected_network)?,
|
||||
}
|
||||
self.check_size()?;
|
||||
self.publisher()
|
||||
.verify(&self.signing_bytes(), &self.signature)
|
||||
.map_err(|_| invalid("invalid podcast record signature"))
|
||||
}
|
||||
|
||||
/// [`Self::verify`] plus time checks: not issued more than
|
||||
/// [`PODCAST_DHT_MAX_CLOCK_SKEW`] in the future and not expired at `now_ms`.
|
||||
pub fn verify_at(&self, expected_network: &NetworkId, now_ms: u64) -> Result<()> {
|
||||
self.verify(expected_network)?;
|
||||
let future_limit = now_ms.saturating_add(PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64);
|
||||
if self.issued_at_ms() > future_limit || self.expires_at_ms() <= now_ms {
|
||||
return Err(invalid("podcast record is expired or from the future"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The card, when this is not a tombstone.
|
||||
pub fn card(&self) -> Option<&PodcastCard> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => Some(card.as_ref()),
|
||||
PodcastRecordBody::Tombstone(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this is a tombstone.
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
matches!(self.body, PodcastRecordBody::Tombstone(_))
|
||||
}
|
||||
|
||||
/// Podcast.
|
||||
pub fn podcast_id(&self) -> &PodcastId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => &card.podcast_id,
|
||||
PodcastRecordBody::Tombstone(tombstone) => &tombstone.podcast_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Publisher device.
|
||||
pub fn publisher(&self) -> EndpointId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.publisher,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.publisher,
|
||||
}
|
||||
}
|
||||
|
||||
/// Network scope.
|
||||
pub fn network_id(&self) -> NetworkId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.network_id,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.network_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Revision.
|
||||
pub fn revision(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.revision,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.revision,
|
||||
}
|
||||
}
|
||||
|
||||
/// Signing time.
|
||||
pub fn issued_at_ms(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.issued_at_ms,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.issued_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Expiry time.
|
||||
pub fn expires_at_ms(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.expires_at_ms,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.expires_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every DHT key this record belongs under: the podcast-id key, then the
|
||||
/// text keys (card tokens or the tombstone's `index_keys`).
|
||||
pub fn dht_keys(&self) -> Vec<PodcastDhtKey> {
|
||||
let network_id = self.network_id();
|
||||
let mut keys = vec![PodcastDhtKey::podcast(&network_id, self.podcast_id())];
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => keys.extend(
|
||||
card.index_tokens()
|
||||
.iter()
|
||||
.map(|token| PodcastDhtKey::token(&network_id, token)),
|
||||
),
|
||||
PodcastRecordBody::Tombstone(tombstone) => {
|
||||
keys.extend(tombstone.index_keys.iter().copied());
|
||||
}
|
||||
}
|
||||
keys
|
||||
}
|
||||
|
||||
/// Whether the record may be stored under `key`.
|
||||
pub fn belongs_under(&self, key: &PodcastDhtKey) -> bool {
|
||||
self.dht_keys().contains(key)
|
||||
}
|
||||
|
||||
/// postcard-encoded size.
|
||||
pub fn encoded_len(&self) -> usize {
|
||||
postcard::to_stdvec(self).map_or(usize::MAX, |bytes| bytes.len())
|
||||
}
|
||||
|
||||
fn check_size(&self) -> Result<()> {
|
||||
if self.encoded_len() > MAX_PODCAST_RECORD_BYTES {
|
||||
Err(invalid("podcast record is too large"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Total order of two records of the same `(podcast_id, publisher)`.
|
||||
///
|
||||
/// Greater wins: higher `revision`; at an equal revision a tombstone beats a
|
||||
/// card; then later `issued_at_ms` (a republish); then later
|
||||
/// `expires_at_ms`; then the lexicographically greater canonical signing
|
||||
/// input. Every node therefore keeps the same winner whatever the delivery
|
||||
/// order.
|
||||
pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) -> Ordering {
|
||||
left.revision()
|
||||
.cmp(&right.revision())
|
||||
.then_with(|| left.is_tombstone().cmp(&right.is_tombstone()))
|
||||
.then_with(|| left.issued_at_ms().cmp(&right.issued_at_ms()))
|
||||
.then_with(|| left.expires_at_ms().cmp(&right.expires_at_ms()))
|
||||
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
|
||||
}
|
||||
|
||||
/// Whether `winner` makes `other` obsolete everywhere, not just under one key:
|
||||
/// `other` is a card of a lower revision, or `winner` is a tombstone of an
|
||||
/// equal or higher revision.
|
||||
pub(crate) fn supersedes_everywhere(
|
||||
winner: &SignedPodcastRecord,
|
||||
other: &SignedPodcastRecord,
|
||||
) -> bool {
|
||||
!other.is_tombstone()
|
||||
&& (other.revision() < winner.revision()
|
||||
|| (winner.is_tombstone() && other.revision() <= winner.revision()))
|
||||
}
|
||||
|
||||
struct Canonical(Vec<u8>);
|
||||
|
||||
impl Canonical {
|
||||
fn new(domain: &str) -> Self {
|
||||
let mut bytes = Vec::with_capacity(512);
|
||||
bytes.extend_from_slice(domain.as_bytes());
|
||||
bytes.push(0);
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
fn raw(&mut self, bytes: &[u8]) {
|
||||
self.0.extend_from_slice(bytes);
|
||||
}
|
||||
|
||||
fn u8(&mut self, value: u8) {
|
||||
self.0.push(value);
|
||||
}
|
||||
|
||||
fn u16(&mut self, value: u16) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn u32(&mut self, value: u32) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn u64(&mut self, value: u64) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn i64(&mut self, value: i64) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn str(&mut self, value: &str) {
|
||||
self.u32(value.len() as u32);
|
||||
self.raw(value.as_bytes());
|
||||
}
|
||||
|
||||
fn opt_str(&mut self, value: Option<&str>) {
|
||||
match value {
|
||||
Some(value) => {
|
||||
self.u8(1);
|
||||
self.str(value);
|
||||
}
|
||||
None => self.u8(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<u8> {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
fn check_header(version: u16, network_id: &NetworkId, expected: &NetworkId) -> Result<()> {
|
||||
if version != PODCAST_DHT_PROTOCOL_VERSION {
|
||||
return Err(invalid("unsupported podcast record version"));
|
||||
}
|
||||
if network_id != expected {
|
||||
return Err(invalid("podcast record belongs to another network"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_lifetime(
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
expires_at_ms: u64,
|
||||
ttl: Duration,
|
||||
) -> Result<()> {
|
||||
if revision == 0
|
||||
|| issued_at_ms == 0
|
||||
|| expires_at_ms <= issued_at_ms
|
||||
|| expires_at_ms - issued_at_ms > ttl.as_millis() as u64
|
||||
{
|
||||
return Err(invalid("invalid podcast record revision or lifetime"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_chars(field: &'static str, value: &str, max_chars: usize) -> Result<()> {
|
||||
if value.len() > max_chars * 4 || value.chars().count() > max_chars {
|
||||
return Err(MusicDhtError::Protocol(format!("card {field} is too long")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_line(field: &'static str, value: &str, max_chars: usize, required: bool) -> Result<()> {
|
||||
if required && value.trim().is_empty() {
|
||||
return Err(MusicDhtError::Protocol(format!("card {field} is empty")));
|
||||
}
|
||||
check_chars(field, value, max_chars)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(MusicDhtError::Protocol(format!(
|
||||
"card {field} contains control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn valid_language(language: &str) -> bool {
|
||||
!language.is_empty()
|
||||
&& language.len() <= MAX_CARD_LANGUAGE_BYTES
|
||||
&& language
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
}
|
||||
|
||||
fn clean_line(text: &str) -> String {
|
||||
text.split(char::is_control)
|
||||
.filter(|part| !part.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn clean_block(text: &str) -> String {
|
||||
text.chars()
|
||||
.filter(|c| !c.is_control() || *c == '\n' || *c == '\t')
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bounded_line(text: Option<&str>, max_chars: usize) -> Option<String> {
|
||||
text.map(clean_line)
|
||||
.filter(|text| !text.is_empty())
|
||||
.map(|text| bounded_text(&text, max_chars))
|
||||
}
|
||||
|
||||
fn invalid(reason: &str) -> MusicDhtError {
|
||||
MusicDhtError::Protocol(reason.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn publication_error(reason: &str) -> MusicDhtError {
|
||||
MusicDhtError::PodcastPublication(reason.to_string())
|
||||
}
|
||||
|
||||
// Statically bound the worst-case card so `MAX_PODCAST_RECORD_BYTES` is a
|
||||
// defensive limit, not the one users hit first.
|
||||
const _: () = assert!(
|
||||
MAX_CARD_TITLE_CHARS * 4
|
||||
+ MAX_CARD_AUTHOR_CHARS * 4
|
||||
+ MAX_CARD_DESCRIPTION_CHARS * 4
|
||||
+ MAX_CARD_CATEGORIES * MAX_CARD_CATEGORY_CHARS * 4
|
||||
+ MAX_CARD_LANGUAGE_BYTES
|
||||
+ 4 * MAX_URL_BYTES
|
||||
+ 1024
|
||||
<= MAX_PODCAST_RECORD_BYTES
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,339 @@
|
||||
//! Advisory detection of feed URLs that look private.
|
||||
//!
|
||||
//! Premium and member feeds usually carry a personal secret in the URL. A
|
||||
//! card publishes its feed URL verbatim, so the UI must ask the user before
|
||||
//! publishing a URL for which [`feed_url_privacy_warning`] returns a warning.
|
||||
//! The heuristics are deliberately broad and can produce false positives
|
||||
//! (and miss secrets); they never block publication by themselves.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::podcast::FeedUrl;
|
||||
|
||||
/// Query-parameter name words that usually carry a credential. A name is
|
||||
/// split into lowercase words at non-alphanumeric characters and camelCase
|
||||
/// boundaries; any listed word triggers a warning.
|
||||
pub const SECRET_QUERY_WORDS: &[&str] = &[
|
||||
"token",
|
||||
"key",
|
||||
"auth",
|
||||
"secret",
|
||||
"sig",
|
||||
"signature",
|
||||
"uid",
|
||||
"user",
|
||||
"pass",
|
||||
"password",
|
||||
"passwd",
|
||||
"pwd",
|
||||
"session",
|
||||
"sess",
|
||||
"sid",
|
||||
"access",
|
||||
"private",
|
||||
"credential",
|
||||
"credentials",
|
||||
"apikey",
|
||||
"jwt",
|
||||
"hmac",
|
||||
];
|
||||
|
||||
/// Substrings that trigger a warning anywhere in a query-parameter name,
|
||||
/// for names such as `apitoken` or `privatefeed` that do not split.
|
||||
const SECRET_QUERY_SUBSTRINGS: &[&str] = &[
|
||||
"token",
|
||||
"secret",
|
||||
"signature",
|
||||
"password",
|
||||
"passwd",
|
||||
"session",
|
||||
"private",
|
||||
"credential",
|
||||
"apikey",
|
||||
];
|
||||
|
||||
/// Hosts (and their subdomains) known to serve personal member feeds.
|
||||
pub const PRIVATE_FEED_HOSTS: &[&str] = &[
|
||||
"patreon.com",
|
||||
"supercast.com",
|
||||
"supportingcast.fm",
|
||||
"glow.fm",
|
||||
];
|
||||
|
||||
/// Minimum length of a query value considered a possible secret.
|
||||
const MIN_SECRET_VALUE_CHARS: usize = 16;
|
||||
/// Minimum length of a path segment considered a possible secret.
|
||||
const MIN_SECRET_SEGMENT_CHARS: usize = 32;
|
||||
/// Minimum Shannon entropy, in bits per character, of a possible secret.
|
||||
const MIN_SECRET_ENTROPY_BITS: f64 = 3.0;
|
||||
|
||||
/// Why a feed URL may contain a personal secret.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "reason", rename_all = "snake_case")]
|
||||
#[non_exhaustive]
|
||||
pub enum FeedUrlPrivacyWarning {
|
||||
/// A query parameter whose name suggests a credential.
|
||||
SecretQueryParameter {
|
||||
/// Parameter name as it appears in the URL.
|
||||
name: String,
|
||||
},
|
||||
/// The host is known to serve personal member feeds.
|
||||
PrivateFeedHost {
|
||||
/// Lowercase host.
|
||||
host: String,
|
||||
},
|
||||
/// A long, random-looking query value.
|
||||
HighEntropyQueryValue {
|
||||
/// Name of the parameter carrying the value.
|
||||
name: String,
|
||||
},
|
||||
/// A long, random-looking path segment.
|
||||
HighEntropyPathSegment {
|
||||
/// Zero-based index of the segment after the host.
|
||||
index: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Display for FeedUrlPrivacyWarning {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::SecretQueryParameter { name } => write!(
|
||||
f,
|
||||
"the feed URL has a query parameter \"{name}\" that looks like a credential"
|
||||
),
|
||||
Self::PrivateFeedHost { host } => {
|
||||
write!(f, "{host} usually serves personal member feeds")
|
||||
}
|
||||
Self::HighEntropyQueryValue { name } => write!(
|
||||
f,
|
||||
"the value of the query parameter \"{name}\" looks like a secret token"
|
||||
),
|
||||
Self::HighEntropyPathSegment { index } => write!(
|
||||
f,
|
||||
"path segment {} of the feed URL looks like a secret token",
|
||||
index + 1
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns why `url` may contain a personal secret, or `None`.
|
||||
///
|
||||
/// Advisory only (see the module documentation). Checks, in order: query
|
||||
/// parameter names ([`SECRET_QUERY_WORDS`]), known member-feed hosts
|
||||
/// ([`PRIVATE_FEED_HOSTS`]), long high-entropy query values, and long
|
||||
/// high-entropy path segments.
|
||||
pub fn feed_url_privacy_warning(url: &FeedUrl) -> Option<FeedUrlPrivacyWarning> {
|
||||
let value = url.as_str();
|
||||
let rest = value.split_once("://").map_or(value, |(_, rest)| rest);
|
||||
let (authority, path_and_query) = match rest.find('/') {
|
||||
Some(index) => rest.split_at(index),
|
||||
None => (rest, "/"),
|
||||
};
|
||||
let host = if authority.starts_with('[') {
|
||||
authority
|
||||
.find(']')
|
||||
.map_or(authority, |end| &authority[..=end])
|
||||
} else {
|
||||
authority.split(':').next().unwrap_or(authority)
|
||||
};
|
||||
let (path, query) = path_and_query
|
||||
.split_once('?')
|
||||
.unwrap_or((path_and_query, ""));
|
||||
let parameters: Vec<(&str, &str)> = query
|
||||
.split('&')
|
||||
.filter(|pair| !pair.is_empty())
|
||||
.map(|pair| pair.split_once('=').unwrap_or((pair, "")))
|
||||
.collect();
|
||||
|
||||
if let Some((name, _)) = parameters
|
||||
.iter()
|
||||
.find(|(name, _)| is_secret_parameter_name(name))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::SecretQueryParameter {
|
||||
name: (*name).to_string(),
|
||||
});
|
||||
}
|
||||
if PRIVATE_FEED_HOSTS
|
||||
.iter()
|
||||
.any(|known| host == *known || host.ends_with(&format!(".{known}")))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::PrivateFeedHost {
|
||||
host: host.to_string(),
|
||||
});
|
||||
}
|
||||
if let Some((name, _)) = parameters
|
||||
.iter()
|
||||
.find(|(_, value)| looks_random(value, MIN_SECRET_VALUE_CHARS))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::HighEntropyQueryValue {
|
||||
name: (*name).to_string(),
|
||||
});
|
||||
}
|
||||
path.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.position(|segment| {
|
||||
let stem = segment.rsplit_once('.').map_or(segment, |(stem, _)| stem);
|
||||
looks_random(stem, MIN_SECRET_SEGMENT_CHARS)
|
||||
})
|
||||
.map(|index| FeedUrlPrivacyWarning::HighEntropyPathSegment { index })
|
||||
}
|
||||
|
||||
fn is_secret_parameter_name(name: &str) -> bool {
|
||||
let words = split_words(name);
|
||||
if words
|
||||
.iter()
|
||||
.any(|word| SECRET_QUERY_WORDS.contains(&word.as_str()))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let compact: String = name
|
||||
.chars()
|
||||
.filter(char::is_ascii_alphanumeric)
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase();
|
||||
SECRET_QUERY_SUBSTRINGS
|
||||
.iter()
|
||||
.any(|needle| compact.contains(needle))
|
||||
}
|
||||
|
||||
/// Splits at non-alphanumeric characters and lower-to-upper case changes.
|
||||
fn split_words(name: &str) -> Vec<String> {
|
||||
let mut words = Vec::new();
|
||||
let mut current = String::new();
|
||||
let mut previous_lower = false;
|
||||
for c in name.chars() {
|
||||
if !c.is_ascii_alphanumeric() {
|
||||
if !current.is_empty() {
|
||||
words.push(std::mem::take(&mut current));
|
||||
}
|
||||
previous_lower = false;
|
||||
continue;
|
||||
}
|
||||
if c.is_ascii_uppercase() && previous_lower && !current.is_empty() {
|
||||
words.push(std::mem::take(&mut current));
|
||||
}
|
||||
previous_lower = c.is_ascii_lowercase() || c.is_ascii_digit();
|
||||
current.push(c.to_ascii_lowercase());
|
||||
}
|
||||
if !current.is_empty() {
|
||||
words.push(current);
|
||||
}
|
||||
words
|
||||
}
|
||||
|
||||
/// A long token-alphabet string mixing letters and digits (or both letter
|
||||
/// cases) with high per-character entropy.
|
||||
fn looks_random(value: &str, min_chars: usize) -> bool {
|
||||
if value.chars().count() < min_chars
|
||||
|| !value
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || "-_.~%=+".contains(c))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let has_digit = value.chars().any(|c| c.is_ascii_digit());
|
||||
let has_lower = value.chars().any(|c| c.is_ascii_lowercase());
|
||||
let has_upper = value.chars().any(|c| c.is_ascii_uppercase());
|
||||
let classes = [has_digit, has_lower, has_upper]
|
||||
.into_iter()
|
||||
.filter(|present| *present)
|
||||
.count();
|
||||
classes >= 2 && shannon_entropy(value) >= MIN_SECRET_ENTROPY_BITS
|
||||
}
|
||||
|
||||
fn shannon_entropy(value: &str) -> f64 {
|
||||
let mut counts: HashMap<char, usize> = HashMap::new();
|
||||
for c in value.chars() {
|
||||
*counts.entry(c).or_default() += 1;
|
||||
}
|
||||
let total = value.chars().count() as f64;
|
||||
counts
|
||||
.values()
|
||||
.map(|count| {
|
||||
let p = *count as f64 / total;
|
||||
-p * p.log2()
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn warn(url: &str) -> Option<FeedUrlPrivacyWarning> {
|
||||
feed_url_privacy_warning(&FeedUrl::parse(url).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_feeds_have_no_warning() {
|
||||
for url in [
|
||||
"https://feeds.example.com/podcast.xml",
|
||||
"https://example.com/feed?format=rss&page=2",
|
||||
"https://anchor.example/s/abc123de/podcast/rss",
|
||||
"https://feeds.example.com/12345/rss",
|
||||
"https://example.com/feed?author=jane",
|
||||
"https://example.com:8443/shows/my-great-show/feed.xml",
|
||||
] {
|
||||
assert_eq!(warn(url), None, "{url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_query_names_warn() {
|
||||
for (url, name) in [
|
||||
("https://example.com/feed?token=abc", "token"),
|
||||
("https://example.com/feed?auth_token=abc", "auth_token"),
|
||||
(
|
||||
"https://example.com/feed?X-Amz-Signature=abc",
|
||||
"X-Amz-Signature",
|
||||
),
|
||||
("https://example.com/feed?accessKey=abc", "accessKey"),
|
||||
("https://example.com/feed?a=1&uid=42", "uid"),
|
||||
("https://example.com/feed?apitoken=x", "apitoken"),
|
||||
("https://example.com/feed?PRIVATE=1", "PRIVATE"),
|
||||
("https://example.com/feed?session_id=1", "session_id"),
|
||||
] {
|
||||
assert_eq!(
|
||||
warn(url),
|
||||
Some(FeedUrlPrivacyWarning::SecretQueryParameter { name: name.into() }),
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_member_hosts_warn() {
|
||||
assert_eq!(
|
||||
warn("https://www.patreon.com/rss/show"),
|
||||
Some(FeedUrlPrivacyWarning::PrivateFeedHost {
|
||||
host: "www.patreon.com".into()
|
||||
})
|
||||
);
|
||||
assert_eq!(warn("https://notpatreon.com/rss"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_entropy_values_and_segments_warn() {
|
||||
assert_eq!(
|
||||
warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb"),
|
||||
Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { name: "c".into() })
|
||||
);
|
||||
assert_eq!(warn("https://example.com/feed?c=aaaaaaaaaaaaaaaa1"), None);
|
||||
assert_eq!(
|
||||
warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB4sYc1HdWe9GuJa3F.rss"),
|
||||
Some(FeedUrlPrivacyWarning::HighEntropyPathSegment { index: 1 })
|
||||
);
|
||||
// Below the segment length: an ordinary short id.
|
||||
assert_eq!(warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB.rss"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warnings_render_without_the_secret() {
|
||||
let warning = warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb").unwrap();
|
||||
assert!(!warning.to_string().contains("Zx81"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
//! Podcast search hits and their merge with music search results.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fmt;
|
||||
|
||||
use federation_net::EndpointId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::PodcastId;
|
||||
use crate::record::{ItemKind, LibraryItem};
|
||||
|
||||
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// Maximum distinct query tokens used by a podcast text search.
|
||||
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
|
||||
/// Maximum publishers reported per hit.
|
||||
pub const MAX_HIT_PUBLISHERS: usize = 64;
|
||||
|
||||
/// Kind of a federated search hit.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SearchHitKind {
|
||||
/// A musical artist.
|
||||
Artist,
|
||||
/// A release.
|
||||
Release,
|
||||
/// A track.
|
||||
Track,
|
||||
/// A public podcast card.
|
||||
Podcast,
|
||||
}
|
||||
|
||||
impl SearchHitKind {
|
||||
/// Stable lowercase name.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Artist => "artist",
|
||||
Self::Release => "release",
|
||||
Self::Track => "track",
|
||||
Self::Podcast => "podcast",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SearchHitKind {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ItemKind> for SearchHitKind {
|
||||
fn from(kind: ItemKind) -> Self {
|
||||
match kind {
|
||||
ItemKind::Artist => Self::Artist,
|
||||
ItemKind::Release => Self::Release,
|
||||
ItemKind::Track => Self::Track,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One public podcast found in the discovery overlay.
|
||||
///
|
||||
/// Cards of the same podcast published by several devices are merged by
|
||||
/// `podcast_id`. The hit carries no personal state of any publisher.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastSearchHit {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Representative card: the most common `feed_url` among the
|
||||
/// publishers' current cards, then the most recently issued card, then
|
||||
/// the greatest publisher key.
|
||||
pub card: PodcastCard,
|
||||
/// Sorted distinct publisher devices with a current valid card, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
|
||||
/// one user may publish from several devices.
|
||||
pub publishers: Vec<EndpointId>,
|
||||
}
|
||||
|
||||
impl PodcastSearchHit {
|
||||
/// Always [`SearchHitKind::Podcast`].
|
||||
pub fn kind(&self) -> SearchHitKind {
|
||||
SearchHitKind::Podcast
|
||||
}
|
||||
}
|
||||
|
||||
/// A music or podcast result of a federated search.
|
||||
///
|
||||
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
|
||||
/// [`FederatedSearchHit::kind`] for the fine-grained kind.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", content = "hit", rename_all = "snake_case")]
|
||||
pub enum FederatedSearchHit {
|
||||
/// A music-DHT record (artist, release or track).
|
||||
Music(LibraryItem),
|
||||
/// A public podcast.
|
||||
Podcast(PodcastSearchHit),
|
||||
}
|
||||
|
||||
impl FederatedSearchHit {
|
||||
/// Kind of the hit: `artist`, `release`, `track` or `podcast`.
|
||||
pub fn kind(&self) -> SearchHitKind {
|
||||
match self {
|
||||
Self::Music(item) => item.kind.into(),
|
||||
Self::Podcast(_) => SearchHitKind::Podcast,
|
||||
}
|
||||
}
|
||||
|
||||
/// Display name: the item name or the podcast title.
|
||||
pub fn title(&self) -> &str {
|
||||
match self {
|
||||
Self::Music(item) => &item.name,
|
||||
Self::Podcast(hit) => &hit.card.title,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges music results (as returned by `MusicDhtService::search_network`)
|
||||
/// with podcast hits for the same query.
|
||||
///
|
||||
/// Results whose normalized name equals the normalized query come first;
|
||||
/// within each tier music keeps its order and precedes podcasts, which keep
|
||||
/// theirs. Existing music consumers that only read `SearchOutcome` are
|
||||
/// unaffected.
|
||||
pub fn merge_search_results(
|
||||
query: &str,
|
||||
music: Vec<LibraryItem>,
|
||||
podcasts: Vec<PodcastSearchHit>,
|
||||
) -> Vec<FederatedSearchHit> {
|
||||
let normalized = normalize_name(query);
|
||||
let mut exact = Vec::new();
|
||||
let mut rest = Vec::new();
|
||||
let hits = music
|
||||
.into_iter()
|
||||
.map(FederatedSearchHit::Music)
|
||||
.chain(podcasts.into_iter().map(FederatedSearchHit::Podcast));
|
||||
for hit in hits {
|
||||
let is_exact = !normalized.is_empty() && normalize_name(hit.title()) == normalized;
|
||||
if is_exact {
|
||||
exact.push(hit);
|
||||
} else {
|
||||
rest.push(hit);
|
||||
}
|
||||
}
|
||||
exact.extend(rest);
|
||||
exact
|
||||
}
|
||||
|
||||
/// Distinct normalized query tokens, at most [`MAX_PODCAST_QUERY_TOKENS`].
|
||||
pub fn podcast_query_tokens(query: &str) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
tokenize(&normalize_name(query))
|
||||
.into_iter()
|
||||
.filter(|token| token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()))
|
||||
.take(MAX_PODCAST_QUERY_TOKENS)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Keeps the winning record per `(podcast_id, publisher)` and groups the
|
||||
/// cards that are still active by podcast. Records must be verified.
|
||||
pub(crate) fn aggregate_hits(
|
||||
records: impl IntoIterator<Item = SignedPodcastRecord>,
|
||||
) -> Vec<PodcastSearchHit> {
|
||||
let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new();
|
||||
for record in records {
|
||||
let owner = (record.podcast_id().clone(), record.publisher());
|
||||
match winners.get(&owner) {
|
||||
Some(current) if compare_records(current, &record).is_ge() => {}
|
||||
_ => {
|
||||
winners.insert(owner, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut by_podcast: BTreeMap<PodcastId, Vec<PodcastCard>> = BTreeMap::new();
|
||||
for record in winners.into_values() {
|
||||
if let Some(card) = record.card() {
|
||||
by_podcast
|
||||
.entry(card.podcast_id.clone())
|
||||
.or_default()
|
||||
.push(card.clone());
|
||||
}
|
||||
}
|
||||
by_podcast
|
||||
.into_iter()
|
||||
.map(|(podcast_id, cards)| {
|
||||
let mut feed_votes: HashMap<&str, usize> = HashMap::new();
|
||||
for card in &cards {
|
||||
*feed_votes.entry(card.feed_url.as_str()).or_default() += 1;
|
||||
}
|
||||
let card = cards
|
||||
.iter()
|
||||
.max_by_key(|card| {
|
||||
(
|
||||
feed_votes[card.feed_url.as_str()],
|
||||
card.issued_at_ms,
|
||||
*card.publisher.as_bytes(),
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.expect("a group has at least one card");
|
||||
let mut publishers: Vec<EndpointId> = cards.iter().map(|card| card.publisher).collect();
|
||||
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
||||
publishers.dedup();
|
||||
publishers.truncate(MAX_HIT_PUBLISHERS);
|
||||
PodcastSearchHit {
|
||||
podcast_id,
|
||||
card,
|
||||
publishers,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Keeps hits whose card is indexed under every query token and ranks
|
||||
/// them: exact normalized title first, then more publishers, then title and
|
||||
/// id.
|
||||
pub(crate) fn rank_hits(
|
||||
mut hits: Vec<PodcastSearchHit>,
|
||||
query: &str,
|
||||
tokens: &[String],
|
||||
) -> Vec<PodcastSearchHit> {
|
||||
let normalized_query = normalize_name(query);
|
||||
hits.retain(|hit| {
|
||||
let indexed = hit.card.index_tokens();
|
||||
tokens.iter().all(|token| indexed.contains(token))
|
||||
});
|
||||
hits.sort_by_cached_key(|hit| {
|
||||
let title = normalize_name(&hit.card.title);
|
||||
(
|
||||
title != normalized_query,
|
||||
Reverse(hit.publishers.len()),
|
||||
title,
|
||||
hit.podcast_id.clone(),
|
||||
)
|
||||
});
|
||||
hits
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
//! SQLite cache of the podcast-discovery overlay: replicas, routing contacts
|
||||
//! and this device's own publications.
|
||||
//!
|
||||
//! The cache is replaceable federation state, kept in its own database file
|
||||
//! (like the similarity-routing cache). Every record read from it is decoded
|
||||
//! and verified again before it is used or served.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use federation_net::{EndpointId, NetworkId};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::error::{MusicDhtError, Result};
|
||||
use crate::podcast::PodcastId;
|
||||
use crate::routing::{NodeContact, NodeId};
|
||||
|
||||
use super::card::{
|
||||
PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere,
|
||||
};
|
||||
|
||||
/// `PRAGMA user_version` of the podcast-discovery cache schema.
|
||||
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1;
|
||||
/// Maximum distinct `(podcast_id, publisher)` records cached under one key.
|
||||
/// When full, the record expiring first is evicted if the new one outlives it.
|
||||
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256;
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_records (
|
||||
dht_key BLOB NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
payload BLOB NOT NULL,
|
||||
PRIMARY KEY (dht_key, podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
|
||||
ON podcast_dht_records(expires_at_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner
|
||||
ON podcast_dht_records(podcast_id, publisher);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
|
||||
peer_id TEXT PRIMARY KEY,
|
||||
node_id BLOB NOT NULL,
|
||||
ticket TEXT NOT NULL,
|
||||
last_seen_ms INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_local (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
revision INTEGER NOT NULL,
|
||||
spec BLOB,
|
||||
record BLOB NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
";
|
||||
|
||||
/// This device's latest publication of one podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct LocalPublication {
|
||||
pub podcast_id: PodcastId,
|
||||
/// Greatest revision ever used for this podcast by this device.
|
||||
pub revision: u64,
|
||||
/// Published card content; `None` once unpublished.
|
||||
pub spec: Option<PodcastCardSpec>,
|
||||
/// Latest signed record (card or tombstone).
|
||||
pub record: SignedPodcastRecord,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct PodcastDhtDatabase {
|
||||
conn: Arc<Mutex<Connection>>,
|
||||
network_id: NetworkId,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PodcastDhtDatabase {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("PodcastDhtDatabase")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl PodcastDhtDatabase {
|
||||
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
|
||||
let path = path.to_path_buf();
|
||||
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
|
||||
let conn = Connection::open(&path).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to open {}: {error}", path.display()))
|
||||
})?;
|
||||
let version: i32 = conn
|
||||
.query_row("PRAGMA user_version", [], |row| row.get(0))
|
||||
.map_err(database_error)?;
|
||||
if version > PODCAST_DHT_CACHE_SCHEMA_VERSION {
|
||||
return Err(MusicDhtError::Database(format!(
|
||||
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
|
||||
)));
|
||||
}
|
||||
conn.execute_batch(SCHEMA).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
|
||||
})?;
|
||||
conn.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION)
|
||||
.map_err(database_error)?;
|
||||
Ok(conn)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))??;
|
||||
Ok(Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
network_id,
|
||||
})
|
||||
}
|
||||
|
||||
async fn call<F, T>(&self, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let conn = Arc::clone(&self.conn);
|
||||
let network_id = self.network_id;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = lock(&conn);
|
||||
operation(&guard, &network_id).map_err(database_error)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
|
||||
}
|
||||
|
||||
/// Stores records under the given keys. The caller has verified every
|
||||
/// record and checked that each key belongs to it. Returns the number of
|
||||
/// `(key, record)` rows written.
|
||||
pub async fn store(
|
||||
&self,
|
||||
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
|
||||
now_ms: u64,
|
||||
) -> Result<usize> {
|
||||
self.call(move |conn, network_id| {
|
||||
let transaction = conn.unchecked_transaction()?;
|
||||
let mut written = 0;
|
||||
for (keys, record) in entries {
|
||||
written += store_one(&transaction, network_id, &keys, &record, now_ms)?;
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(written)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Valid, unexpired records under `key`, newest expiry first. Cards must
|
||||
/// contain every token in `filter_tokens`; tombstones are always
|
||||
/// returned so stale replicas elsewhere can be overruled.
|
||||
pub async fn records_by_key(
|
||||
&self,
|
||||
key: PodcastDhtKey,
|
||||
filter_tokens: Vec<String>,
|
||||
now_ms: u64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<SignedPodcastRecord>> {
|
||||
self.call(move |conn, network_id| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT payload FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND expires_at_ms > ?2
|
||||
ORDER BY expires_at_ms DESC, podcast_id, publisher",
|
||||
)?;
|
||||
let rows = statement.query_map(
|
||||
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|
||||
|row| row.get::<_, Vec<u8>>(0),
|
||||
)?;
|
||||
let mut records = Vec::new();
|
||||
for row in rows {
|
||||
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
||||
continue;
|
||||
};
|
||||
if !record.belongs_under(&key) {
|
||||
continue;
|
||||
}
|
||||
let matches = record.card().is_none_or(|card| {
|
||||
let tokens = card.index_tokens();
|
||||
filter_tokens.iter().all(|token| tokens.contains(token))
|
||||
});
|
||||
if matches {
|
||||
records.push(record);
|
||||
if records.len() == limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(records)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
|
||||
self.call(move |conn, _| {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
|
||||
[clamp_i64(now_ms)],
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
|
||||
self.call(move |conn, _| {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_peers
|
||||
(peer_id, node_id, ticket, last_seen_ms)
|
||||
VALUES (?1, ?2, ?3, ?4)",
|
||||
params![
|
||||
contact.peer_id.to_string(),
|
||||
contact.node_id.as_bytes().as_slice(),
|
||||
contact.ticket,
|
||||
clamp_i64(contact.last_seen_ms),
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
|
||||
self.call(move |conn, _| {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
|
||||
[peer.to_string()],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
|
||||
self.call(move |conn, _| {
|
||||
let mut statement =
|
||||
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
|
||||
let rows = statement.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut peers = Vec::new();
|
||||
for row in rows {
|
||||
let (peer, ticket, last_seen_ms) = row?;
|
||||
let Ok(peer_id) = EndpointId::from_str(&peer) else {
|
||||
continue;
|
||||
};
|
||||
peers.push(NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer_id),
|
||||
peer_id,
|
||||
ticket,
|
||||
last_seen_ms: last_seen_ms.max(0) as u64,
|
||||
});
|
||||
}
|
||||
Ok(peers)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn local_publication(
|
||||
&self,
|
||||
podcast_id: PodcastId,
|
||||
) -> Result<Option<LocalPublication>> {
|
||||
self.call(move |conn, _| {
|
||||
conn.query_row(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
local_from_row,
|
||||
)
|
||||
.optional()
|
||||
.map(Option::flatten)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
|
||||
self.call(move |conn, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
ORDER BY podcast_id",
|
||||
)?;
|
||||
let rows = statement.query_map([], local_from_row)?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
if let Some(publication) = row? {
|
||||
out.push(publication);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_local_publication(
|
||||
&self,
|
||||
publication: LocalPublication,
|
||||
now_ms: u64,
|
||||
) -> Result<()> {
|
||||
self.call(move |conn, _| {
|
||||
let spec = publication
|
||||
.spec
|
||||
.as_ref()
|
||||
.map(postcard::to_stdvec)
|
||||
.transpose()
|
||||
.map_err(to_sql_error)?;
|
||||
let record = postcard::to_stdvec(&publication.record).map_err(to_sql_error)?;
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_local
|
||||
(podcast_id, revision, spec, record, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![
|
||||
publication.podcast_id.as_str(),
|
||||
publication.revision as i64,
|
||||
spec,
|
||||
record,
|
||||
clamp_i64(now_ms),
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn store_one(
|
||||
conn: &Connection,
|
||||
network_id: &NetworkId,
|
||||
keys: &[PodcastDhtKey],
|
||||
record: &SignedPodcastRecord,
|
||||
now_ms: u64,
|
||||
) -> rusqlite::Result<usize> {
|
||||
let podcast_id = record.podcast_id().as_str().to_string();
|
||||
let publisher = record.publisher().to_string();
|
||||
|
||||
// Every cached version of this (podcast, publisher), re-verified.
|
||||
let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new();
|
||||
let mut invalid_keys = Vec::new();
|
||||
{
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT dht_key, payload FROM podcast_dht_records
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
||||
Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?))
|
||||
})?;
|
||||
for row in rows {
|
||||
let (key, payload) = row?;
|
||||
let decoded = decode_valid(&payload, network_id, now_ms);
|
||||
match (<[u8; 32]>::try_from(key.as_slice()), decoded) {
|
||||
(Ok(key_bytes), Some(cached)) => {
|
||||
existing.insert(key_bytes, cached);
|
||||
}
|
||||
_ => invalid_keys.push(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
for key in invalid_keys {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key, podcast_id, publisher],
|
||||
)?;
|
||||
}
|
||||
|
||||
// A tombstone or a newer revision cached under any key wins everywhere.
|
||||
if existing
|
||||
.values()
|
||||
.any(|cached| supersedes_everywhere(cached, record))
|
||||
{
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
||||
let mut written = 0;
|
||||
for key in keys {
|
||||
if let Some(cached) = existing.get(key.as_bytes())
|
||||
&& compare_records(cached, record).is_ge()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? {
|
||||
continue;
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_records
|
||||
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET
|
||||
tombstone = excluded.tombstone,
|
||||
expires_at_ms = excluded.expires_at_ms,
|
||||
payload = excluded.payload",
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
payload,
|
||||
],
|
||||
)?;
|
||||
written += 1;
|
||||
}
|
||||
|
||||
// Drop versions the new record makes obsolete under other keys too.
|
||||
if written > 0 {
|
||||
for (key, cached) in &existing {
|
||||
if !keys.iter().any(|stored| stored.as_bytes() == key)
|
||||
&& supersedes_everywhere(record, cached)
|
||||
{
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Ensures there is room for one more record under `key`.
|
||||
fn make_room(
|
||||
conn: &Connection,
|
||||
key: &PodcastDhtKey,
|
||||
record: &SignedPodcastRecord,
|
||||
now_ms: u64,
|
||||
) -> rusqlite::Result<bool> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2",
|
||||
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|
||||
)?;
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1",
|
||||
[key.as_bytes().as_slice()],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY {
|
||||
return Ok(true);
|
||||
}
|
||||
let victim: Option<(String, String, i64)> = conn
|
||||
.query_row(
|
||||
"SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records
|
||||
WHERE dht_key = ?1
|
||||
ORDER BY expires_at_ms ASC, podcast_id, publisher
|
||||
LIMIT 1",
|
||||
[key.as_bytes().as_slice()],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
||||
)
|
||||
.optional()?;
|
||||
match victim {
|
||||
Some((podcast_id, publisher, expires_at_ms))
|
||||
if expires_at_ms < clamp_i64(record.expires_at_ms()) =>
|
||||
{
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_bytes().as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_valid(
|
||||
payload: &[u8],
|
||||
network_id: &NetworkId,
|
||||
now_ms: u64,
|
||||
) -> Option<SignedPodcastRecord> {
|
||||
let record: SignedPodcastRecord = postcard::from_bytes(payload).ok()?;
|
||||
record.verify_at(network_id, now_ms).ok()?;
|
||||
Some(record)
|
||||
}
|
||||
|
||||
fn local_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Option<LocalPublication>> {
|
||||
let podcast_id: String = row.get(0)?;
|
||||
let revision: i64 = row.get(1)?;
|
||||
let spec: Option<Vec<u8>> = row.get(2)?;
|
||||
let record: Vec<u8> = row.get(3)?;
|
||||
let Ok(podcast_id) = PodcastId::parse(&podcast_id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let spec = match spec {
|
||||
Some(bytes) => match postcard::from_bytes::<PodcastCardSpec>(&bytes) {
|
||||
Ok(spec) => Some(spec),
|
||||
Err(_) => return Ok(None),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let Ok(record) = postcard::from_bytes::<SignedPodcastRecord>(&record) else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(LocalPublication {
|
||||
podcast_id,
|
||||
revision: revision as u64,
|
||||
spec,
|
||||
record,
|
||||
}))
|
||||
}
|
||||
|
||||
fn clamp_i64(value: u64) -> i64 {
|
||||
value.min(i64::MAX as u64) as i64
|
||||
}
|
||||
|
||||
fn to_sql_error(error: impl std::error::Error + Send + Sync + 'static) -> rusqlite::Error {
|
||||
rusqlite::Error::ToSqlConversionFailure(Box::new(error))
|
||||
}
|
||||
|
||||
fn database_error(error: rusqlite::Error) -> MusicDhtError {
|
||||
MusicDhtError::Database(error.to_string())
|
||||
}
|
||||
|
||||
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
|
||||
mutex.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
@@ -0,0 +1,845 @@
|
||||
//! Unit tests of the podcast-discovery overlay: records, ordering, cache and
|
||||
//! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`.
|
||||
|
||||
use federation_net::{NetworkId, SecretKey};
|
||||
|
||||
use super::card::supersedes_everywhere;
|
||||
use super::search::{aggregate_hits, rank_hits};
|
||||
use super::*;
|
||||
use crate::podcast::{FeedUrl, PodcastMeta};
|
||||
use crate::record::{ItemId, ItemKind, LibraryItem};
|
||||
|
||||
const NOW: u64 = 1_750_000_000_000;
|
||||
|
||||
fn network() -> NetworkId {
|
||||
NetworkId::from_name("podcast-dht-unit")
|
||||
}
|
||||
|
||||
fn secret(seed: u8) -> SecretKey {
|
||||
SecretKey::from_bytes(&[seed; 32])
|
||||
}
|
||||
|
||||
fn feed(path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&format!("https://feeds.example.test/{path}")).unwrap()
|
||||
}
|
||||
|
||||
fn spec(path: &str, title: &str) -> PodcastCardSpec {
|
||||
let feed_url = feed(path);
|
||||
PodcastCardSpec {
|
||||
podcast_id: PodcastId::from_feed_url(&feed_url),
|
||||
feed_url,
|
||||
id_feed_url: None,
|
||||
title: title.to_string(),
|
||||
author: Some("Jane Doe".to_string()),
|
||||
description: Some("A show about\nthings.".to_string()),
|
||||
image_url: Some("https://cdn.example.test/art.jpg".to_string()),
|
||||
link: Some("https://example.test/show".to_string()),
|
||||
language: Some("en".to_string()),
|
||||
categories: vec!["Technology".to_string()],
|
||||
episode_count: Some(12),
|
||||
latest_episode_at_ms: Some(1_700_000_000_000),
|
||||
}
|
||||
}
|
||||
|
||||
fn card_with(
|
||||
spec: PodcastCardSpec,
|
||||
key: &SecretKey,
|
||||
revision: u64,
|
||||
issued: u64,
|
||||
) -> SignedPodcastRecord {
|
||||
let card = PodcastCard::from_spec(spec, network(), key.public(), revision, issued);
|
||||
SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes)).unwrap()
|
||||
}
|
||||
|
||||
fn card(key: &SecretKey, revision: u64) -> SignedPodcastRecord {
|
||||
card_with(spec("show.xml", "Rust Radio"), key, revision, NOW)
|
||||
}
|
||||
|
||||
fn tombstone(key: &SecretKey, revision: u64, issued: u64) -> SignedPodcastRecord {
|
||||
let retracted = card(key, 1);
|
||||
let tombstone = PodcastTombstone::new(
|
||||
network(),
|
||||
key.public(),
|
||||
retracted.podcast_id().clone(),
|
||||
revision,
|
||||
issued,
|
||||
retracted.dht_keys(),
|
||||
);
|
||||
SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap()
|
||||
}
|
||||
|
||||
fn resign(record: &mut SignedPodcastRecord, key: &SecretKey) {
|
||||
record.signature = key.sign(&record.signing_bytes());
|
||||
}
|
||||
|
||||
fn subscription(spec: &PodcastCardSpec) -> SubscriptionRecord {
|
||||
SubscriptionRecord {
|
||||
podcast_id: spec.podcast_id.clone(),
|
||||
subscribed: true,
|
||||
podcast: PodcastMeta {
|
||||
feed_url: spec.feed_url.clone(),
|
||||
title: spec.title.clone(),
|
||||
author: spec.author.clone(),
|
||||
description: spec.description.clone(),
|
||||
link: spec.link.clone(),
|
||||
image_url: spec.image_url.clone(),
|
||||
},
|
||||
feed_aliases: Vec::new(),
|
||||
generation: 1,
|
||||
hlc_ms: 100,
|
||||
op_id: "private-op-id-123".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- signatures
|
||||
|
||||
#[test]
|
||||
fn signed_card_verifies_and_rejects_tampering() {
|
||||
let key = secret(1);
|
||||
let record = card(&key, 1);
|
||||
record.verify_at(&network(), NOW).unwrap();
|
||||
|
||||
let mut tampered = record.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
||||
card.title = "Evil Radio".to_string();
|
||||
}
|
||||
assert!(tampered.verify(&network()).is_err());
|
||||
|
||||
let mut moved = record.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut moved.body {
|
||||
card.revision += 1;
|
||||
}
|
||||
assert!(moved.verify(&network()).is_err());
|
||||
|
||||
assert!(record.verify(&NetworkId::from_name("other")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_signed_by_another_key_is_rejected() {
|
||||
let publisher = secret(1);
|
||||
let forger = secret(2);
|
||||
let card = PodcastCard::from_spec(
|
||||
spec("show.xml", "Rust Radio"),
|
||||
network(),
|
||||
publisher.public(),
|
||||
1,
|
||||
NOW,
|
||||
);
|
||||
let forged = SignedPodcastRecord::sign_card(card, |bytes| forger.sign(bytes)).unwrap();
|
||||
assert!(forged.verify(&network()).is_err());
|
||||
|
||||
let mut forged_tombstone = tombstone(&forger, 2, NOW);
|
||||
if let PodcastRecordBody::Tombstone(tombstone) = &mut forged_tombstone.body {
|
||||
tombstone.publisher = publisher.public();
|
||||
}
|
||||
resign(&mut forged_tombstone, &forger);
|
||||
assert!(forged_tombstone.verify(&network()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_encoding_is_stable() {
|
||||
let key = secret(9);
|
||||
let record = card_with(spec("vector.xml", "Vector Show"), &key, 7, NOW);
|
||||
let bytes = record.signing_bytes();
|
||||
assert!(bytes.starts_with(b"furumi-podcast-card:v1\0\x01\x00"));
|
||||
assert_eq!(
|
||||
blake3::hash(&bytes).to_hex().as_str(),
|
||||
"cbbc512b077ad05baf01728fb5d179a1d89301f19ea15c66058df284a716e9b5",
|
||||
"card signing input changed: this breaks every published card"
|
||||
);
|
||||
let tombstone = PodcastTombstone::new(
|
||||
network(),
|
||||
key.public(),
|
||||
record.podcast_id().clone(),
|
||||
8,
|
||||
NOW,
|
||||
record.dht_keys(),
|
||||
);
|
||||
let bytes = tombstone.signing_bytes();
|
||||
assert!(bytes.starts_with(b"furumi-podcast-card-tombstone:v1\0\x01\x00"));
|
||||
assert_eq!(
|
||||
blake3::hash(&bytes).to_hex().as_str(),
|
||||
"032ccef911bff5f2f37367137d428d60c111efb65529c8d75754434434dbd083",
|
||||
"tombstone signing input changed"
|
||||
);
|
||||
assert_eq!(
|
||||
hex(PodcastDhtKey::token(&network(), "vector").as_bytes()),
|
||||
"4367d03f8ea2a541ebec43b73d767b42ad733e3f26f90372cff9f92cf19ccbf1"
|
||||
);
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_fields_are_unambiguous_in_the_signing_input() {
|
||||
let key = secret(1);
|
||||
let mut a = spec("show.xml", "Rust Radio");
|
||||
a.author = Some(String::new());
|
||||
let mut b = spec("show.xml", "Rust Radio");
|
||||
b.author = None;
|
||||
let a = PodcastCard::from_spec(a, network(), key.public(), 1, NOW);
|
||||
let b = PodcastCard::from_spec(b, network(), key.public(), 1, NOW);
|
||||
assert_ne!(a.signing_bytes(), b.signing_bytes());
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- bounds
|
||||
|
||||
#[test]
|
||||
fn card_text_bounds_accept_the_limit_and_reject_one_more() {
|
||||
let key = secret(1);
|
||||
let sign = |spec: PodcastCardSpec| {
|
||||
let card = PodcastCard::from_spec(spec, network(), key.public(), 1, NOW);
|
||||
SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes))
|
||||
};
|
||||
let mut at = spec("show.xml", &"я".repeat(MAX_CARD_TITLE_CHARS));
|
||||
at.author = Some("a".repeat(MAX_CARD_AUTHOR_CHARS));
|
||||
at.description = Some("d".repeat(MAX_CARD_DESCRIPTION_CHARS));
|
||||
at.categories = (0..MAX_CARD_CATEGORIES)
|
||||
.map(|index| format!("{index}{}", "c".repeat(MAX_CARD_CATEGORY_CHARS - 1)))
|
||||
.collect();
|
||||
at.language = Some("x".repeat(MAX_CARD_LANGUAGE_BYTES));
|
||||
let record = sign(at.clone()).unwrap();
|
||||
assert!(record.encoded_len() <= MAX_PODCAST_RECORD_BYTES);
|
||||
|
||||
let mut over = at.clone();
|
||||
over.title.push('я');
|
||||
assert!(sign(over).is_err());
|
||||
let mut over = at.clone();
|
||||
over.author.as_mut().unwrap().push('a');
|
||||
assert!(sign(over).is_err());
|
||||
let mut over = at.clone();
|
||||
over.description.as_mut().unwrap().push('d');
|
||||
assert!(sign(over).is_err());
|
||||
let mut over = at.clone();
|
||||
over.categories.push("extra".into());
|
||||
assert!(sign(over).is_err());
|
||||
let mut over = at.clone();
|
||||
over.categories[0].push('c');
|
||||
assert!(sign(over).is_err());
|
||||
let mut over = at.clone();
|
||||
over.language.as_mut().unwrap().push('x');
|
||||
assert!(sign(over).is_err());
|
||||
let mut over = at;
|
||||
over.title = " ".into();
|
||||
assert!(sign(over).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn card_rejects_bad_urls_ids_and_control_characters() {
|
||||
let key = secret(1);
|
||||
let sign = |spec: PodcastCardSpec| {
|
||||
let card = PodcastCard::from_spec(spec, network(), key.public(), 1, NOW);
|
||||
SignedPodcastRecord::sign_card(card, |bytes| key.sign(bytes))
|
||||
};
|
||||
let mut bad = spec("show.xml", "Rust Radio");
|
||||
bad.image_url = Some("file:///etc/passwd".into());
|
||||
assert!(sign(bad).is_err());
|
||||
let mut bad = spec("show.xml", "Rust Radio");
|
||||
bad.podcast_id = PodcastId::from_feed_url(&feed("other.xml"));
|
||||
assert!(sign(bad).is_err(), "id must hash from a URL in the card");
|
||||
let mut bad = spec("show.xml", "Rust Radio");
|
||||
bad.title = "Rust\u{7}Radio".into();
|
||||
assert!(sign(bad).is_err());
|
||||
let mut bad = spec("show.xml", "Rust Radio");
|
||||
bad.id_feed_url = Some(bad.feed_url.clone());
|
||||
assert!(sign(bad).is_err());
|
||||
|
||||
let mut moved = spec("new.xml", "Rust Radio");
|
||||
moved.id_feed_url = Some(feed("show.xml"));
|
||||
moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml"));
|
||||
sign(moved).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_spec_fits_the_card() {
|
||||
let mut raw = spec("show.xml", &format!(" {}\n", "t".repeat(2_000)));
|
||||
raw.author = Some("\u{1}".into());
|
||||
raw.description = Some(format!("{}\u{0}", "d".repeat(5_000)));
|
||||
raw.image_url = Some("not a url".into());
|
||||
raw.language = Some("en US".into());
|
||||
raw.categories = (0..20).map(|index| format!("Cat {}", index % 10)).collect();
|
||||
raw.categories.push("cat 1".into());
|
||||
raw.latest_episode_at_ms = Some(-5);
|
||||
let bounded = raw.bounded();
|
||||
bounded.validate().unwrap();
|
||||
assert_eq!(bounded.title.chars().count(), MAX_CARD_TITLE_CHARS);
|
||||
assert_eq!(bounded.author, None);
|
||||
assert_eq!(bounded.image_url, None);
|
||||
assert_eq!(bounded.language, None);
|
||||
assert_eq!(bounded.categories.len(), MAX_CARD_CATEGORIES);
|
||||
assert_eq!(bounded.latest_episode_at_ms, None);
|
||||
assert_eq!(bounded.bounded(), bounded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_tokens_are_distinct_bounded_and_skip_long_tokens() {
|
||||
let long = "x".repeat(MAX_INDEX_TOKEN_BYTES + 1);
|
||||
let title = format!("The Rust the {long} radio");
|
||||
assert_eq!(
|
||||
card_index_tokens(&title, Some("Rust Jane")),
|
||||
vec!["the", "rust", "radio", "jane"]
|
||||
);
|
||||
let many: Vec<String> = (0..50).map(|index| format!("w{index}")).collect();
|
||||
let tokens = card_index_tokens(&many.join(" "), Some("author"));
|
||||
assert_eq!(tokens.len(), MAX_CARD_INDEX_TOKENS);
|
||||
assert!(!tokens.contains(&"author".to_string()));
|
||||
let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW);
|
||||
assert_eq!(record.dht_keys().len(), MAX_RECORD_DHT_KEYS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_tombstone_key_lists_are_rejected() {
|
||||
let key = secret(1);
|
||||
let id = PodcastId::from_feed_url(&feed("show.xml"));
|
||||
let keys: Vec<_> = (0..=MAX_CARD_INDEX_TOKENS)
|
||||
.map(|index| PodcastDhtKey::token(&network(), &format!("t{index}")))
|
||||
.collect();
|
||||
let mut tombstone = PodcastTombstone::new(network(), key.public(), id.clone(), 2, NOW, keys);
|
||||
assert_eq!(tombstone.index_keys.len(), MAX_CARD_INDEX_TOKENS);
|
||||
SignedPodcastRecord::sign_tombstone(tombstone.clone(), |bytes| key.sign(bytes)).unwrap();
|
||||
tombstone
|
||||
.index_keys
|
||||
.push(PodcastDhtKey::from_bytes([0xff; 32]));
|
||||
assert!(
|
||||
SignedPodcastRecord::sign_tombstone(tombstone.clone(), |bytes| key.sign(bytes)).is_err()
|
||||
);
|
||||
tombstone.index_keys.pop();
|
||||
tombstone.index_keys.reverse();
|
||||
assert!(
|
||||
SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).is_err(),
|
||||
"unsorted keys are not canonical"
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ lifetime
|
||||
|
||||
#[test]
|
||||
fn expiry_future_issue_and_overlong_lifetimes_are_rejected() {
|
||||
let key = secret(1);
|
||||
let record = card(&key, 1);
|
||||
let ttl = PODCAST_CARD_TTL.as_millis() as u64;
|
||||
record.verify_at(&network(), NOW + ttl - 1).unwrap();
|
||||
assert!(record.verify_at(&network(), NOW + ttl).is_err());
|
||||
|
||||
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
|
||||
assert!(record.verify_at(&network(), NOW - skew).is_ok());
|
||||
assert!(record.verify_at(&network(), NOW - skew - 1).is_err());
|
||||
|
||||
let mut overlong = record.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut overlong.body {
|
||||
card.expires_at_ms += 1;
|
||||
}
|
||||
resign(&mut overlong, &key);
|
||||
assert!(overlong.verify(&network()).is_err());
|
||||
|
||||
let grave = tombstone(&key, 2, NOW);
|
||||
let tomb_ttl = PODCAST_TOMBSTONE_TTL.as_millis() as u64;
|
||||
assert!(tomb_ttl > ttl);
|
||||
grave.verify_at(&network(), NOW + tomb_ttl - 1).unwrap();
|
||||
assert!(grave.verify_at(&network(), NOW + tomb_ttl).is_err());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ ordering
|
||||
|
||||
#[test]
|
||||
fn record_order_is_total_and_tombstones_win_ties() {
|
||||
let key = secret(1);
|
||||
let older = card(&key, 1);
|
||||
let newer = card(&key, 2);
|
||||
let refresh = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + 1);
|
||||
let edited = card_with(spec("show.xml", "Rust Radio!"), &key, 1, NOW);
|
||||
let grave = tombstone(&key, 1, NOW);
|
||||
|
||||
assert!(compare_records(&newer, &older).is_gt());
|
||||
assert!(compare_records(&refresh, &older).is_gt());
|
||||
assert!(compare_records(&grave, &newer).is_lt());
|
||||
assert!(compare_records(&grave, &older).is_gt());
|
||||
assert!(compare_records(&grave, &refresh).is_gt());
|
||||
assert_ne!(compare_records(&edited, &older), std::cmp::Ordering::Equal);
|
||||
assert_eq!(
|
||||
compare_records(&edited, &older),
|
||||
compare_records(&older, &edited).reverse()
|
||||
);
|
||||
assert!(supersedes_everywhere(&grave, &older));
|
||||
assert!(supersedes_everywhere(&newer, &older));
|
||||
assert!(!supersedes_everywhere(&refresh, &older));
|
||||
assert!(!supersedes_everywhere(&older, &grave));
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------- cache
|
||||
|
||||
async fn open_db(temp: &tempfile::TempDir) -> store::PodcastDhtDatabase {
|
||||
store::PodcastDhtDatabase::open(&temp.path().join("podcast-dht.sqlite3"), network())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn entry(record: &SignedPodcastRecord) -> (Vec<PodcastDhtKey>, SignedPodcastRecord) {
|
||||
(record.dht_keys(), record.clone())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_replay_of_an_older_revision_is_ignored() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let key = secret(1);
|
||||
let newer = card(&key, 2);
|
||||
let older = card(&key, 1);
|
||||
let id_key = PodcastDhtKey::podcast(&network(), newer.podcast_id());
|
||||
assert!(db.store(vec![entry(&newer)], NOW).await.unwrap() > 0);
|
||||
assert_eq!(db.store(vec![entry(&older)], NOW).await.unwrap(), 0);
|
||||
assert_eq!(
|
||||
db.records_by_key(id_key, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![newer.clone()]
|
||||
);
|
||||
// Storing the same record again changes nothing.
|
||||
assert_eq!(db.store(vec![entry(&newer)], NOW).await.unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_converges_regardless_of_delivery_order() {
|
||||
let key = secret(1);
|
||||
let records = [
|
||||
card(&key, 1),
|
||||
card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + 5),
|
||||
card_with(spec("show.xml", "Rust Radio Weekly"), &key, 2, NOW),
|
||||
tombstone(&key, 2, NOW + 1),
|
||||
card_with(spec("show.xml", "Rust Radio"), &key, 3, NOW + 2),
|
||||
];
|
||||
let id_key = PodcastDhtKey::podcast(&network(), records[0].podcast_id());
|
||||
let mut results = Vec::new();
|
||||
for order in [
|
||||
[0, 1, 2, 3, 4],
|
||||
[4, 3, 2, 1, 0],
|
||||
[3, 0, 4, 2, 1],
|
||||
[2, 4, 0, 3, 1],
|
||||
] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
for index in order {
|
||||
db.store(vec![entry(&records[index])], NOW).await.unwrap();
|
||||
}
|
||||
results.push(
|
||||
db.records_by_key(id_key, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
results
|
||||
.iter()
|
||||
.all(|result| *result == vec![records[4].clone()])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let key = secret(1);
|
||||
let active = card(&key, 1);
|
||||
db.store(vec![entry(&active)], NOW).await.unwrap();
|
||||
let token_key = PodcastDhtKey::token(&network(), "rust");
|
||||
assert_eq!(
|
||||
db.records_by_key(token_key, vec!["rust".into()], NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![active.clone()]
|
||||
);
|
||||
|
||||
// A tombstone stored only under the id key still removes the card under
|
||||
// the text keys of this node.
|
||||
let grave = tombstone(&key, 2, NOW);
|
||||
let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id());
|
||||
db.store(vec![(vec![id_key], grave.clone())], NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.records_by_key(token_key, vec!["rust".into()], NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
Vec::<SignedPodcastRecord>::new()
|
||||
);
|
||||
// An equal-revision card arriving later loses, under any key.
|
||||
let same = card(&key, 2);
|
||||
assert_eq!(db.store(vec![entry(&same)], NOW).await.unwrap(), 0);
|
||||
assert_eq!(
|
||||
db.records_by_key(id_key, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![grave.clone()]
|
||||
);
|
||||
// The tombstone is retained (and served) until it expires.
|
||||
let card_ttl = PODCAST_CARD_TTL.as_millis() as u64;
|
||||
assert_eq!(
|
||||
db.records_by_key(id_key, Vec::new(), NOW + card_ttl + 1, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![grave.clone()]
|
||||
);
|
||||
// Republishing with a higher revision wins over the tombstone.
|
||||
let again = card(&key, 3);
|
||||
assert!(db.store(vec![entry(&again)], NOW).await.unwrap() > 0);
|
||||
assert_eq!(
|
||||
db.records_by_key(id_key, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![again]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_drops_stale_text_keys_of_older_revisions() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let key = secret(1);
|
||||
let old = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW);
|
||||
let renamed = card_with(spec("show.xml", "Ferris Hour"), &key, 2, NOW);
|
||||
db.store(vec![entry(&old), entry(&renamed)], NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
let rust = PodcastDhtKey::token(&network(), "rust");
|
||||
let ferris = PodcastDhtKey::token(&network(), "ferris");
|
||||
assert!(
|
||||
db.records_by_key(rust, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
db.records_by_key(ferris, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![renamed]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_read_path_filters_tokens_expiry_and_corruption() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let key = secret(1);
|
||||
let record = card(&key, 1);
|
||||
db.store(vec![entry(&record)], NOW).await.unwrap();
|
||||
let rust = PodcastDhtKey::token(&network(), "rust");
|
||||
assert_eq!(
|
||||
db.records_by_key(rust, vec!["rust".into(), "radio".into()], NOW, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
db.records_by_key(rust, vec!["rust".into(), "jazz".into()], NOW, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
let ttl = PODCAST_CARD_TTL.as_millis() as u64;
|
||||
assert!(
|
||||
db.records_by_key(rust, Vec::new(), NOW + ttl, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// A row whose payload no longer verifies is never served.
|
||||
let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap();
|
||||
let mut tampered = record.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
||||
card.title = "Rust Radio Evil".into();
|
||||
}
|
||||
conn.execute(
|
||||
"UPDATE podcast_dht_records SET payload = ?1",
|
||||
[postcard::to_stdvec(&tampered).unwrap()],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
db.records_by_key(rust, Vec::new(), NOW, 10)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
db.delete_expired(NOW + ttl).await.unwrap(),
|
||||
MAX_RECORD_DHT_KEYS.min(record.dht_keys().len())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_bounds_records_per_key() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let token_key = PodcastDhtKey::token(&network(), "rust");
|
||||
let mut entries = Vec::new();
|
||||
for index in 0..=MAX_PODCAST_RECORDS_PER_KEY {
|
||||
let key = SecretKey::from_bytes(&{
|
||||
let mut bytes = [0u8; 32];
|
||||
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
|
||||
bytes
|
||||
});
|
||||
// Later publishers expire later, so the first one is evicted.
|
||||
let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64);
|
||||
entries.push((vec![token_key], record));
|
||||
}
|
||||
let first = entries[0].1.clone();
|
||||
for chunk in entries.chunks(64) {
|
||||
db.store(chunk.to_vec(), NOW).await.unwrap();
|
||||
}
|
||||
let cached = db
|
||||
.records_by_key(token_key, Vec::new(), NOW, usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
|
||||
assert!(!cached.contains(&first));
|
||||
// A record expiring before every cached one is not admitted.
|
||||
assert_eq!(
|
||||
db.store(vec![(vec![token_key], first)], NOW).await.unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_refuses_a_newer_schema_and_keeps_local_publications() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("podcast-dht.sqlite3");
|
||||
let key = secret(1);
|
||||
let record = card(&key, 5);
|
||||
{
|
||||
let db = store::PodcastDhtDatabase::open(&path, network())
|
||||
.await
|
||||
.unwrap();
|
||||
db.put_local_publication(
|
||||
store::LocalPublication {
|
||||
podcast_id: record.podcast_id().clone(),
|
||||
revision: 5,
|
||||
spec: Some(record.card().unwrap().spec()),
|
||||
record: record.clone(),
|
||||
},
|
||||
NOW,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let db = store::PodcastDhtDatabase::open(&path, network())
|
||||
.await
|
||||
.unwrap();
|
||||
let local = db.local_publications().await.unwrap();
|
||||
assert_eq!(local.len(), 1);
|
||||
assert_eq!(local[0].revision, 5);
|
||||
assert_eq!(local[0].record, record);
|
||||
drop(db);
|
||||
|
||||
rusqlite::Connection::open(&path)
|
||||
.unwrap()
|
||||
.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1)
|
||||
.unwrap();
|
||||
assert!(
|
||||
store::PodcastDhtDatabase::open(&path, network())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ batches
|
||||
|
||||
#[test]
|
||||
fn store_batches_are_bounded_and_keys_must_belong() {
|
||||
let key = secret(1);
|
||||
let record = card(&key, 1);
|
||||
let good = PodcastStoreEntry {
|
||||
keys: record.dht_keys(),
|
||||
record: record.clone(),
|
||||
};
|
||||
validate_store_batch(vec![good.clone()], &network(), NOW).unwrap();
|
||||
assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err());
|
||||
assert!(
|
||||
validate_store_batch(
|
||||
vec![good.clone(); MAX_PODCAST_STORE_RECORDS],
|
||||
&network(),
|
||||
NOW
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
validate_store_batch(
|
||||
vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1],
|
||||
&network(),
|
||||
NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let mut foreign = good.clone();
|
||||
foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")];
|
||||
assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err());
|
||||
let mut duplicated = good.clone();
|
||||
duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]];
|
||||
assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err());
|
||||
let mut forged = good;
|
||||
resign(&mut forged.record, &secret(2));
|
||||
assert!(validate_store_batch(vec![forged], &network(), NOW).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_tokens_are_bounded() {
|
||||
let at: Vec<String> = (0..MAX_PODCAST_QUERY_TOKENS)
|
||||
.map(|i| format!("t{i}"))
|
||||
.collect();
|
||||
validate_filter_tokens(&at).unwrap();
|
||||
let mut over = at.clone();
|
||||
over.push("x".into());
|
||||
assert!(validate_filter_tokens(&over).is_err());
|
||||
assert!(validate_filter_tokens(&["x".repeat(MAX_INDEX_TOKEN_BYTES + 1)]).is_err());
|
||||
assert!(validate_filter_tokens(&[String::new()]).is_err());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ privacy
|
||||
|
||||
#[test]
|
||||
fn card_from_subscription_is_a_public_projection_only() {
|
||||
let base = spec("show.xml", "Rust Radio");
|
||||
let mut subscription = subscription(&base);
|
||||
let spec = PodcastCardSpec::from_subscription(&subscription).unwrap();
|
||||
assert_eq!(spec.feed_url, base.feed_url);
|
||||
assert_eq!(spec.id_feed_url, None);
|
||||
let record = card_with(spec, &secret(1), 1, NOW);
|
||||
let bytes = postcard::to_stdvec(&record).unwrap();
|
||||
let private = b"private-op-id-123";
|
||||
assert!(!bytes.windows(private.len()).any(|window| window == private));
|
||||
assert!(
|
||||
!record
|
||||
.signing_bytes()
|
||||
.windows(private.len())
|
||||
.any(|window| window == private)
|
||||
);
|
||||
|
||||
// After a feed move, the card binds the id to the original URL.
|
||||
subscription.feed_aliases = vec![base.feed_url.clone()];
|
||||
subscription.podcast.feed_url = feed("moved.xml");
|
||||
let moved = PodcastCardSpec::from_subscription(&subscription).unwrap();
|
||||
assert_eq!(moved.feed_url, feed("moved.xml"));
|
||||
assert_eq!(moved.id_feed_url, Some(base.feed_url.clone()));
|
||||
moved.validate().unwrap();
|
||||
|
||||
// Without the original URL the id cannot be proven and nothing is built.
|
||||
subscription.feed_aliases.clear();
|
||||
assert!(PodcastCardSpec::from_subscription(&subscription).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn privacy_warnings_cover_both_feed_urls() {
|
||||
let mut card = spec("show.xml", "Rust Radio");
|
||||
assert!(card.privacy_warnings().is_empty());
|
||||
card.id_feed_url =
|
||||
Some(FeedUrl::parse("https://feeds.example.test/show.xml?token=abc").unwrap());
|
||||
assert_eq!(card.privacy_warnings().len(), 1);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------- search
|
||||
|
||||
fn music(name: &str, kind: ItemKind) -> LibraryItem {
|
||||
LibraryItem {
|
||||
id: ItemId::from_bytes([1; 32]),
|
||||
owner: secret(5).public(),
|
||||
kind,
|
||||
name: name.to_string(),
|
||||
normalized_name: crate::normalize_name(name),
|
||||
artist_names: Vec::new(),
|
||||
featured_artist_names: Vec::new(),
|
||||
year: None,
|
||||
release_type: None,
|
||||
release_title: None,
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
duration_seconds: None,
|
||||
content_id: None,
|
||||
revision: 1,
|
||||
deleted: false,
|
||||
updated_at_ms: NOW,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hits_merge_publishers_and_drop_tombstoned_ones() {
|
||||
let a = secret(1);
|
||||
let b = secret(2);
|
||||
let c = secret(3);
|
||||
let mut moved = spec("moved.xml", "Rust Radio");
|
||||
moved.id_feed_url = Some(feed("show.xml"));
|
||||
moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml"));
|
||||
let records = vec![
|
||||
card(&a, 1),
|
||||
card(&a, 2),
|
||||
card_with(moved, &b, 1, NOW + 10),
|
||||
card(&c, 1),
|
||||
tombstone(&c, 2, NOW),
|
||||
card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW),
|
||||
];
|
||||
let hits = rank_hits(
|
||||
aggregate_hits(records),
|
||||
"rust radio",
|
||||
&podcast_query_tokens("rust radio"),
|
||||
);
|
||||
assert_eq!(hits.len(), 1, "Rust Hour lacks the token radio");
|
||||
let hit = &hits[0];
|
||||
assert_eq!(hit.kind(), SearchHitKind::Podcast);
|
||||
let mut expected = vec![a.public(), b.public()];
|
||||
expected.sort_by_key(|key| *key.as_bytes());
|
||||
assert_eq!(hit.publishers, expected);
|
||||
// Equal votes per feed URL: the most recently issued card represents.
|
||||
assert_eq!(hit.card.publisher, b.public());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn federated_results_are_typed_and_exact_matches_lead() {
|
||||
let podcast = aggregate_hits(vec![card(&secret(1), 1)]).remove(0);
|
||||
let merged = merge_search_results(
|
||||
"rust radio",
|
||||
vec![
|
||||
music("Rust Radio Theme", ItemKind::Track),
|
||||
music("Rust", ItemKind::Artist),
|
||||
],
|
||||
vec![podcast],
|
||||
);
|
||||
let kinds: Vec<_> = merged.iter().map(FederatedSearchHit::kind).collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
vec![
|
||||
SearchHitKind::Podcast,
|
||||
SearchHitKind::Track,
|
||||
SearchHitKind::Artist
|
||||
]
|
||||
);
|
||||
let json = serde_json::to_value(&merged[0]).unwrap();
|
||||
assert_eq!(json["type"], "podcast");
|
||||
assert_eq!(json["hit"]["card"]["title"], "Rust Radio");
|
||||
let json = serde_json::to_value(&merged[1]).unwrap();
|
||||
assert_eq!(json["type"], "music");
|
||||
assert_eq!(json["hit"]["kind"], "track");
|
||||
assert_eq!(SearchHitKind::Podcast.as_str(), "podcast");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_tokens_are_normalized_distinct_and_bounded() {
|
||||
assert_eq!(
|
||||
podcast_query_tokens(" Rust, RUST radio!"),
|
||||
vec!["rust", "radio"]
|
||||
);
|
||||
let long: Vec<String> = (0..20).map(|i| format!("w{i}")).collect();
|
||||
assert_eq!(
|
||||
podcast_query_tokens(&long.join(" ")).len(),
|
||||
MAX_PODCAST_QUERY_TOKENS
|
||||
);
|
||||
assert!(podcast_query_tokens("!!!").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_responses_are_bounded() {
|
||||
let response = PodcastDhtResponse::error("é".repeat(MAX_ERROR_BYTES));
|
||||
assert!(response.error.as_ref().unwrap().len() <= MAX_ERROR_BYTES);
|
||||
validate_response(&response).unwrap();
|
||||
}
|
||||
Reference in new issue
Block a user