Подкасты в FRID: публичное обнаружение podcast_dht — подписанные карточки, tombstone и поиск (MR-F3) #3

Merged
ab merged 9 commits from sanapach/frid:podcasts/discovery into main 2026-11-20 13:51:54 +00:00
7 changed files with 1276 additions and 281 deletions
Showing only changes of commit 81ccd924d0 - Show all commits

No files matched your search

+24 -8
View File
@@ -56,7 +56,13 @@ pub struct CapabilityManifest {
}
impl CapabilityManifest {
/// Creates a manifest containing every protocol owned by Frid.
/// Creates a manifest containing every protocol owned by Frid that a
/// music-DHT application serves.
///
/// The public podcast-discovery overlay is opt-in: applications that run
/// [`crate::podcast_dht::PodcastDht`] add it with
/// [`Self::with_podcast_dht`], so an application that does not serve the
/// overlay never advertises it.
pub fn frid(application: impl Into<String>, application_version: impl Into<String>) -> Self {
let mut protocols = BTreeMap::new();
protocols.insert(
@@ -81,10 +87,6 @@ impl CapabilityManifest {
SIMILARITY_DHT_ID.to_string(),
crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION,
);
protocols.insert(
PODCAST_DHT_ID.to_string(),
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
);
protocols.insert(
DEVICE_SYNC_ID.to_string(),
crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION,
@@ -111,6 +113,17 @@ impl CapabilityManifest {
self
}
/// Advertises the public podcast-discovery overlay
/// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]).
/// Call it only when the application serves
/// [`crate::podcast_dht::PODCAST_DHT_ALPN`].
pub fn with_podcast_dht(self) -> Self {
self.with_protocol(
PODCAST_DHT_ID,
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
)
}
/// Version of protocol `id` advertised by the peer, if any.
pub fn protocol_version(&self, id: &str) -> Option<u16> {
self.protocols.get(id).copied()
@@ -237,7 +250,6 @@ mod tests {
CATALOG_ID,
SIMILARITY_ID,
SIMILARITY_DHT_ID,
PODCAST_DHT_ID,
DEVICE_SYNC_ID,
JAM_ID,
] {
@@ -247,13 +259,17 @@ mod tests {
}
#[test]
fn podcast_dht_capability_is_advertised_and_removable() {
let manifest = CapabilityManifest::frid("furumi", "1.2.3");
fn podcast_dht_capability_is_opt_in_and_removable() {
let plain = CapabilityManifest::frid("furumi", "1.2.3");
assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None);
assert!(!plain.supports_podcast_dht());
let manifest = plain.with_podcast_dht();
assert_eq!(
manifest.protocol_version(PODCAST_DHT_ID),
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
);
assert!(manifest.supports_podcast_dht());
manifest.validate().unwrap();
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
assert!(!without.supports_podcast_dht());
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
+13 -12
View File
@@ -396,6 +396,19 @@ impl PodcastCard {
card_index_tokens(&self.title, self.author.as_deref())
}
/// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url`
/// when the feed has not moved). A valid card's id hashes from it, so it
/// is the only URL of the card bound to the podcast.
pub fn origin_feed_url(&self) -> &FeedUrl {
self.id_feed_url.as_ref().unwrap_or(&self.feed_url)
}
/// Whether `feed_url` is the URL the id is derived from. Otherwise
/// `feed_url` is an unverified claim of the publisher that the feed moved.
pub fn feed_url_is_origin(&self) -> bool {
self.id_feed_url.is_none()
}
/// Canonical signing input (see `PODCASTS.md`).
pub fn signing_bytes(&self) -> Vec<u8> {
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
@@ -735,18 +748,6 @@ pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord)
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
}
/// Whether `winner` makes `other` obsolete everywhere, not just under one key:
/// `other` is a card of a lower revision, or `winner` is a tombstone of an
/// equal or higher revision.
pub(crate) fn supersedes_everywhere(
winner: &SignedPodcastRecord,
other: &SignedPodcastRecord,
) -> bool {
!other.is_tombstone()
&& (other.revision() < winner.revision()
|| (winner.is_tombstone() && other.revision() <= winner.revision()))
}
struct Canonical(Vec<u8>);
impl Canonical {
+244 -36
View File
@@ -27,6 +27,13 @@
//! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with
//! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take
//! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`].
//! Advertise the overlay with [`CapabilityManifest::with_podcast_dht`] (it is
//! opt-in) and pass peer manifests to [`PodcastDht::note_peer_capabilities`].
//!
//! Only the URL a podcast id hashes from is bound to the podcast; search hits
//! offer that URL ([`PodcastSearchHit::feed_url`]) and report moved-feed URLs
//! as unverified claims. The publisher key is the device's transport identity
//! and therefore linkable to its network address and music-DHT records.
mod card;
mod privacy;
@@ -36,7 +43,7 @@ mod store;
#[cfg(test)]
mod tests;
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;
use std::str::FromStr;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
@@ -69,10 +76,16 @@ pub use privacy::{
FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning,
};
pub use search::{
FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit,
SearchHitKind, merge_search_results, podcast_query_tokens,
ClaimedFeedUrl, FederatedSearchHit, MAX_HIT_FEED_URL_CLAIMS, MAX_HIT_PUBLISHERS,
MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, SearchHitKind, merge_search_results,
podcast_query_tokens,
};
pub use store::{
MAX_PERSISTED_PODCAST_DHT_PEERS, MAX_PODCAST_CACHE_BYTES, MAX_PODCAST_CACHE_RECORDS,
MAX_PODCAST_RECORDS_PER_KEY, MAX_PODCAST_RECORDS_PER_PUBLISHER,
MAX_PODCAST_RESPONSE_TOMBSTONES, MAX_PODCAST_TOMBSTONES_PER_KEY,
PODCAST_DHT_CACHE_SCHEMA_VERSION,
};
pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION};
use card::publication_error;
use search::{aggregate_hits, rank_hits};
@@ -105,6 +118,16 @@ pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60);
/// Maximum inbound requests processed concurrently; further streams are
/// dropped.
pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64;
/// Maximum inbound requests of one peer processed concurrently; further
/// streams of that peer are dropped.
pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8;
/// Maximum distinct `(podcast_id, publisher)` records one lookup keeps;
/// records of further owners are ignored.
pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512;
/// A node stores a placement only when fewer than this many of its known
/// overlay contacts are closer to the key than itself (`2 * K`: the `K`
/// closest with slack for differing routing views).
pub const PODCAST_STORE_RESPONSIBILITY_WINDOW: usize = 2 * K;
/// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is
/// not contacted.
pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60);
@@ -228,7 +251,7 @@ pub struct PodcastDhtPublishStats {
}
/// Result of [`PodcastDht::sync_shares`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PodcastDhtSyncStats {
/// Cards published with a new revision, or refreshed because they used
/// half their lifetime.
@@ -237,10 +260,38 @@ pub struct PodcastDhtSyncStats {
pub unchanged: usize,
/// Podcasts unpublished with a tombstone.
pub unpublished: usize,
/// Shared podcasts not published because their card has privacy
/// warnings the user has not confirmed. The UI asks, then calls
/// [`PodcastDht::publish`] or `sync_shares` with
/// [`PodcastShareState::privacy_confirmed`]. An earlier publication of
/// the podcast stays as it was.
pub held_back: Vec<PodcastPrivacyHold>,
/// Podcasts whose card could not be built or signed. They are not
/// published, and an earlier publication of them is unpublished.
pub failed: Vec<PodcastSyncFailure>,
/// Network statistics of the pushed records.
pub push: PodcastDhtPublishStats,
}
/// A shared podcast held back by [`PodcastDht::sync_shares`] until the user
/// confirms publishing URLs that look private.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PodcastPrivacyHold {
/// Podcast.
pub podcast_id: PodcastId,
/// Warnings of [`PodcastCardSpec::privacy_warnings`].
pub warnings: Vec<FeedUrlPrivacyWarning>,
}
/// A podcast [`PodcastDht::sync_shares`] could not publish or unpublish.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PodcastSyncFailure {
/// Podcast.
pub podcast_id: PodcastId,
/// Diagnostic.
pub error: String,
}
/// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PodcastShareState {
@@ -251,6 +302,11 @@ pub struct PodcastShareState {
/// Card to publish when shared; `None` derives it with
/// [`PodcastCardSpec::from_subscription`].
pub card: Option<PodcastCardSpec>,
/// Whether the user confirmed publishing this card's feed URLs despite
/// its [`PodcastCardSpec::privacy_warnings`]. Without it, a card with
/// warnings is only published when this device already publishes the
/// same feed URLs.
pub privacy_confirmed: bool,
}
impl PodcastShareState {
@@ -258,6 +314,22 @@ impl PodcastShareState {
pub fn is_shared(&self) -> bool {
effective_share(Some(&self.subscription), self.share.as_ref())
}
/// The bounded, validated card [`PodcastDht::sync_shares`] would
/// publish: [`Self::card`] or one derived from the subscription.
pub fn card_spec(&self) -> Result<PodcastCardSpec> {
let card = match &self.card {
Some(card) => card.bounded(),
None => PodcastCardSpec::from_subscription(&self.subscription)?,
};
if card.podcast_id != self.subscription.podcast_id {
return Err(publication_error(
"card and subscription describe different podcasts",
));
}
card.validate()?;
Ok(card)
}
}
/// This device's publication state of one podcast.
@@ -284,6 +356,7 @@ pub struct PodcastDht {
failed: Mutex<HashMap<EndpointId, u64>>,
publish_gate: Semaphore,
serve_limit: Arc<Semaphore>,
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
}
impl std::fmt::Debug for PodcastDht {
@@ -330,6 +403,7 @@ impl PodcastDht {
failed: Mutex::new(HashMap::new()),
publish_gate: Semaphore::new(1),
serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)),
inflight: Arc::new(Mutex::new(HashMap::new())),
}))
}
@@ -341,6 +415,10 @@ impl PodcastDht {
/// Serves authenticated overlay requests until `acceptor` closes.
pub async fn serve(self: Arc<Self>, mut acceptor: StreamAcceptor) {
while let Some(stream) = acceptor.accept().await {
let Some(slot) = PeerSlot::acquire(&self.inflight, stream.peer_id) else {
debug!(peer = %stream.peer_id, "podcast DHT request dropped: peer limit");
continue;
};
let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else {
debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded");
continue;
@@ -348,6 +426,7 @@ impl PodcastDht {
let this = Arc::clone(&self);
tokio::spawn(async move {
let _permit = permit;
let _slot = slot;
let peer = stream.peer_id;
match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await {
Ok(Ok(())) => {}
@@ -421,7 +500,10 @@ impl PodcastDht {
}
/// Discovers overlay peers among the routing table and the music DHT
/// contacts.
/// contacts with one `FindNode` each. Music-DHT contacts whose capability
/// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are
/// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become
/// overlay contacts when they answer.
pub async fn bootstrap(&self) -> Result<()> {
let known_feature: HashSet<EndpointId> = lock(&self.routing)
.contacts()
@@ -521,44 +603,92 @@ impl PodcastDht {
/// [`Self::republish`]). Every other podcast this device published,
/// including podcasts absent from `states`, is unpublished. The call is
/// idempotent: repeating it publishes nothing new.
///
/// Each podcast is handled on its own, so revocation is fail-safe: a
/// shared podcast whose card cannot be built is reported in
/// [`PodcastDhtSyncStats::failed`] and unpublished, and never prevents
/// other podcasts from being unpublished. A card with
/// [`PodcastCardSpec::privacy_warnings`] is held back
/// ([`PodcastDhtSyncStats::held_back`]) unless
/// [`PodcastShareState::privacy_confirmed`] is set or this device already
/// publishes the same feed URLs.
pub async fn sync_shares(&self, states: Vec<PodcastShareState>) -> Result<PodcastDhtSyncStats> {
let mut desired: HashMap<PodcastId, PodcastCardSpec> = HashMap::new();
for state in states {
let _permit = self.publish_permit().await?;
let local = self.database.local_publications().await?;
let active: HashMap<&PodcastId, &PodcastCardSpec> = local
.iter()
.filter_map(|publication| {
publication
.spec
.as_ref()
.map(|spec| (&publication.podcast_id, spec))
})
.collect();
let mut stats = PodcastDhtSyncStats::default();
let mut desired: BTreeMap<PodcastId, PodcastCardSpec> = BTreeMap::new();
let mut held: HashSet<PodcastId> = HashSet::new();
for state in &states {
if !state.is_shared() {
continue;
}
let card = match state.card {
Some(card) => card.bounded(),
None => PodcastCardSpec::from_subscription(&state.subscription)?,
let podcast_id = state.subscription.podcast_id.clone();
let card = match state.card_spec() {
Ok(card) => card,
Err(error) => {
debug!(%podcast_id, %error, "podcast card cannot be published");
stats.failed.push(PodcastSyncFailure {
podcast_id,
error: error.to_string(),
});
continue;
}
};
if card.podcast_id != state.subscription.podcast_id {
return Err(publication_error(
"card and subscription describe different podcasts",
));
let warnings = card.privacy_warnings();
let already_published = active.get(&podcast_id).is_some_and(|published| {
published.feed_url == card.feed_url && published.id_feed_url == card.id_feed_url
});
if !warnings.is_empty() && !state.privacy_confirmed && !already_published {
held.insert(podcast_id.clone());
stats.held_back.push(PodcastPrivacyHold {
podcast_id,
warnings,
});
continue;
}
card.validate()?;
desired.insert(card.podcast_id.clone(), card);
desired.insert(podcast_id, card);
}
let _permit = self.publish_permit().await?;
let local = self.database.local_publications().await?;
let mut stats = PodcastDhtSyncStats::default();
let mut records = Vec::new();
for publication in &local {
if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) {
records.extend(self.unpublish_locked(&publication.podcast_id).await?);
stats.unpublished += 1;
let podcast_id = &publication.podcast_id;
if publication.spec.is_none()
|| desired.contains_key(podcast_id)
|| held.contains(podcast_id)
{
continue;
}
match self.unpublish_locked(podcast_id).await {
Ok(record) => {
records.extend(record);
stats.unpublished += 1;
}
Err(error) => stats.failed.push(PodcastSyncFailure {
podcast_id: podcast_id.clone(),
error: error.to_string(),
}),
}
}
let mut desired: Vec<_> = desired.into_values().collect();
desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id));
for card in desired {
match self.sign_card_locked(card, false).await? {
Some(record) => {
for (podcast_id, card) in desired {
match self.sign_card_locked(card, false).await {
Ok(Some(record)) => {
records.push(record);
stats.published += 1;
}
None => stats.unchanged += 1,
Ok(None) => stats.unchanged += 1,
Err(error) => stats.failed.push(PodcastSyncFailure {
podcast_id,
error: error.to_string(),
}),
}
}
stats.push = self.push(records).await?;
@@ -613,8 +743,8 @@ impl PodcastDht {
/// normalized token of `query` ([`podcast_query_tokens`]).
///
/// Hits are merged by `podcast_id` over publishers and ranked: exact
/// normalized title first, then more publishers. Partial results are
/// returned when peers fail.
/// normalized title first, then hits with a card under the origin feed
/// URL. Partial results are returned when peers fail.
pub async fn search(&self, query: &str, limit: usize) -> Result<Vec<PodcastSearchHit>> {
let tokens = podcast_query_tokens(query);
if tokens.is_empty() || limit == 0 {
@@ -966,8 +1096,25 @@ impl PodcastDht {
))
}
PodcastDhtOperation::StoreBatch { records } => {
let entries = validate_store_batch(records, &self.network_id, now_ms())?;
let stored = self.database.store(entries, now_ms()).await?;
let entries =
validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?;
let nodes: Vec<NodeId> = lock(&self.routing)
.contacts()
.into_iter()
.map(|contact| contact.node_id)
.collect();
let entries: Vec<_> = entries
.into_iter()
.filter_map(|(mut keys, record)| {
keys.retain(|key| is_responsible(&self.own_contact.node_id, &nodes, key));
(!keys.is_empty()).then_some((keys, record))
})
.collect();
let stored = if entries.is_empty() {
0
} else {
self.database.store(entries, now_ms()).await?
};
Ok(PodcastDhtResponse::success(
Vec::new(),
Vec::new(),
@@ -1099,11 +1246,17 @@ impl PodcastDht {
}
}
/// Validates a store batch received from a peer: bounds, every record's
/// signature and lifetime, and that every key belongs to its record.
/// Validates a store batch received from `requester`: bounds, that every
/// record is published by the requester itself, every record's signature and
/// lifetime, and that every key belongs to its record.
///
/// Replicas never forward records, so only a record's publisher stores it.
/// Nobody can therefore replay or inject another device's records, and
/// per-publisher quotas apply to the storing peer.
fn validate_store_batch(
entries: Vec<PodcastStoreEntry>,
network_id: &NetworkId,
requester: EndpointId,
now_ms: u64,
) -> Result<Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>> {
if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS {
@@ -1117,6 +1270,11 @@ fn validate_store_batch(
if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() {
return Err(protocol_error("invalid podcast DHT store keys"));
}
if entry.record.publisher() != requester {
return Err(protocol_error(
"podcast records must be stored by their publisher",
));
}
entry.record.verify_at(network_id, now_ms)?;
let valid_keys = entry.record.dht_keys();
if keys.iter().any(|key| !valid_keys.contains(key)) {
@@ -1128,6 +1286,18 @@ fn validate_store_batch(
.collect()
}
/// Whether a node is among the nodes that should store `key`: fewer than
/// [`PODCAST_STORE_RESPONSIBILITY_WINDOW`] known contacts are closer.
fn is_responsible(own: &NodeId, contacts: &[NodeId], key: &PodcastDhtKey) -> bool {
let own_distance = distance(own.as_bytes(), key.as_bytes());
contacts
.iter()
.filter(|node| distance(node.as_bytes(), key.as_bytes()) < own_distance)
.take(PODCAST_STORE_RESPONSIBILITY_WINDOW)
.count()
< PODCAST_STORE_RESPONSIBILITY_WINDOW
}
fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
if tokens.len() > MAX_PODCAST_QUERY_TOKENS
|| tokens
@@ -1139,7 +1309,8 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
Ok(())
}
/// Adds verified records found under `key` to the per-owner winners.
/// Adds verified records found under `key` to the per-owner winners, keeping
/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners.
fn merge_records(
winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>,
records: Vec<SignedPodcastRecord>,
@@ -1161,6 +1332,7 @@ fn merge_records(
let owner = (record.podcast_id().clone(), record.publisher());
match winners.get(&owner) {
Some(current) if compare_records(current, &record).is_ge() => {}
None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {}
_ => {
winners.insert(owner, record);
}
@@ -1168,6 +1340,42 @@ fn merge_records(
}
}
/// One in-flight inbound request of a peer; released on drop.
struct PeerSlot {
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
peer: EndpointId,
}
impl PeerSlot {
fn acquire(
inflight: &Arc<Mutex<HashMap<EndpointId, usize>>>,
peer: EndpointId,
) -> Option<Self> {
let mut map = lock(inflight);
let count = map.entry(peer).or_default();
if *count >= MAX_PODCAST_DHT_REQUESTS_PER_PEER {
return None;
}
*count += 1;
Some(Self {
inflight: Arc::clone(inflight),
peer,
})
}
}
impl Drop for PeerSlot {
fn drop(&mut self) {
let mut map = lock(&self.inflight);
if let Some(count) = map.get_mut(&self.peer) {
*count = count.saturating_sub(1);
if *count == 0 {
map.remove(&self.peer);
}
}
}
}
fn validate_contact(
contact: &NodeContact,
network_id: &NetworkId,
+77 -19
View File
@@ -8,15 +8,17 @@ use federation_net::EndpointId;
use serde::{Deserialize, Serialize};
use crate::normalization::{normalize_name, tokenize};
use crate::podcast::PodcastId;
use crate::podcast::{FeedUrl, PodcastId};
use crate::record::{ItemKind, LibraryItem};
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
/// Maximum distinct query tokens used by a podcast text search.
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
/// Maximum publishers reported per hit.
/// Maximum publishers reported per hit and per claimed feed URL.
pub const MAX_HIT_PUBLISHERS: usize = 64;
/// Maximum claimed moved feed URLs reported per hit.
pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8;
/// Kind of a federated search hit.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -64,18 +66,35 @@ impl From<ItemKind> for SearchHitKind {
///
/// Cards of the same podcast published by several devices are merged by
/// `podcast_id`. The hit carries no personal state of any publisher.
///
/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every
/// other field is a claim of some publisher device, and device identities are
/// free, so neither the metadata nor the number of publishers is a trust
/// signal.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PodcastSearchHit {
/// Podcast.
pub podcast_id: PodcastId,
/// Representative card: the most common `feed_url` among the
/// publishers' current cards, then the most recently issued card, then
/// the greatest publisher key.
/// The URL `podcast_id` is derived from
/// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with
/// this URL; follow the feed's own HTTP redirects and
/// `itunes:new-feed-url` from there.
pub feed_url: FeedUrl,
/// Representative card for display: a card whose `feed_url` is the
/// origin URL when one exists, then the title most publishers agree on,
/// then the smallest publisher key. Its metadata is a publisher's claim.
pub card: PodcastCard,
/// Sorted distinct publisher devices with a current valid card, at most
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
/// one user may publish from several devices.
/// one user may publish from several devices, and anyone can mint
/// devices.
pub publishers: Vec<EndpointId>,
/// Feed URLs other than [`Self::feed_url`] that publishers claim the
/// podcast moved to, most claimed first, at most
/// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this
/// podcast or record it as an alias unless the feed at
/// [`Self::feed_url`] itself redirects there.
pub claimed_feed_urls: Vec<ClaimedFeedUrl>,
}
impl PodcastSearchHit {
@@ -85,6 +104,16 @@ impl PodcastSearchHit {
}
}
/// An unverified moved-feed URL claimed by some publishers of a podcast.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ClaimedFeedUrl {
/// Claimed current feed URL.
pub feed_url: FeedUrl,
/// Sorted distinct publisher devices claiming it, at most
/// [`MAX_HIT_PUBLISHERS`].
pub publishers: Vec<EndpointId>,
}
/// A music or podcast result of a federated search.
///
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
@@ -184,37 +213,66 @@ pub(crate) fn aggregate_hits(
by_podcast
.into_iter()
.map(|(podcast_id, cards)| {
let mut feed_votes: HashMap<&str, usize> = HashMap::new();
let mut title_votes: HashMap<String, usize> = HashMap::new();
for card in &cards {
*feed_votes.entry(card.feed_url.as_str()).or_default() += 1;
*title_votes.entry(normalize_name(&card.title)).or_default() += 1;
}
// Publisher-controlled times and revisions never break ties.
let card = cards
.iter()
.max_by_key(|card| {
(
feed_votes[card.feed_url.as_str()],
card.issued_at_ms,
*card.publisher.as_bytes(),
card.feed_url_is_origin(),
title_votes[&normalize_name(&card.title)],
Reverse(*card.publisher.as_bytes()),
)
})
.cloned()
.expect("a group has at least one card");
let mut publishers: Vec<EndpointId> = cards.iter().map(|card| card.publisher).collect();
publishers.sort_by_key(|publisher| *publisher.as_bytes());
publishers.dedup();
publishers.truncate(MAX_HIT_PUBLISHERS);
let mut claims: BTreeMap<&FeedUrl, Vec<EndpointId>> = BTreeMap::new();
for card in cards.iter().filter(|card| !card.feed_url_is_origin()) {
claims
.entry(&card.feed_url)
.or_default()
.push(card.publisher);
}
let mut claimed_feed_urls: Vec<ClaimedFeedUrl> = claims
.into_iter()
.map(|(feed_url, publishers)| ClaimedFeedUrl {
feed_url: feed_url.clone(),
publishers: sorted_publishers(publishers),
})
.collect();
claimed_feed_urls.sort_by(|left, right| {
right
.publishers
.len()
.cmp(&left.publishers.len())
.then_with(|| left.feed_url.cmp(&right.feed_url))
});
claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS);
PodcastSearchHit {
podcast_id,
feed_url: card.origin_feed_url().clone(),
publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()),
card,
publishers,
claimed_feed_urls,
}
})
.collect()
}
fn sorted_publishers(mut publishers: Vec<EndpointId>) -> Vec<EndpointId> {
publishers.sort_by_key(|publisher| *publisher.as_bytes());
publishers.dedup();
publishers.truncate(MAX_HIT_PUBLISHERS);
publishers
}
/// Keeps hits whose card is indexed under every query token and ranks
/// them: exact normalized title first, then more publishers, then title and
/// id.
/// them: exact normalized title first, then hits with a card published under
/// the origin feed URL, then title and id. The publisher count is not used:
/// anyone can inflate it with new device identities.
pub(crate) fn rank_hits(
mut hits: Vec<PodcastSearchHit>,
query: &str,
@@ -229,7 +287,7 @@ pub(crate) fn rank_hits(
let title = normalize_name(&hit.card.title);
(
title != normalized_query,
Reverse(hit.publishers.len()),
!hit.card.feed_url_is_origin(),
title,
hit.podcast_id.clone(),
)
+343 -150
View File
@@ -5,7 +5,7 @@
//! (like the similarity-routing cache). Every record read from it is decoded
//! and verified again before it is used or served.
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use std::str::FromStr;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
@@ -17,32 +17,66 @@ use crate::error::{MusicDhtError, Result};
use crate::podcast::PodcastId;
use crate::routing::{NodeContact, NodeId};
use super::card::{
PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere,
};
use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records};
/// `PRAGMA user_version` of the podcast-discovery cache schema.
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1;
/// Maximum distinct `(podcast_id, publisher)` records cached under one key.
/// When full, the record expiring first is evicted if the new one outlives it.
/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1
/// (pre-release) stored one payload per key; its replica table is dropped on
/// open because the cache is replaceable. Own publications are kept.
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2;
/// Maximum records (cards and tombstones) indexed under one key. A full key
/// admits no new `(podcast_id, publisher)` until a record expires: existing
/// records are never evicted for newcomers, and their owners keep refreshing
/// them in place.
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256;
/// Maximum tombstones newly indexed under one key. A tombstone is admitted
/// under a text key only in place of a record of the same
/// `(podcast_id, publisher)` already indexed there, so this budget applies to
/// the podcast-id key, where tombstones of unknown cards land.
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32;
/// Maximum tombstones in one `FindValue` response; cards fill the rest.
pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32;
/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a
/// replica.
pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256;
/// Maximum `(podcast_id, publisher)` records in a replica cache.
pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096;
/// Maximum total encoded size of the records in a replica cache.
pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024;
/// Maximum persisted overlay contacts; the least recently seen are dropped.
pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024;
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS podcast_dht_records (
dht_key BLOB NOT NULL,
podcast_id TEXT NOT NULL,
publisher TEXT NOT NULL,
tombstone INTEGER NOT NULL,
expires_at_ms INTEGER NOT NULL,
payload BLOB NOT NULL,
PRIMARY KEY (dht_key, podcast_id, publisher)
podcast_id TEXT NOT NULL,
publisher TEXT NOT NULL,
tombstone INTEGER NOT NULL,
expires_at_ms INTEGER NOT NULL,
payload_bytes INTEGER NOT NULL,
payload BLOB NOT NULL,
PRIMARY KEY (podcast_id, publisher)
);
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
ON podcast_dht_records(expires_at_ms);
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner
ON podcast_dht_records(podcast_id, publisher);
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher
ON podcast_dht_records(publisher);
CREATE TABLE IF NOT EXISTS podcast_dht_keys (
dht_key BLOB NOT NULL,
podcast_id TEXT NOT NULL,
publisher TEXT NOT NULL,
tombstone INTEGER NOT NULL,
expires_at_ms INTEGER NOT NULL,
first_stored_at_ms INTEGER NOT NULL,
PRIMARY KEY (dht_key, podcast_id, publisher)
);
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner
ON podcast_dht_keys(podcast_id, publisher);
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry
ON podcast_dht_keys(expires_at_ms);
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
peer_id TEXT PRIMARY KEY,
@@ -60,6 +94,28 @@ CREATE TABLE IF NOT EXISTS podcast_dht_local (
);
";
/// Admission limits of a replica cache.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct StoreLimits {
pub per_key: usize,
pub tombstones_per_key: usize,
pub per_publisher: usize,
pub records: usize,
pub bytes: usize,
}
impl Default for StoreLimits {
fn default() -> Self {
Self {
per_key: MAX_PODCAST_RECORDS_PER_KEY,
tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY,
per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER,
records: MAX_PODCAST_CACHE_RECORDS,
bytes: MAX_PODCAST_CACHE_BYTES,
}
}
}
/// This device's latest publication of one podcast.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct LocalPublication {
@@ -76,6 +132,7 @@ pub(crate) struct LocalPublication {
pub(crate) struct PodcastDhtDatabase {
conn: Arc<Mutex<Connection>>,
network_id: NetworkId,
limits: StoreLimits,
}
impl std::fmt::Debug for PodcastDhtDatabase {
@@ -88,6 +145,14 @@ impl std::fmt::Debug for PodcastDhtDatabase {
impl PodcastDhtDatabase {
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
Self::open_with_limits(path, network_id, StoreLimits::default()).await
}
pub async fn open_with_limits(
path: &Path,
network_id: NetworkId,
limits: StoreLimits,
) -> Result<Self> {
let path = path.to_path_buf();
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
let conn = Connection::open(&path).map_err(|error| {
@@ -101,6 +166,11 @@ impl PodcastDhtDatabase {
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
)));
}
if version == 1 {
// Replicas are replaceable; the old layout is dropped.
conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;")
.map_err(database_error)?;
}
conn.execute_batch(SCHEMA).map_err(|error| {
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
})?;
@@ -113,37 +183,39 @@ impl PodcastDhtDatabase {
Ok(Self {
conn: Arc::new(Mutex::new(conn)),
network_id,
limits,
})
}
async fn call<F, T>(&self, operation: F) -> Result<T>
where
F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result<T> + Send + 'static,
F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result<T> + Send + 'static,
T: Send + 'static,
{
let conn = Arc::clone(&self.conn);
let network_id = self.network_id;
let limits = self.limits;
tokio::task::spawn_blocking(move || {
let guard = lock(&conn);
operation(&guard, &network_id).map_err(database_error)
operation(&guard, &network_id, &limits).map_err(database_error)
})
.await
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
}
/// Stores records under the given keys. The caller has verified every
/// record and checked that each key belongs to it. Returns the number of
/// `(key, record)` rows written.
/// record, checked that each key belongs to it and that the sender is its
/// publisher. Returns the number of `(key, record)` placements written.
pub async fn store(
&self,
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
now_ms: u64,
) -> Result<usize> {
self.call(move |conn, network_id| {
self.call(move |conn, network_id, limits| {
let transaction = conn.unchecked_transaction()?;
let mut written = 0;
for (keys, record) in entries {
written += store_one(&transaction, network_id, &keys, &record, now_ms)?;
written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?;
}
transaction.commit()?;
Ok(written)
@@ -151,9 +223,11 @@ impl PodcastDhtDatabase {
.await
}
/// Valid, unexpired records under `key`, newest expiry first. Cards must
/// contain every token in `filter_tokens`; tombstones are always
/// returned so stale replicas elsewhere can be overruled.
/// Valid, unexpired records under `key`: at most
/// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale
/// replicas elsewhere can be overruled) and cards containing every token
/// in `filter_tokens` up to `limit` records in all. Records indexed longest
/// come first. Cards precede tombstones.
pub async fn records_by_key(
&self,
key: PodcastDhtKey,
@@ -161,52 +235,70 @@ impl PodcastDhtDatabase {
now_ms: u64,
limit: usize,
) -> Result<Vec<SignedPodcastRecord>> {
self.call(move |conn, network_id| {
self.call(move |conn, network_id, _| {
let mut statement = conn.prepare(
"SELECT payload FROM podcast_dht_records
WHERE dht_key = ?1 AND expires_at_ms > ?2
ORDER BY expires_at_ms DESC, podcast_id, publisher",
"SELECT r.payload FROM podcast_dht_keys k
JOIN podcast_dht_records r
ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher
WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3
ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher",
)?;
let rows = statement.query_map(
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|row| row.get::<_, Vec<u8>>(0),
)?;
let mut records = Vec::new();
for row in rows {
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
continue;
};
if !record.belongs_under(&key) {
continue;
let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result<Vec<_>> {
let mut records = Vec::new();
if limit == 0 {
return Ok(records);
}
let matches = record.card().is_none_or(|card| {
let tokens = card.index_tokens();
filter_tokens.iter().all(|token| tokens.contains(token))
});
if matches {
records.push(record);
if records.len() == limit {
break;
let rows = statement.query_map(
params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)],
|row| row.get::<_, Vec<u8>>(0),
)?;
for row in rows {
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
continue;
};
if record.is_tombstone() != tombstone || !record.belongs_under(&key) {
continue;
}
let matches = record.card().is_none_or(|card| {
let tokens = card.index_tokens();
filter_tokens.iter().all(|token| tokens.contains(token))
});
if matches {
records.push(record);
if records.len() == limit {
break;
}
}
}
}
Ok(records)
};
let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?;
let mut records = select(false, limit - tombstones.len())?;
records.extend(tombstones);
Ok(records)
})
.await
}
/// Removes expired records and their key placements. Returns the number
/// of records removed.
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
let now = clamp_i64(now_ms);
conn.execute(
"DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1",
[now],
)?;
conn.execute(
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
[clamp_i64(now_ms)],
[now],
)
})
.await
}
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
conn.execute(
"INSERT OR REPLACE INTO podcast_dht_peers
(peer_id, node_id, ticket, last_seen_ms)
@@ -218,13 +310,20 @@ impl PodcastDhtDatabase {
clamp_i64(contact.last_seen_ms),
],
)?;
conn.execute(
"DELETE FROM podcast_dht_peers WHERE peer_id IN (
SELECT peer_id FROM podcast_dht_peers
ORDER BY last_seen_ms DESC, peer_id
LIMIT -1 OFFSET ?1)",
[MAX_PERSISTED_PODCAST_DHT_PEERS as i64],
)?;
Ok(())
})
.await
}
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
conn.execute(
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
[peer.to_string()],
@@ -235,7 +334,7 @@ impl PodcastDhtDatabase {
}
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
let mut statement =
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
let rows = statement.query_map([], |row| {
@@ -267,7 +366,7 @@ impl PodcastDhtDatabase {
&self,
podcast_id: PodcastId,
) -> Result<Option<LocalPublication>> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
conn.query_row(
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
WHERE podcast_id = ?1",
@@ -281,7 +380,7 @@ impl PodcastDhtDatabase {
}
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
let mut statement = conn.prepare(
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
ORDER BY podcast_id",
@@ -303,7 +402,7 @@ impl PodcastDhtDatabase {
publication: LocalPublication,
now_ms: u64,
) -> Result<()> {
self.call(move |conn, _| {
self.call(move |conn, _, _| {
let spec = publication
.spec
.as_ref()
@@ -332,141 +431,235 @@ impl PodcastDhtDatabase {
fn store_one(
conn: &Connection,
network_id: &NetworkId,
limits: &StoreLimits,
keys: &[PodcastDhtKey],
record: &SignedPodcastRecord,
now_ms: u64,
) -> rusqlite::Result<usize> {
let podcast_id = record.podcast_id().as_str().to_string();
let publisher = record.publisher().to_string();
let now = clamp_i64(now_ms);
// Every cached version of this (podcast, publisher), re-verified.
let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new();
let mut invalid_keys = Vec::new();
{
let mut statement = conn.prepare(
"SELECT dht_key, payload FROM podcast_dht_records
// The cached record of this (podcast, publisher), re-verified.
let cached: Option<(Vec<u8>, i64)> = conn
.query_row(
"SELECT payload, payload_bytes FROM podcast_dht_records
WHERE podcast_id = ?1 AND publisher = ?2",
)?;
let rows = statement.query_map(params![podcast_id, publisher], |row| {
Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?))
})?;
for row in rows {
let (key, payload) = row?;
let decoded = decode_valid(&payload, network_id, now_ms);
match (<[u8; 32]>::try_from(key.as_slice()), decoded) {
(Ok(key_bytes), Some(cached)) => {
existing.insert(key_bytes, cached);
}
_ => invalid_keys.push(key),
params![podcast_id, publisher],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()?;
let (existing, existing_bytes) = match cached {
Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) {
Some(existing) => (Some(existing), bytes.max(0) as usize),
None => {
delete_owner(conn, &podcast_id, &publisher)?;
(None, 0)
}
},
None => {
// Stray placements without a record.
delete_owner(conn, &podcast_id, &publisher)?;
(None, 0)
}
}
for key in invalid_keys {
conn.execute(
"DELETE FROM podcast_dht_records
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
params![key, podcast_id, publisher],
)?;
}
};
// A tombstone or a newer revision cached under any key wins everywhere.
if existing
.values()
.any(|cached| supersedes_everywhere(cached, record))
// Higher revision, tombstone at an equal revision, later republish...
let replace = match &existing {
Some(existing) => match compare_records(existing, record) {
std::cmp::Ordering::Greater => return Ok(0),
std::cmp::Ordering::Equal => false,
std::cmp::Ordering::Less => true,
},
None => true,
};
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
if replace
&& !admits_record(
conn,
limits,
&publisher,
existing.is_none(),
existing_bytes,
payload.len(),
now,
)?
{
return Ok(0);
}
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
let mut indexed: HashSet<[u8; 32]> = HashSet::new();
{
let mut statement = conn.prepare(
"SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
)?;
let rows = statement.query_map(params![podcast_id, publisher], |row| {
row.get::<_, Vec<u8>>(0)
})?;
for row in rows {
if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) {
indexed.insert(key);
}
}
}
let mut written = 0;
if replace {
// Placements the new version no longer belongs under are dropped (a
// renamed card, or a tombstone that does not list the key); the others
// now serve the new version.
let belongs: HashSet<[u8; 32]> = record
.dht_keys()
.iter()
.map(|key| *key.as_bytes())
.collect();
let stale: Vec<[u8; 32]> = indexed
.iter()
.filter(|key| !belongs.contains(*key))
.copied()
.collect();
for key in stale {
conn.execute(
"DELETE FROM podcast_dht_keys
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
params![key.as_slice(), podcast_id, publisher],
)?;
indexed.remove(&key);
}
written += conn.execute(
"UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4
WHERE podcast_id = ?1 AND publisher = ?2",
params![
podcast_id,
publisher,
record.is_tombstone(),
clamp_i64(record.expires_at_ms()),
],
)?;
}
// New placements. A tombstone only takes a new place under the podcast-id
// key: under a text key it may only replace a record of its owner, so
// nobody can index tombstones under someone else's search terms.
let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id());
for key in keys {
if let Some(cached) = existing.get(key.as_bytes())
&& compare_records(cached, record).is_ge()
{
if indexed.contains(key.as_bytes()) {
continue;
}
if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? {
if record.is_tombstone() && *key != id_key {
continue;
}
if !admits_placement(conn, limits, key, record.is_tombstone(), now)? {
continue;
}
conn.execute(
"INSERT INTO podcast_dht_records
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET
tombstone = excluded.tombstone,
expires_at_ms = excluded.expires_at_ms,
payload = excluded.payload",
"INSERT INTO podcast_dht_keys
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![
key.as_bytes().as_slice(),
podcast_id,
publisher,
record.is_tombstone(),
clamp_i64(record.expires_at_ms()),
payload,
now,
],
)?;
indexed.insert(*key.as_bytes());
written += 1;
}
// Drop versions the new record makes obsolete under other keys too.
if written > 0 {
for (key, cached) in &existing {
if !keys.iter().any(|stored| stored.as_bytes() == key)
&& supersedes_everywhere(record, cached)
{
conn.execute(
"DELETE FROM podcast_dht_records
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
params![key.as_slice(), podcast_id, publisher],
)?;
}
}
if indexed.is_empty() {
delete_owner(conn, &podcast_id, &publisher)?;
return Ok(0);
}
if replace {
conn.execute(
"INSERT INTO podcast_dht_records
(podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(podcast_id, publisher) DO UPDATE SET
tombstone = excluded.tombstone,
expires_at_ms = excluded.expires_at_ms,
payload_bytes = excluded.payload_bytes,
payload = excluded.payload",
params![
podcast_id,
publisher,
record.is_tombstone(),
clamp_i64(record.expires_at_ms()),
payload.len() as i64,
payload,
],
)?;
}
Ok(written)
}
/// Ensures there is room for one more record under `key`.
fn make_room(
/// Whether the cache may hold the new version of a record: per-publisher and
/// global record counts for a new owner, and the global byte budget.
fn admits_record(
conn: &Connection,
key: &PodcastDhtKey,
record: &SignedPodcastRecord,
now_ms: u64,
limits: &StoreLimits,
publisher: &str,
new_owner: bool,
previous_bytes: usize,
new_bytes: usize,
now: i64,
) -> rusqlite::Result<bool> {
conn.execute(
"DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2",
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
let (records, bytes): (i64, i64) = conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records
WHERE expires_at_ms > ?1",
[now],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
let count: i64 = conn.query_row(
"SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1",
[key.as_bytes().as_slice()],
|row| row.get(0),
)?;
if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY {
let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
if bytes.saturating_add(new_bytes) > limits.bytes {
return Ok(false);
}
if !new_owner {
return Ok(true);
}
let victim: Option<(String, String, i64)> = conn
.query_row(
"SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records
WHERE dht_key = ?1
ORDER BY expires_at_ms ASC, podcast_id, publisher
LIMIT 1",
[key.as_bytes().as_slice()],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)
.optional()?;
match victim {
Some((podcast_id, publisher, expires_at_ms))
if expires_at_ms < clamp_i64(record.expires_at_ms()) =>
{
conn.execute(
"DELETE FROM podcast_dht_records
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
params![key.as_bytes().as_slice(), podcast_id, publisher],
)?;
Ok(true)
}
_ => Ok(false),
if records.max(0) as usize >= limits.records {
return Ok(false);
}
let own: i64 = conn.query_row(
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
params![publisher, now],
|row| row.get(0),
)?;
Ok((own.max(0) as usize) < limits.per_publisher)
}
/// Whether `key` has room for one more placement. Existing placements are
/// never evicted.
fn admits_placement(
conn: &Connection,
limits: &StoreLimits,
key: &PodcastDhtKey,
tombstone: bool,
now: i64,
) -> rusqlite::Result<bool> {
let (total, tombstones): (i64, i64) = conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
WHERE dht_key = ?1 AND expires_at_ms > ?2",
params![key.as_bytes().as_slice(), now],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
Ok((total.max(0) as usize) < limits.per_key
&& (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key))
}
fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> {
conn.execute(
"DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
params![podcast_id, publisher],
)?;
conn.execute(
"DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2",
params![podcast_id, publisher],
)?;
Ok(())
}
fn decode_valid(
+411 -39
View File
@@ -3,7 +3,8 @@
use federation_net::{NetworkId, SecretKey};
use super::card::supersedes_everywhere;
use crate::routing::NodeId;
use super::search::{aggregate_hits, rank_hits};
use super::*;
use crate::podcast::{FeedUrl, PodcastMeta};
@@ -362,10 +363,6 @@ fn record_order_is_total_and_tombstones_win_ties() {
compare_records(&edited, &older),
compare_records(&older, &edited).reverse()
);
assert!(supersedes_everywhere(&grave, &older));
assert!(supersedes_everywhere(&newer, &older));
assert!(!supersedes_everywhere(&refresh, &older));
assert!(!supersedes_everywhere(&older, &grave));
}
// --------------------------------------------------------------------- cache
@@ -437,7 +434,7 @@ async fn cache_converges_regardless_of_delivery_order() {
}
#[tokio::test]
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() {
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key_it_holds() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let key = secret(1);
@@ -451,7 +448,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key()
vec![active.clone()]
);
// A tombstone stored only under the id key still removes the card under
// A tombstone stored only under the id key still replaces the card under
// the text keys of this node.
let grave = tombstone(&key, 2, NOW);
let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id());
@@ -462,7 +459,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key()
db.records_by_key(token_key, vec!["rust".into()], NOW, 10)
.await
.unwrap(),
Vec::<SignedPodcastRecord>::new()
vec![grave.clone()]
);
// An equal-revision card arriving later loses, under any key.
let same = card(&key, 2);
@@ -564,43 +561,247 @@ async fn cache_read_path_filters_tokens_expiry_and_corruption() {
.unwrap()
.is_empty()
);
assert_eq!(
db.delete_expired(NOW + ttl).await.unwrap(),
MAX_RECORD_DHT_KEYS.min(record.dht_keys().len())
);
assert_eq!(db.delete_expired(NOW + ttl).await.unwrap(), 1);
}
fn numbered_secret(index: usize) -> SecretKey {
let mut bytes = [0u8; 32];
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
bytes[31] = 0x5a;
SecretKey::from_bytes(&bytes)
}
async fn open_db_with(
temp: &tempfile::TempDir,
limits: store::StoreLimits,
) -> store::PodcastDhtDatabase {
store::PodcastDhtDatabase::open_with_limits(
&temp.path().join("podcast-dht.sqlite3"),
network(),
limits,
)
.await
.unwrap()
}
#[tokio::test]
async fn cache_bounds_records_per_key() {
async fn cache_full_key_keeps_its_records_and_refuses_newcomers() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let token_key = PodcastDhtKey::token(&network(), "rust");
let mut entries = Vec::new();
for index in 0..=MAX_PODCAST_RECORDS_PER_KEY {
let key = SecretKey::from_bytes(&{
let mut bytes = [0u8; 32];
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
bytes
});
// Later publishers expire later, so the first one is evicted.
let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64);
// Later publishers expire later; that buys them nothing.
let record = card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(index),
1,
NOW + index as u64,
);
entries.push((vec![token_key], record));
}
let first = entries[0].1.clone();
for chunk in entries.chunks(64) {
db.store(chunk.to_vec(), NOW).await.unwrap();
let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone();
// The first publisher arrived first; the rest a moment later.
db.store(entries[..1].to_vec(), NOW).await.unwrap();
for chunk in entries[1..].chunks(64) {
db.store(chunk.to_vec(), NOW + 1).await.unwrap();
}
let cached = db
.records_by_key(token_key, Vec::new(), NOW, usize::MAX)
.await
.unwrap();
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
assert!(!cached.contains(&first));
// A record expiring before every cached one is not admitted.
assert_eq!(
db.store(vec![(vec![token_key], first)], NOW).await.unwrap(),
0
assert!(cached.contains(&first));
assert!(!cached.contains(&last));
assert_eq!(cached[0], first, "records indexed longest come first");
// The first publisher keeps refreshing its place.
let refresh = card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(0),
1,
NOW + 1_000,
);
assert_eq!(
db.store(vec![(vec![token_key], refresh)], NOW)
.await
.unwrap(),
1
);
}
#[tokio::test]
async fn sybil_tombstones_cannot_suppress_a_card() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let victim = card(&secret(1), 1);
let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id());
let rust = PodcastDhtKey::token(&network(), "rust");
db.store(vec![entry(&victim)], NOW).await.unwrap();
// 256 throwaway identities sign tombstones of the victim's podcast that
// list the victim card's text keys, and issue them as late as allowed.
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
let sybils: Vec<_> = (0..256)
.map(|index| {
let key = numbered_secret(index);
let tombstone = PodcastTombstone::new(
network(),
key.public(),
victim.podcast_id().clone(),
u64::MAX,
NOW + skew,
victim.dht_keys(),
);
let record =
SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap();
(record.dht_keys(), record)
})
.collect();
let mut written = 0;
for chunk in sybils.chunks(MAX_PODCAST_STORE_RECORDS) {
written += db.store(chunk.to_vec(), NOW).await.unwrap();
}
assert_eq!(
written, MAX_PODCAST_TOMBSTONES_PER_KEY,
"only the id key's tombstone budget is used"
);
let by_id = db
.records_by_key(id_key, Vec::new(), NOW, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert!(by_id.contains(&victim));
assert_eq!(
by_id.iter().filter(|record| record.is_tombstone()).count(),
MAX_PODCAST_RESPONSE_TOMBSTONES
);
assert_eq!(
db.records_by_key(rust, vec!["rust".into()], NOW, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap(),
vec![victim.clone()],
"no tombstone is indexed under a text key without its owner's card"
);
// Eleven hours later the victim's republish still lands.
let later = NOW + 11 * 3_600_000;
let republished = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later);
assert_eq!(
db.store(vec![entry(&republished)], later).await.unwrap(),
republished.dht_keys().len()
);
// Sybil cards issued in the future cannot displace it either.
let sybil_cards: Vec<_> = (300..600)
.map(|index| {
entry(&card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(index),
u64::MAX,
later + skew,
))
})
.collect();
for chunk in sybil_cards.chunks(MAX_PODCAST_STORE_RECORDS) {
db.store(chunk.to_vec(), later).await.unwrap();
}
for key in [id_key, rust] {
let records = db
.records_by_key(key, vec![], later, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert!(records.contains(&republished));
assert!(records.len() <= MAX_PODCAST_DHT_RECORDS);
}
}
#[tokio::test]
async fn cache_enforces_publisher_record_and_byte_quotas() {
let limits = store::StoreLimits {
per_key: 100,
tombstones_per_key: 10,
per_publisher: 3,
records: 5,
bytes: usize::MAX,
};
let temp = tempfile::tempdir().unwrap();
let db = open_db_with(&temp, limits).await;
let show = |index: usize, key: &SecretKey| {
entry(&card_with(
spec(&format!("show-{index}.xml"), "Rust Radio"),
key,
1,
NOW,
))
};
let heavy = secret(1);
let stored: Vec<usize> = {
let mut out = Vec::new();
for index in 0..4 {
out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap());
}
out
};
assert!(stored[..3].iter().all(|written| *written > 0));
assert_eq!(
stored[3], 0,
"the fourth podcast of one publisher is refused"
);
// A refresh of an admitted record is always accepted.
let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1);
assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0);
assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0);
assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0);
assert_eq!(
db.store(vec![show(12, &secret(4))], NOW).await.unwrap(),
0,
"the cache holds at most five records"
);
let one = card(&secret(1), 1);
let limits = store::StoreLimits {
bytes: one.encoded_len() * 2,
..store::StoreLimits::default()
};
let temp = tempfile::tempdir().unwrap();
let db = open_db_with(&temp, limits).await;
assert!(
db.store(vec![entry(&card(&secret(1), 1))], NOW)
.await
.unwrap()
> 0
);
assert!(
db.store(vec![entry(&card(&secret(2), 1))], NOW)
.await
.unwrap()
> 0
);
assert_eq!(
db.store(vec![entry(&card(&secret(3), 1))], NOW)
.await
.unwrap(),
0,
"the byte budget is full"
);
}
#[tokio::test]
async fn cache_stores_each_payload_once() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let many: Vec<String> = (0..50).map(|index| format!("w{index}")).collect();
let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW);
assert_eq!(
db.store(vec![entry(&record)], NOW).await.unwrap(),
MAX_RECORD_DHT_KEYS
);
let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap();
let payloads: i64 = conn
.query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(payloads, 1);
}
#[tokio::test]
@@ -634,6 +835,25 @@ async fn cache_refuses_a_newer_schema_and_keeps_local_publications() {
assert_eq!(local[0].record, record);
drop(db);
// A pre-release version-1 replica table is dropped; own publications
// survive.
{
let conn = rusqlite::Connection::open(&path).unwrap();
conn.execute_batch(
"DROP TABLE podcast_dht_records;
CREATE TABLE podcast_dht_records (dht_key BLOB, payload BLOB);
INSERT INTO podcast_dht_records VALUES (x'00', x'00');
PRAGMA user_version = 1;",
)
.unwrap();
}
let db = store::PodcastDhtDatabase::open(&path, network())
.await
.unwrap();
assert_eq!(db.local_publications().await.unwrap().len(), 1);
assert!(db.store(vec![entry(&record)], NOW).await.unwrap() > 0);
drop(db);
rusqlite::Connection::open(&path)
.unwrap()
.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1)
@@ -655,12 +875,14 @@ fn store_batches_are_bounded_and_keys_must_belong() {
keys: record.dht_keys(),
record: record.clone(),
};
validate_store_batch(vec![good.clone()], &network(), NOW).unwrap();
assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err());
let publisher = key.public();
validate_store_batch(vec![good.clone()], &network(), publisher, NOW).unwrap();
assert!(validate_store_batch(Vec::new(), &network(), publisher, NOW).is_err());
assert!(
validate_store_batch(
vec![good.clone(); MAX_PODCAST_STORE_RECORDS],
&network(),
publisher,
NOW
)
.is_ok()
@@ -669,19 +891,87 @@ fn store_batches_are_bounded_and_keys_must_belong() {
validate_store_batch(
vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1],
&network(),
publisher,
NOW
)
.is_err()
);
let mut foreign = good.clone();
foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")];
assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err());
assert!(validate_store_batch(vec![foreign], &network(), publisher, NOW).is_err());
let mut duplicated = good.clone();
duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]];
assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err());
let mut forged = good;
assert!(validate_store_batch(vec![duplicated], &network(), publisher, NOW).is_err());
let mut forged = good.clone();
resign(&mut forged.record, &secret(2));
assert!(validate_store_batch(vec![forged], &network(), NOW).is_err());
assert!(validate_store_batch(vec![forged], &network(), publisher, NOW).is_err());
// A valid record of another publisher may not be stored (or replayed)
// by anyone else.
let error = validate_store_batch(vec![good], &network(), secret(2).public(), NOW).unwrap_err();
assert!(error.to_string().contains("publisher"), "{error}");
}
#[test]
fn replicas_only_accept_keys_they_are_close_to() {
let own = NodeId::from_bytes([0; 32]);
let key = PodcastDhtKey::from_bytes([0; 32]);
let far: Vec<NodeId> = (0..100)
.map(|index| NodeId::from_bytes([0x80 | index as u8; 32]))
.collect();
assert!(is_responsible(&own, &far, &key));
let near: Vec<NodeId> = (1..=PODCAST_STORE_RESPONSIBILITY_WINDOW)
.map(|index| {
let mut bytes = [0; 32];
bytes[31] = index as u8;
NodeId::from_bytes(bytes)
})
.collect();
let own = NodeId::from_bytes([0xff; 32]);
assert!(!is_responsible(&own, &near, &key));
assert!(is_responsible(&own, &near[1..], &key));
}
#[test]
fn inbound_requests_are_limited_per_peer() {
let inflight = Arc::new(Mutex::new(HashMap::new()));
let peer = secret(1).public();
let slots: Vec<_> = (0..MAX_PODCAST_DHT_REQUESTS_PER_PEER)
.map(|_| PeerSlot::acquire(&inflight, peer).unwrap())
.collect();
assert!(PeerSlot::acquire(&inflight, peer).is_none());
assert!(PeerSlot::acquire(&inflight, secret(2).public()).is_some());
drop(slots);
assert!(PeerSlot::acquire(&inflight, peer).is_some());
assert!(lock(&inflight).is_empty());
}
#[test]
fn lookups_keep_a_bounded_number_of_owners() {
let mut winners = HashMap::new();
let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id());
let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS)
.map(|index| {
card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(index),
1,
now_ms(),
)
})
.collect();
let first = records[0].clone();
merge_records(&mut winners, records, &network(), &id_key, &[]);
assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS);
// A newer record of a known owner still replaces its winner.
let newer = card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(0),
2,
now_ms(),
);
merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]);
let owner = (first.podcast_id().clone(), first.publisher());
assert_eq!(winners[&owner], newer);
}
#[test]
@@ -763,18 +1053,27 @@ fn music(name: &str, kind: ItemKind) -> LibraryItem {
}
}
fn moved_spec(path: &str, origin: &str, title: &str) -> PodcastCardSpec {
let mut moved = spec(path, title);
moved.id_feed_url = Some(feed(origin));
moved.podcast_id = PodcastId::from_feed_url(&feed(origin));
moved
}
#[test]
fn hits_merge_publishers_and_drop_tombstoned_ones() {
let a = secret(1);
let b = secret(2);
let c = secret(3);
let mut moved = spec("moved.xml", "Rust Radio");
moved.id_feed_url = Some(feed("show.xml"));
moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml"));
let records = vec![
card(&a, 1),
card(&a, 2),
card_with(moved, &b, 1, NOW + 10),
card_with(
moved_spec("moved.xml", "show.xml", "Rust Radio"),
&b,
1,
NOW + 10,
),
card(&c, 1),
tombstone(&c, 2, NOW),
card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW),
@@ -790,8 +1089,81 @@ fn hits_merge_publishers_and_drop_tombstoned_ones() {
let mut expected = vec![a.public(), b.public()];
expected.sort_by_key(|key| *key.as_bytes());
assert_eq!(hit.publishers, expected);
// Equal votes per feed URL: the most recently issued card represents.
assert_eq!(hit.card.publisher, b.public());
// The card under the origin URL represents, whoever issued later.
assert_eq!(hit.card.publisher, a.public());
assert_eq!(hit.feed_url, feed("show.xml"));
assert_eq!(
hit.claimed_feed_urls,
vec![ClaimedFeedUrl {
feed_url: feed("moved.xml"),
publishers: vec![b.public()],
}]
);
}
#[test]
fn sybil_cards_cannot_redirect_an_existing_podcast() {
let genuine = card(&secret(1), 1);
let id = genuine.podcast_id().clone();
let evil_feed = FeedUrl::parse("https://evil.example/feed.xml").unwrap();
let hijack = |seed: usize| {
let mut spec = spec("show.xml", "Rust Radio (official)");
spec.id_feed_url = Some(spec.feed_url.clone());
spec.feed_url = evil_feed.clone();
spec.validate().unwrap();
assert_eq!(spec.podcast_id, id, "a hijack card claims the genuine id");
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
card_with(spec, &numbered_secret(seed), u64::MAX, NOW + skew)
};
let mut records = vec![genuine.clone()];
records.extend((0..10).map(hijack));
let hit = aggregate_hits(records).remove(0);
assert_eq!(
hit.feed_url,
feed("show.xml"),
"only the origin URL is offered"
);
assert_eq!(
hit.card,
*genuine.card().unwrap(),
"the origin card represents"
);
assert_eq!(hit.claimed_feed_urls.len(), 1);
assert_eq!(hit.claimed_feed_urls[0].feed_url, evil_feed);
assert_eq!(hit.claimed_feed_urls[0].publishers.len(), 10);
// Without any origin card the hit still offers the origin URL, never the
// claimed one, and ranks below podcasts that have an origin card.
let only_claims = aggregate_hits((0..3).map(hijack)).remove(0);
assert_eq!(only_claims.feed_url, feed("show.xml"));
assert!(!only_claims.card.feed_url_is_origin());
let honest = aggregate_hits(vec![card_with(
spec("b.xml", "Rust Radio Weekly"),
&secret(2),
1,
NOW,
)])
.remove(0);
let ranked = rank_hits(
vec![only_claims.clone(), honest.clone()],
"rust",
&podcast_query_tokens("rust"),
);
assert_eq!(ranked[0].podcast_id, honest.podcast_id);
// Publisher counts do not rank.
let crowded = aggregate_hits(
(20..60).map(|seed| card_with(spec("c.xml", "Rust Zone"), &numbered_secret(seed), 1, NOW)),
)
.remove(0);
let ranked = rank_hits(
vec![crowded, honest.clone()],
"rust",
&podcast_query_tokens("rust"),
);
assert_eq!(
ranked[0].podcast_id, honest.podcast_id,
"ranked by title, not crowd"
);
}
#[test]
+164 -17
View File
@@ -1,15 +1,23 @@
//! Integration test: public podcast discovery on three real loopback iroh
//! nodes.
//!
//! * A publishes a podcast its user shared; B is one of A's trusted devices
//! (in this test merely another overlay node, then offline); C is an
//! unrelated peer outside A's trusted-device group, with no device-sync
//! pairing.
//! * C finds the card by text and by id; a podcast A never shared is never
//! discoverable; after A unpublishes, the tombstone hides the card from C.
//! * Forged, tampered, replayed, malformed, oversized and wrong-version
//! requests are rejected by a real peer.
//! * D, a music-DHT peer without the overlay, is never added to the overlay.
//! * A publishes a podcast its user shared. B is another device of the same
//! user: it applies the same merged registers (as device sync would deliver
//! them) and publishes the same podcast under its own key. C is an
//! unrelated peer outside the user's trusted-device group, with no
//! device-sync pairing.
//! * C finds the podcast by text and by id, merged over publishers {A, B};
//! once B unshares and goes offline, C still finds A's card, so B is
//! irrelevant to discovery. A podcast the user never shared (held by both
//! A and B) is never discoverable; after A unpublishes, the tombstone hides
//! the card from C.
//! * Forged, foreign, tampered, replayed, malformed, oversized and
//! wrong-version requests are rejected by a real peer.
//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but
//! never answers on the overlay ALPN, so it never becomes an overlay contact
//! and is never asked for records or to store them.
//! * `sync_shares` stays fail-safe for revocation and holds back URLs that
//! look private (single node).
use std::path::Path;
use std::sync::Arc;
@@ -214,14 +222,15 @@ async fn scenario() {
(a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(())
})
.await;
// D never answers on the overlay ALPN, so it never becomes an overlay
// contact and is never queried or asked to store records.
// D is dialed by bootstrap but never answers on the overlay ALPN, so it
// never becomes an overlay contact: it is never asked for records or to
// store them.
a.dht.bootstrap().await.unwrap();
assert_eq!(a.dht.known_peers(), 2);
// A capability manifest without podcast_dht evicts a peer; one with it
// lets the peer be learned again.
let manifest = CapabilityManifest::frid("furumi", "test");
let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht();
a.dht
.note_peer_capabilities(
c.service.endpoint_id(),
@@ -260,11 +269,13 @@ async fn scenario() {
subscription: shared.clone(),
share: Some(share(&shared, true, 2)),
card: None,
privacy_confirmed: false,
},
PodcastShareState {
subscription: private.clone(),
share: None,
card: None,
privacy_confirmed: false,
},
];
let stats = a.dht.sync_shares(states.clone()).await.unwrap();
@@ -294,7 +305,54 @@ async fn scenario() {
assert_eq!(published.len(), 1);
let first_card = published[0].record.clone();
// B (A's trusted device) is not needed for discovery.
// B, the same user's other device, holds the same merged registers and
// publishes the same podcast under its own key; the private podcast
// stays private there too.
let b_states = vec![
PodcastShareState {
subscription: shared.clone(),
share: Some(share(&shared, true, 2)),
card: None,
privacy_confirmed: false,
},
PodcastShareState {
subscription: private.clone(),
share: None,
card: None,
privacy_confirmed: false,
},
];
let stats = b.dht.sync_shares(b_states).await.unwrap();
assert_eq!((stats.published, stats.unpublished), (1, 0));
let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()];
both.sort_by_key(|key| *key.as_bytes());
let hits = wait_for("C sees both publishers", async || {
let hits = search(&c, "rust radio").await;
(hits.len() == 1 && hits[0].publishers == both).then_some(hits)
})
.await;
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
assert!(hits[0].claimed_feed_urls.is_empty());
// B unshares (for example a stale register on B) and goes offline: B is
// not needed for discovery, A's card is still found.
let stats = b
.dht
.sync_shares(vec![PodcastShareState {
subscription: shared.clone(),
share: Some(share(&shared, false, 3)),
card: None,
privacy_confirmed: false,
}])
.await
.unwrap();
assert_eq!(stats.unpublished, 1);
let a_only = vec![a.service.endpoint_id()];
wait_for("B's tombstone reaches C's lookup", async || {
let hits = search(&c, "rust radio").await;
(hits.len() == 1 && hits[0].publishers == a_only).then_some(())
})
.await;
b.stop().await;
let hits = wait_for("C finds the card by text", async || {
@@ -305,6 +363,7 @@ async fn scenario() {
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].podcast_id, shared.podcast_id);
assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]);
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url);
assert_eq!(hits[0].card.title, "Rust Radio Weekly");
assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens");
@@ -350,7 +409,11 @@ async fn scenario() {
let response = store_request(&c, &a, forged).await;
assert!(!response.ok && response.stored == 0, "{response:?}");
// Tampered: A's genuine card with a different feed URL.
// Foreign: A's genuine card stored by C. Only publishers store records.
let response = store_request(&c, &a, first_card.clone()).await;
assert!(!response.ok && response.stored == 0, "{response:?}");
// Tampered: A's genuine card with a different description.
let mut tampered = first_card.clone();
if let PodcastRecordBody::Card(card) = &mut tampered.body {
card.description = Some("Now with malware".into());
@@ -388,6 +451,7 @@ async fn scenario() {
subscription: shared.clone(),
share: Some(share(&shared, false, 3)),
card: None,
privacy_confirmed: false,
}];
let stats = a.dht.sync_shares(unshare).await.unwrap();
assert_eq!(stats.unpublished, 1);
@@ -407,10 +471,12 @@ async fn scenario() {
.is_none()
);
// Replaying the old card, even to the node that stores the tombstone,
// does not bring it back.
let response = store_request(&c, &a, first_card).await;
// A replaying its own old card to a replica that stores the tombstone
// does not bring it back; nobody else may replay it at all.
let response = store_request(&a, &c, first_card.clone()).await;
assert!(response.ok && response.stored == 0, "{response:?}");
let response = store_request(&c, &a, first_card).await;
assert!(!response.ok, "{response:?}");
assert!(search(&c, "rust radio").await.is_empty());
assert!(search(&a, "rust radio").await.is_empty());
@@ -419,3 +485,84 @@ async fn scenario() {
d_events.abort();
d.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn sync_shares_is_fail_safe_and_holds_back_private_urls() {
tokio::time::timeout(TEST_TIMEOUT, single_node_scenario())
.await
.unwrap();
}
async fn single_node_scenario() {
let temp = tempfile::tempdir().unwrap();
let network = NetworkId::from_name("podcast-dht-sync-shares");
let a = Node::start(&temp.path().join("a"), network).await;
let state =
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
subscription: subscription.clone(),
share: Some(share(subscription, shared, generation)),
card: None,
privacy_confirmed: false,
};
let first = subscription("first.xml", "First Show");
let second = subscription("second.xml", "Second Show");
let stats = a
.dht
.sync_shares(vec![state(&first, true, 2), state(&second, true, 2)])
.await
.unwrap();
assert_eq!(stats.published, 2);
assert!(stats.failed.is_empty() && stats.held_back.is_empty());
// In one call the user unshares the first podcast while the second
// one's card cannot be built (the URL its id derives from is unknown).
let mut broken = second.clone();
broken.podcast.feed_url = feed("elsewhere.xml");
let stats = a
.dht
.sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)])
.await
.unwrap();
assert_eq!(stats.failed.len(), 1);
assert_eq!(stats.failed[0].podcast_id, second.podcast_id);
assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast");
let published = a.dht.published().await.unwrap();
assert!(published.iter().all(|publication| !publication.active));
assert!(
published
.iter()
.all(|publication| publication.record.is_tombstone())
);
// Republishing re-sends tombstones only; nothing comes back.
a.dht.republish().await.unwrap();
assert!(search(&a, "first show").await.is_empty());
assert!(search(&a, "second show").await.is_empty());
// A feed URL that looks private is held back until the user confirms.
let mut member = subscription("member.xml?token=abc", "Member Show");
member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url);
let stats = a
.dht
.sync_shares(vec![state(&member, true, 2)])
.await
.unwrap();
assert_eq!(stats.published, 0);
assert_eq!(stats.held_back.len(), 1);
assert_eq!(stats.held_back[0].podcast_id, member.podcast_id);
assert!(!stats.held_back[0].warnings.is_empty());
assert!(search(&a, "member show").await.is_empty());
let mut confirmed = state(&member, true, 2);
confirmed.privacy_confirmed = true;
let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap();
assert_eq!(stats.published, 1);
assert_eq!(search(&a, "member show").await.len(), 1);
// Once published with the user's consent, the same URLs are kept.
let stats = a
.dht
.sync_shares(vec![state(&member, true, 2)])
.await
.unwrap();
assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0));
a.stop().await;
}