Harden podcast discovery against sybil and hijack records
Replicas now accept records only from their publisher, keep one payload per (podcast, publisher), store placements only for keys they are close to, and admit records within per-key, per-tombstone, per-publisher and total budgets without ever evicting existing records. Tombstones cannot claim text keys without their owner's card. Search offers only the feed URL the podcast id hashes from and reports moved-feed URLs as claims; publisher counts no longer rank. sync_shares handles each podcast on its own, always revokes, and holds back cards with privacy warnings. Lookups and inbound requests per peer are bounded, persisted contacts are capped, and the podcast_dht capability is opt-in. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fef155d6ef
commit
81ccd924d0
7 files changed
+1276
-281
No files matched your search
@@ -56,7 +56,13 @@ pub struct CapabilityManifest {
|
||||
}
|
||||
|
||||
impl CapabilityManifest {
|
||||
/// Creates a manifest containing every protocol owned by Frid.
|
||||
/// Creates a manifest containing every protocol owned by Frid that a
|
||||
/// music-DHT application serves.
|
||||
///
|
||||
/// The public podcast-discovery overlay is opt-in: applications that run
|
||||
/// [`crate::podcast_dht::PodcastDht`] add it with
|
||||
/// [`Self::with_podcast_dht`], so an application that does not serve the
|
||||
/// overlay never advertises it.
|
||||
pub fn frid(application: impl Into<String>, application_version: impl Into<String>) -> Self {
|
||||
let mut protocols = BTreeMap::new();
|
||||
protocols.insert(
|
||||
@@ -81,10 +87,6 @@ impl CapabilityManifest {
|
||||
SIMILARITY_DHT_ID.to_string(),
|
||||
crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION,
|
||||
);
|
||||
protocols.insert(
|
||||
PODCAST_DHT_ID.to_string(),
|
||||
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
||||
);
|
||||
protocols.insert(
|
||||
DEVICE_SYNC_ID.to_string(),
|
||||
crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION,
|
||||
@@ -111,6 +113,17 @@ impl CapabilityManifest {
|
||||
self
|
||||
}
|
||||
|
||||
/// Advertises the public podcast-discovery overlay
|
||||
/// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
/// Call it only when the application serves
|
||||
/// [`crate::podcast_dht::PODCAST_DHT_ALPN`].
|
||||
pub fn with_podcast_dht(self) -> Self {
|
||||
self.with_protocol(
|
||||
PODCAST_DHT_ID,
|
||||
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
||||
)
|
||||
}
|
||||
|
||||
/// Version of protocol `id` advertised by the peer, if any.
|
||||
pub fn protocol_version(&self, id: &str) -> Option<u16> {
|
||||
self.protocols.get(id).copied()
|
||||
@@ -237,7 +250,6 @@ mod tests {
|
||||
CATALOG_ID,
|
||||
SIMILARITY_ID,
|
||||
SIMILARITY_DHT_ID,
|
||||
PODCAST_DHT_ID,
|
||||
DEVICE_SYNC_ID,
|
||||
JAM_ID,
|
||||
] {
|
||||
@@ -247,13 +259,17 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_dht_capability_is_advertised_and_removable() {
|
||||
let manifest = CapabilityManifest::frid("furumi", "1.2.3");
|
||||
fn podcast_dht_capability_is_opt_in_and_removable() {
|
||||
let plain = CapabilityManifest::frid("furumi", "1.2.3");
|
||||
assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None);
|
||||
assert!(!plain.supports_podcast_dht());
|
||||
let manifest = plain.with_podcast_dht();
|
||||
assert_eq!(
|
||||
manifest.protocol_version(PODCAST_DHT_ID),
|
||||
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
);
|
||||
assert!(manifest.supports_podcast_dht());
|
||||
manifest.validate().unwrap();
|
||||
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
|
||||
assert!(!without.supports_podcast_dht());
|
||||
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
|
||||
|
||||
@@ -396,6 +396,19 @@ impl PodcastCard {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
|
||||
/// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url`
|
||||
/// when the feed has not moved). A valid card's id hashes from it, so it
|
||||
/// is the only URL of the card bound to the podcast.
|
||||
pub fn origin_feed_url(&self) -> &FeedUrl {
|
||||
self.id_feed_url.as_ref().unwrap_or(&self.feed_url)
|
||||
}
|
||||
|
||||
/// Whether `feed_url` is the URL the id is derived from. Otherwise
|
||||
/// `feed_url` is an unverified claim of the publisher that the feed moved.
|
||||
pub fn feed_url_is_origin(&self) -> bool {
|
||||
self.id_feed_url.is_none()
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
|
||||
@@ -735,18 +748,6 @@ pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord)
|
||||
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
|
||||
}
|
||||
|
||||
/// Whether `winner` makes `other` obsolete everywhere, not just under one key:
|
||||
/// `other` is a card of a lower revision, or `winner` is a tombstone of an
|
||||
/// equal or higher revision.
|
||||
pub(crate) fn supersedes_everywhere(
|
||||
winner: &SignedPodcastRecord,
|
||||
other: &SignedPodcastRecord,
|
||||
) -> bool {
|
||||
!other.is_tombstone()
|
||||
&& (other.revision() < winner.revision()
|
||||
|| (winner.is_tombstone() && other.revision() <= winner.revision()))
|
||||
}
|
||||
|
||||
struct Canonical(Vec<u8>);
|
||||
|
||||
impl Canonical {
|
||||
|
||||
@@ -27,6 +27,13 @@
|
||||
//! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with
|
||||
//! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take
|
||||
//! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`].
|
||||
//! Advertise the overlay with [`CapabilityManifest::with_podcast_dht`] (it is
|
||||
//! opt-in) and pass peer manifests to [`PodcastDht::note_peer_capabilities`].
|
||||
//!
|
||||
//! Only the URL a podcast id hashes from is bound to the podcast; search hits
|
||||
//! offer that URL ([`PodcastSearchHit::feed_url`]) and report moved-feed URLs
|
||||
//! as unverified claims. The publisher key is the device's transport identity
|
||||
//! and therefore linkable to its network address and music-DHT records.
|
||||
|
||||
mod card;
|
||||
mod privacy;
|
||||
@@ -36,7 +43,7 @@ mod store;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
@@ -69,10 +76,16 @@ pub use privacy::{
|
||||
FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning,
|
||||
};
|
||||
pub use search::{
|
||||
FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit,
|
||||
SearchHitKind, merge_search_results, podcast_query_tokens,
|
||||
ClaimedFeedUrl, FederatedSearchHit, MAX_HIT_FEED_URL_CLAIMS, MAX_HIT_PUBLISHERS,
|
||||
MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, SearchHitKind, merge_search_results,
|
||||
podcast_query_tokens,
|
||||
};
|
||||
pub use store::{
|
||||
MAX_PERSISTED_PODCAST_DHT_PEERS, MAX_PODCAST_CACHE_BYTES, MAX_PODCAST_CACHE_RECORDS,
|
||||
MAX_PODCAST_RECORDS_PER_KEY, MAX_PODCAST_RECORDS_PER_PUBLISHER,
|
||||
MAX_PODCAST_RESPONSE_TOMBSTONES, MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||
PODCAST_DHT_CACHE_SCHEMA_VERSION,
|
||||
};
|
||||
pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION};
|
||||
|
||||
use card::publication_error;
|
||||
use search::{aggregate_hits, rank_hits};
|
||||
@@ -105,6 +118,16 @@ pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60);
|
||||
/// Maximum inbound requests processed concurrently; further streams are
|
||||
/// dropped.
|
||||
pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64;
|
||||
/// Maximum inbound requests of one peer processed concurrently; further
|
||||
/// streams of that peer are dropped.
|
||||
pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8;
|
||||
/// Maximum distinct `(podcast_id, publisher)` records one lookup keeps;
|
||||
/// records of further owners are ignored.
|
||||
pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512;
|
||||
/// A node stores a placement only when fewer than this many of its known
|
||||
/// overlay contacts are closer to the key than itself (`2 * K`: the `K`
|
||||
/// closest with slack for differing routing views).
|
||||
pub const PODCAST_STORE_RESPONSIBILITY_WINDOW: usize = 2 * K;
|
||||
/// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is
|
||||
/// not contacted.
|
||||
pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60);
|
||||
@@ -228,7 +251,7 @@ pub struct PodcastDhtPublishStats {
|
||||
}
|
||||
|
||||
/// Result of [`PodcastDht::sync_shares`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct PodcastDhtSyncStats {
|
||||
/// Cards published with a new revision, or refreshed because they used
|
||||
/// half their lifetime.
|
||||
@@ -237,10 +260,38 @@ pub struct PodcastDhtSyncStats {
|
||||
pub unchanged: usize,
|
||||
/// Podcasts unpublished with a tombstone.
|
||||
pub unpublished: usize,
|
||||
/// Shared podcasts not published because their card has privacy
|
||||
/// warnings the user has not confirmed. The UI asks, then calls
|
||||
/// [`PodcastDht::publish`] or `sync_shares` with
|
||||
/// [`PodcastShareState::privacy_confirmed`]. An earlier publication of
|
||||
/// the podcast stays as it was.
|
||||
pub held_back: Vec<PodcastPrivacyHold>,
|
||||
/// Podcasts whose card could not be built or signed. They are not
|
||||
/// published, and an earlier publication of them is unpublished.
|
||||
pub failed: Vec<PodcastSyncFailure>,
|
||||
/// Network statistics of the pushed records.
|
||||
pub push: PodcastDhtPublishStats,
|
||||
}
|
||||
|
||||
/// A shared podcast held back by [`PodcastDht::sync_shares`] until the user
|
||||
/// confirms publishing URLs that look private.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PodcastPrivacyHold {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Warnings of [`PodcastCardSpec::privacy_warnings`].
|
||||
pub warnings: Vec<FeedUrlPrivacyWarning>,
|
||||
}
|
||||
|
||||
/// A podcast [`PodcastDht::sync_shares`] could not publish or unpublish.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PodcastSyncFailure {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Diagnostic.
|
||||
pub error: String,
|
||||
}
|
||||
|
||||
/// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PodcastShareState {
|
||||
@@ -251,6 +302,11 @@ pub struct PodcastShareState {
|
||||
/// Card to publish when shared; `None` derives it with
|
||||
/// [`PodcastCardSpec::from_subscription`].
|
||||
pub card: Option<PodcastCardSpec>,
|
||||
/// Whether the user confirmed publishing this card's feed URLs despite
|
||||
/// its [`PodcastCardSpec::privacy_warnings`]. Without it, a card with
|
||||
/// warnings is only published when this device already publishes the
|
||||
/// same feed URLs.
|
||||
pub privacy_confirmed: bool,
|
||||
}
|
||||
|
||||
impl PodcastShareState {
|
||||
@@ -258,6 +314,22 @@ impl PodcastShareState {
|
||||
pub fn is_shared(&self) -> bool {
|
||||
effective_share(Some(&self.subscription), self.share.as_ref())
|
||||
}
|
||||
|
||||
/// The bounded, validated card [`PodcastDht::sync_shares`] would
|
||||
/// publish: [`Self::card`] or one derived from the subscription.
|
||||
pub fn card_spec(&self) -> Result<PodcastCardSpec> {
|
||||
let card = match &self.card {
|
||||
Some(card) => card.bounded(),
|
||||
None => PodcastCardSpec::from_subscription(&self.subscription)?,
|
||||
};
|
||||
if card.podcast_id != self.subscription.podcast_id {
|
||||
return Err(publication_error(
|
||||
"card and subscription describe different podcasts",
|
||||
));
|
||||
}
|
||||
card.validate()?;
|
||||
Ok(card)
|
||||
}
|
||||
}
|
||||
|
||||
/// This device's publication state of one podcast.
|
||||
@@ -284,6 +356,7 @@ pub struct PodcastDht {
|
||||
failed: Mutex<HashMap<EndpointId, u64>>,
|
||||
publish_gate: Semaphore,
|
||||
serve_limit: Arc<Semaphore>,
|
||||
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PodcastDht {
|
||||
@@ -330,6 +403,7 @@ impl PodcastDht {
|
||||
failed: Mutex::new(HashMap::new()),
|
||||
publish_gate: Semaphore::new(1),
|
||||
serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)),
|
||||
inflight: Arc::new(Mutex::new(HashMap::new())),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -341,6 +415,10 @@ impl PodcastDht {
|
||||
/// Serves authenticated overlay requests until `acceptor` closes.
|
||||
pub async fn serve(self: Arc<Self>, mut acceptor: StreamAcceptor) {
|
||||
while let Some(stream) = acceptor.accept().await {
|
||||
let Some(slot) = PeerSlot::acquire(&self.inflight, stream.peer_id) else {
|
||||
debug!(peer = %stream.peer_id, "podcast DHT request dropped: peer limit");
|
||||
continue;
|
||||
};
|
||||
let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else {
|
||||
debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded");
|
||||
continue;
|
||||
@@ -348,6 +426,7 @@ impl PodcastDht {
|
||||
let this = Arc::clone(&self);
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
let _slot = slot;
|
||||
let peer = stream.peer_id;
|
||||
match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await {
|
||||
Ok(Ok(())) => {}
|
||||
@@ -421,7 +500,10 @@ impl PodcastDht {
|
||||
}
|
||||
|
||||
/// Discovers overlay peers among the routing table and the music DHT
|
||||
/// contacts.
|
||||
/// contacts with one `FindNode` each. Music-DHT contacts whose capability
|
||||
/// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are
|
||||
/// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become
|
||||
/// overlay contacts when they answer.
|
||||
pub async fn bootstrap(&self) -> Result<()> {
|
||||
let known_feature: HashSet<EndpointId> = lock(&self.routing)
|
||||
.contacts()
|
||||
@@ -521,44 +603,92 @@ impl PodcastDht {
|
||||
/// [`Self::republish`]). Every other podcast this device published,
|
||||
/// including podcasts absent from `states`, is unpublished. The call is
|
||||
/// idempotent: repeating it publishes nothing new.
|
||||
///
|
||||
/// Each podcast is handled on its own, so revocation is fail-safe: a
|
||||
/// shared podcast whose card cannot be built is reported in
|
||||
/// [`PodcastDhtSyncStats::failed`] and unpublished, and never prevents
|
||||
/// other podcasts from being unpublished. A card with
|
||||
/// [`PodcastCardSpec::privacy_warnings`] is held back
|
||||
/// ([`PodcastDhtSyncStats::held_back`]) unless
|
||||
/// [`PodcastShareState::privacy_confirmed`] is set or this device already
|
||||
/// publishes the same feed URLs.
|
||||
pub async fn sync_shares(&self, states: Vec<PodcastShareState>) -> Result<PodcastDhtSyncStats> {
|
||||
let mut desired: HashMap<PodcastId, PodcastCardSpec> = HashMap::new();
|
||||
for state in states {
|
||||
let _permit = self.publish_permit().await?;
|
||||
let local = self.database.local_publications().await?;
|
||||
let active: HashMap<&PodcastId, &PodcastCardSpec> = local
|
||||
.iter()
|
||||
.filter_map(|publication| {
|
||||
publication
|
||||
.spec
|
||||
.as_ref()
|
||||
.map(|spec| (&publication.podcast_id, spec))
|
||||
})
|
||||
.collect();
|
||||
let mut stats = PodcastDhtSyncStats::default();
|
||||
let mut desired: BTreeMap<PodcastId, PodcastCardSpec> = BTreeMap::new();
|
||||
let mut held: HashSet<PodcastId> = HashSet::new();
|
||||
for state in &states {
|
||||
if !state.is_shared() {
|
||||
continue;
|
||||
}
|
||||
let card = match state.card {
|
||||
Some(card) => card.bounded(),
|
||||
None => PodcastCardSpec::from_subscription(&state.subscription)?,
|
||||
let podcast_id = state.subscription.podcast_id.clone();
|
||||
let card = match state.card_spec() {
|
||||
Ok(card) => card,
|
||||
Err(error) => {
|
||||
debug!(%podcast_id, %error, "podcast card cannot be published");
|
||||
stats.failed.push(PodcastSyncFailure {
|
||||
podcast_id,
|
||||
error: error.to_string(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if card.podcast_id != state.subscription.podcast_id {
|
||||
return Err(publication_error(
|
||||
"card and subscription describe different podcasts",
|
||||
));
|
||||
let warnings = card.privacy_warnings();
|
||||
let already_published = active.get(&podcast_id).is_some_and(|published| {
|
||||
published.feed_url == card.feed_url && published.id_feed_url == card.id_feed_url
|
||||
});
|
||||
if !warnings.is_empty() && !state.privacy_confirmed && !already_published {
|
||||
held.insert(podcast_id.clone());
|
||||
stats.held_back.push(PodcastPrivacyHold {
|
||||
podcast_id,
|
||||
warnings,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
card.validate()?;
|
||||
desired.insert(card.podcast_id.clone(), card);
|
||||
desired.insert(podcast_id, card);
|
||||
}
|
||||
|
||||
let _permit = self.publish_permit().await?;
|
||||
let local = self.database.local_publications().await?;
|
||||
let mut stats = PodcastDhtSyncStats::default();
|
||||
let mut records = Vec::new();
|
||||
for publication in &local {
|
||||
if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) {
|
||||
records.extend(self.unpublish_locked(&publication.podcast_id).await?);
|
||||
stats.unpublished += 1;
|
||||
let podcast_id = &publication.podcast_id;
|
||||
if publication.spec.is_none()
|
||||
|| desired.contains_key(podcast_id)
|
||||
|| held.contains(podcast_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
match self.unpublish_locked(podcast_id).await {
|
||||
Ok(record) => {
|
||||
records.extend(record);
|
||||
stats.unpublished += 1;
|
||||
}
|
||||
Err(error) => stats.failed.push(PodcastSyncFailure {
|
||||
podcast_id: podcast_id.clone(),
|
||||
error: error.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
let mut desired: Vec<_> = desired.into_values().collect();
|
||||
desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id));
|
||||
for card in desired {
|
||||
match self.sign_card_locked(card, false).await? {
|
||||
Some(record) => {
|
||||
for (podcast_id, card) in desired {
|
||||
match self.sign_card_locked(card, false).await {
|
||||
Ok(Some(record)) => {
|
||||
records.push(record);
|
||||
stats.published += 1;
|
||||
}
|
||||
None => stats.unchanged += 1,
|
||||
Ok(None) => stats.unchanged += 1,
|
||||
Err(error) => stats.failed.push(PodcastSyncFailure {
|
||||
podcast_id,
|
||||
error: error.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
stats.push = self.push(records).await?;
|
||||
@@ -613,8 +743,8 @@ impl PodcastDht {
|
||||
/// normalized token of `query` ([`podcast_query_tokens`]).
|
||||
///
|
||||
/// Hits are merged by `podcast_id` over publishers and ranked: exact
|
||||
/// normalized title first, then more publishers. Partial results are
|
||||
/// returned when peers fail.
|
||||
/// normalized title first, then hits with a card under the origin feed
|
||||
/// URL. Partial results are returned when peers fail.
|
||||
pub async fn search(&self, query: &str, limit: usize) -> Result<Vec<PodcastSearchHit>> {
|
||||
let tokens = podcast_query_tokens(query);
|
||||
if tokens.is_empty() || limit == 0 {
|
||||
@@ -966,8 +1096,25 @@ impl PodcastDht {
|
||||
))
|
||||
}
|
||||
PodcastDhtOperation::StoreBatch { records } => {
|
||||
let entries = validate_store_batch(records, &self.network_id, now_ms())?;
|
||||
let stored = self.database.store(entries, now_ms()).await?;
|
||||
let entries =
|
||||
validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?;
|
||||
let nodes: Vec<NodeId> = lock(&self.routing)
|
||||
.contacts()
|
||||
.into_iter()
|
||||
.map(|contact| contact.node_id)
|
||||
.collect();
|
||||
let entries: Vec<_> = entries
|
||||
.into_iter()
|
||||
.filter_map(|(mut keys, record)| {
|
||||
keys.retain(|key| is_responsible(&self.own_contact.node_id, &nodes, key));
|
||||
(!keys.is_empty()).then_some((keys, record))
|
||||
})
|
||||
.collect();
|
||||
let stored = if entries.is_empty() {
|
||||
0
|
||||
} else {
|
||||
self.database.store(entries, now_ms()).await?
|
||||
};
|
||||
Ok(PodcastDhtResponse::success(
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
@@ -1099,11 +1246,17 @@ impl PodcastDht {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates a store batch received from a peer: bounds, every record's
|
||||
/// signature and lifetime, and that every key belongs to its record.
|
||||
/// Validates a store batch received from `requester`: bounds, that every
|
||||
/// record is published by the requester itself, every record's signature and
|
||||
/// lifetime, and that every key belongs to its record.
|
||||
///
|
||||
/// Replicas never forward records, so only a record's publisher stores it.
|
||||
/// Nobody can therefore replay or inject another device's records, and
|
||||
/// per-publisher quotas apply to the storing peer.
|
||||
fn validate_store_batch(
|
||||
entries: Vec<PodcastStoreEntry>,
|
||||
network_id: &NetworkId,
|
||||
requester: EndpointId,
|
||||
now_ms: u64,
|
||||
) -> Result<Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>> {
|
||||
if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS {
|
||||
@@ -1117,6 +1270,11 @@ fn validate_store_batch(
|
||||
if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() {
|
||||
return Err(protocol_error("invalid podcast DHT store keys"));
|
||||
}
|
||||
if entry.record.publisher() != requester {
|
||||
return Err(protocol_error(
|
||||
"podcast records must be stored by their publisher",
|
||||
));
|
||||
}
|
||||
entry.record.verify_at(network_id, now_ms)?;
|
||||
let valid_keys = entry.record.dht_keys();
|
||||
if keys.iter().any(|key| !valid_keys.contains(key)) {
|
||||
@@ -1128,6 +1286,18 @@ fn validate_store_batch(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether a node is among the nodes that should store `key`: fewer than
|
||||
/// [`PODCAST_STORE_RESPONSIBILITY_WINDOW`] known contacts are closer.
|
||||
fn is_responsible(own: &NodeId, contacts: &[NodeId], key: &PodcastDhtKey) -> bool {
|
||||
let own_distance = distance(own.as_bytes(), key.as_bytes());
|
||||
contacts
|
||||
.iter()
|
||||
.filter(|node| distance(node.as_bytes(), key.as_bytes()) < own_distance)
|
||||
.take(PODCAST_STORE_RESPONSIBILITY_WINDOW)
|
||||
.count()
|
||||
< PODCAST_STORE_RESPONSIBILITY_WINDOW
|
||||
}
|
||||
|
||||
fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
|
||||
if tokens.len() > MAX_PODCAST_QUERY_TOKENS
|
||||
|| tokens
|
||||
@@ -1139,7 +1309,8 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adds verified records found under `key` to the per-owner winners.
|
||||
/// Adds verified records found under `key` to the per-owner winners, keeping
|
||||
/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners.
|
||||
fn merge_records(
|
||||
winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>,
|
||||
records: Vec<SignedPodcastRecord>,
|
||||
@@ -1161,6 +1332,7 @@ fn merge_records(
|
||||
let owner = (record.podcast_id().clone(), record.publisher());
|
||||
match winners.get(&owner) {
|
||||
Some(current) if compare_records(current, &record).is_ge() => {}
|
||||
None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {}
|
||||
_ => {
|
||||
winners.insert(owner, record);
|
||||
}
|
||||
@@ -1168,6 +1340,42 @@ fn merge_records(
|
||||
}
|
||||
}
|
||||
|
||||
/// One in-flight inbound request of a peer; released on drop.
|
||||
struct PeerSlot {
|
||||
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
|
||||
peer: EndpointId,
|
||||
}
|
||||
|
||||
impl PeerSlot {
|
||||
fn acquire(
|
||||
inflight: &Arc<Mutex<HashMap<EndpointId, usize>>>,
|
||||
peer: EndpointId,
|
||||
) -> Option<Self> {
|
||||
let mut map = lock(inflight);
|
||||
let count = map.entry(peer).or_default();
|
||||
if *count >= MAX_PODCAST_DHT_REQUESTS_PER_PEER {
|
||||
return None;
|
||||
}
|
||||
*count += 1;
|
||||
Some(Self {
|
||||
inflight: Arc::clone(inflight),
|
||||
peer,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PeerSlot {
|
||||
fn drop(&mut self) {
|
||||
let mut map = lock(&self.inflight);
|
||||
if let Some(count) = map.get_mut(&self.peer) {
|
||||
*count = count.saturating_sub(1);
|
||||
if *count == 0 {
|
||||
map.remove(&self.peer);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_contact(
|
||||
contact: &NodeContact,
|
||||
network_id: &NetworkId,
|
||||
|
||||
@@ -8,15 +8,17 @@ use federation_net::EndpointId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::PodcastId;
|
||||
use crate::podcast::{FeedUrl, PodcastId};
|
||||
use crate::record::{ItemKind, LibraryItem};
|
||||
|
||||
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// Maximum distinct query tokens used by a podcast text search.
|
||||
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
|
||||
/// Maximum publishers reported per hit.
|
||||
/// Maximum publishers reported per hit and per claimed feed URL.
|
||||
pub const MAX_HIT_PUBLISHERS: usize = 64;
|
||||
/// Maximum claimed moved feed URLs reported per hit.
|
||||
pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8;
|
||||
|
||||
/// Kind of a federated search hit.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -64,18 +66,35 @@ impl From<ItemKind> for SearchHitKind {
|
||||
///
|
||||
/// Cards of the same podcast published by several devices are merged by
|
||||
/// `podcast_id`. The hit carries no personal state of any publisher.
|
||||
///
|
||||
/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every
|
||||
/// other field is a claim of some publisher device, and device identities are
|
||||
/// free, so neither the metadata nor the number of publishers is a trust
|
||||
/// signal.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastSearchHit {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Representative card: the most common `feed_url` among the
|
||||
/// publishers' current cards, then the most recently issued card, then
|
||||
/// the greatest publisher key.
|
||||
/// The URL `podcast_id` is derived from
|
||||
/// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with
|
||||
/// this URL; follow the feed's own HTTP redirects and
|
||||
/// `itunes:new-feed-url` from there.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Representative card for display: a card whose `feed_url` is the
|
||||
/// origin URL when one exists, then the title most publishers agree on,
|
||||
/// then the smallest publisher key. Its metadata is a publisher's claim.
|
||||
pub card: PodcastCard,
|
||||
/// Sorted distinct publisher devices with a current valid card, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
|
||||
/// one user may publish from several devices.
|
||||
/// one user may publish from several devices, and anyone can mint
|
||||
/// devices.
|
||||
pub publishers: Vec<EndpointId>,
|
||||
/// Feed URLs other than [`Self::feed_url`] that publishers claim the
|
||||
/// podcast moved to, most claimed first, at most
|
||||
/// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this
|
||||
/// podcast or record it as an alias unless the feed at
|
||||
/// [`Self::feed_url`] itself redirects there.
|
||||
pub claimed_feed_urls: Vec<ClaimedFeedUrl>,
|
||||
}
|
||||
|
||||
impl PodcastSearchHit {
|
||||
@@ -85,6 +104,16 @@ impl PodcastSearchHit {
|
||||
}
|
||||
}
|
||||
|
||||
/// An unverified moved-feed URL claimed by some publishers of a podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ClaimedFeedUrl {
|
||||
/// Claimed current feed URL.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Sorted distinct publisher devices claiming it, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`].
|
||||
pub publishers: Vec<EndpointId>,
|
||||
}
|
||||
|
||||
/// A music or podcast result of a federated search.
|
||||
///
|
||||
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
|
||||
@@ -184,37 +213,66 @@ pub(crate) fn aggregate_hits(
|
||||
by_podcast
|
||||
.into_iter()
|
||||
.map(|(podcast_id, cards)| {
|
||||
let mut feed_votes: HashMap<&str, usize> = HashMap::new();
|
||||
let mut title_votes: HashMap<String, usize> = HashMap::new();
|
||||
for card in &cards {
|
||||
*feed_votes.entry(card.feed_url.as_str()).or_default() += 1;
|
||||
*title_votes.entry(normalize_name(&card.title)).or_default() += 1;
|
||||
}
|
||||
// Publisher-controlled times and revisions never break ties.
|
||||
let card = cards
|
||||
.iter()
|
||||
.max_by_key(|card| {
|
||||
(
|
||||
feed_votes[card.feed_url.as_str()],
|
||||
card.issued_at_ms,
|
||||
*card.publisher.as_bytes(),
|
||||
card.feed_url_is_origin(),
|
||||
title_votes[&normalize_name(&card.title)],
|
||||
Reverse(*card.publisher.as_bytes()),
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.expect("a group has at least one card");
|
||||
let mut publishers: Vec<EndpointId> = cards.iter().map(|card| card.publisher).collect();
|
||||
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
||||
publishers.dedup();
|
||||
publishers.truncate(MAX_HIT_PUBLISHERS);
|
||||
let mut claims: BTreeMap<&FeedUrl, Vec<EndpointId>> = BTreeMap::new();
|
||||
for card in cards.iter().filter(|card| !card.feed_url_is_origin()) {
|
||||
claims
|
||||
.entry(&card.feed_url)
|
||||
.or_default()
|
||||
.push(card.publisher);
|
||||
}
|
||||
let mut claimed_feed_urls: Vec<ClaimedFeedUrl> = claims
|
||||
.into_iter()
|
||||
.map(|(feed_url, publishers)| ClaimedFeedUrl {
|
||||
feed_url: feed_url.clone(),
|
||||
publishers: sorted_publishers(publishers),
|
||||
})
|
||||
.collect();
|
||||
claimed_feed_urls.sort_by(|left, right| {
|
||||
right
|
||||
.publishers
|
||||
.len()
|
||||
.cmp(&left.publishers.len())
|
||||
.then_with(|| left.feed_url.cmp(&right.feed_url))
|
||||
});
|
||||
claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS);
|
||||
PodcastSearchHit {
|
||||
podcast_id,
|
||||
feed_url: card.origin_feed_url().clone(),
|
||||
publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()),
|
||||
card,
|
||||
publishers,
|
||||
claimed_feed_urls,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sorted_publishers(mut publishers: Vec<EndpointId>) -> Vec<EndpointId> {
|
||||
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
||||
publishers.dedup();
|
||||
publishers.truncate(MAX_HIT_PUBLISHERS);
|
||||
publishers
|
||||
}
|
||||
|
||||
/// Keeps hits whose card is indexed under every query token and ranks
|
||||
/// them: exact normalized title first, then more publishers, then title and
|
||||
/// id.
|
||||
/// them: exact normalized title first, then hits with a card published under
|
||||
/// the origin feed URL, then title and id. The publisher count is not used:
|
||||
/// anyone can inflate it with new device identities.
|
||||
pub(crate) fn rank_hits(
|
||||
mut hits: Vec<PodcastSearchHit>,
|
||||
query: &str,
|
||||
@@ -229,7 +287,7 @@ pub(crate) fn rank_hits(
|
||||
let title = normalize_name(&hit.card.title);
|
||||
(
|
||||
title != normalized_query,
|
||||
Reverse(hit.publishers.len()),
|
||||
!hit.card.feed_url_is_origin(),
|
||||
title,
|
||||
hit.podcast_id.clone(),
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! (like the similarity-routing cache). Every record read from it is decoded
|
||||
//! and verified again before it is used or served.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
@@ -17,32 +17,66 @@ use crate::error::{MusicDhtError, Result};
|
||||
use crate::podcast::PodcastId;
|
||||
use crate::routing::{NodeContact, NodeId};
|
||||
|
||||
use super::card::{
|
||||
PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere,
|
||||
};
|
||||
use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// `PRAGMA user_version` of the podcast-discovery cache schema.
|
||||
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1;
|
||||
/// Maximum distinct `(podcast_id, publisher)` records cached under one key.
|
||||
/// When full, the record expiring first is evicted if the new one outlives it.
|
||||
/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1
|
||||
/// (pre-release) stored one payload per key; its replica table is dropped on
|
||||
/// open because the cache is replaceable. Own publications are kept.
|
||||
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2;
|
||||
/// Maximum records (cards and tombstones) indexed under one key. A full key
|
||||
/// admits no new `(podcast_id, publisher)` until a record expires: existing
|
||||
/// records are never evicted for newcomers, and their owners keep refreshing
|
||||
/// them in place.
|
||||
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256;
|
||||
/// Maximum tombstones newly indexed under one key. A tombstone is admitted
|
||||
/// under a text key only in place of a record of the same
|
||||
/// `(podcast_id, publisher)` already indexed there, so this budget applies to
|
||||
/// the podcast-id key, where tombstones of unknown cards land.
|
||||
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32;
|
||||
/// Maximum tombstones in one `FindValue` response; cards fill the rest.
|
||||
pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32;
|
||||
/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a
|
||||
/// replica.
|
||||
pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256;
|
||||
/// Maximum `(podcast_id, publisher)` records in a replica cache.
|
||||
pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096;
|
||||
/// Maximum total encoded size of the records in a replica cache.
|
||||
pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// Maximum persisted overlay contacts; the least recently seen are dropped.
|
||||
pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024;
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_records (
|
||||
dht_key BLOB NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
payload BLOB NOT NULL,
|
||||
PRIMARY KEY (dht_key, podcast_id, publisher)
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
payload_bytes INTEGER NOT NULL,
|
||||
payload BLOB NOT NULL,
|
||||
PRIMARY KEY (podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
|
||||
ON podcast_dht_records(expires_at_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner
|
||||
ON podcast_dht_records(podcast_id, publisher);
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher
|
||||
ON podcast_dht_records(publisher);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_keys (
|
||||
dht_key BLOB NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
first_stored_at_ms INTEGER NOT NULL,
|
||||
PRIMARY KEY (dht_key, podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner
|
||||
ON podcast_dht_keys(podcast_id, publisher);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry
|
||||
ON podcast_dht_keys(expires_at_ms);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
|
||||
peer_id TEXT PRIMARY KEY,
|
||||
@@ -60,6 +94,28 @@ CREATE TABLE IF NOT EXISTS podcast_dht_local (
|
||||
);
|
||||
";
|
||||
|
||||
/// Admission limits of a replica cache.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct StoreLimits {
|
||||
pub per_key: usize,
|
||||
pub tombstones_per_key: usize,
|
||||
pub per_publisher: usize,
|
||||
pub records: usize,
|
||||
pub bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for StoreLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
per_key: MAX_PODCAST_RECORDS_PER_KEY,
|
||||
tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||
per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER,
|
||||
records: MAX_PODCAST_CACHE_RECORDS,
|
||||
bytes: MAX_PODCAST_CACHE_BYTES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This device's latest publication of one podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct LocalPublication {
|
||||
@@ -76,6 +132,7 @@ pub(crate) struct LocalPublication {
|
||||
pub(crate) struct PodcastDhtDatabase {
|
||||
conn: Arc<Mutex<Connection>>,
|
||||
network_id: NetworkId,
|
||||
limits: StoreLimits,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PodcastDhtDatabase {
|
||||
@@ -88,6 +145,14 @@ impl std::fmt::Debug for PodcastDhtDatabase {
|
||||
|
||||
impl PodcastDhtDatabase {
|
||||
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
|
||||
Self::open_with_limits(path, network_id, StoreLimits::default()).await
|
||||
}
|
||||
|
||||
pub async fn open_with_limits(
|
||||
path: &Path,
|
||||
network_id: NetworkId,
|
||||
limits: StoreLimits,
|
||||
) -> Result<Self> {
|
||||
let path = path.to_path_buf();
|
||||
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
|
||||
let conn = Connection::open(&path).map_err(|error| {
|
||||
@@ -101,6 +166,11 @@ impl PodcastDhtDatabase {
|
||||
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
|
||||
)));
|
||||
}
|
||||
if version == 1 {
|
||||
// Replicas are replaceable; the old layout is dropped.
|
||||
conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;")
|
||||
.map_err(database_error)?;
|
||||
}
|
||||
conn.execute_batch(SCHEMA).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
|
||||
})?;
|
||||
@@ -113,37 +183,39 @@ impl PodcastDhtDatabase {
|
||||
Ok(Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
network_id,
|
||||
limits,
|
||||
})
|
||||
}
|
||||
|
||||
async fn call<F, T>(&self, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result<T> + Send + 'static,
|
||||
F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let conn = Arc::clone(&self.conn);
|
||||
let network_id = self.network_id;
|
||||
let limits = self.limits;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = lock(&conn);
|
||||
operation(&guard, &network_id).map_err(database_error)
|
||||
operation(&guard, &network_id, &limits).map_err(database_error)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
|
||||
}
|
||||
|
||||
/// Stores records under the given keys. The caller has verified every
|
||||
/// record and checked that each key belongs to it. Returns the number of
|
||||
/// `(key, record)` rows written.
|
||||
/// record, checked that each key belongs to it and that the sender is its
|
||||
/// publisher. Returns the number of `(key, record)` placements written.
|
||||
pub async fn store(
|
||||
&self,
|
||||
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
|
||||
now_ms: u64,
|
||||
) -> Result<usize> {
|
||||
self.call(move |conn, network_id| {
|
||||
self.call(move |conn, network_id, limits| {
|
||||
let transaction = conn.unchecked_transaction()?;
|
||||
let mut written = 0;
|
||||
for (keys, record) in entries {
|
||||
written += store_one(&transaction, network_id, &keys, &record, now_ms)?;
|
||||
written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?;
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(written)
|
||||
@@ -151,9 +223,11 @@ impl PodcastDhtDatabase {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Valid, unexpired records under `key`, newest expiry first. Cards must
|
||||
/// contain every token in `filter_tokens`; tombstones are always
|
||||
/// returned so stale replicas elsewhere can be overruled.
|
||||
/// Valid, unexpired records under `key`: at most
|
||||
/// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale
|
||||
/// replicas elsewhere can be overruled) and cards containing every token
|
||||
/// in `filter_tokens` up to `limit` records in all. Records indexed longest
|
||||
/// come first. Cards precede tombstones.
|
||||
pub async fn records_by_key(
|
||||
&self,
|
||||
key: PodcastDhtKey,
|
||||
@@ -161,52 +235,70 @@ impl PodcastDhtDatabase {
|
||||
now_ms: u64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<SignedPodcastRecord>> {
|
||||
self.call(move |conn, network_id| {
|
||||
self.call(move |conn, network_id, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT payload FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND expires_at_ms > ?2
|
||||
ORDER BY expires_at_ms DESC, podcast_id, publisher",
|
||||
"SELECT r.payload FROM podcast_dht_keys k
|
||||
JOIN podcast_dht_records r
|
||||
ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher
|
||||
WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3
|
||||
ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher",
|
||||
)?;
|
||||
let rows = statement.query_map(
|
||||
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|
||||
|row| row.get::<_, Vec<u8>>(0),
|
||||
)?;
|
||||
let mut records = Vec::new();
|
||||
for row in rows {
|
||||
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
||||
continue;
|
||||
};
|
||||
if !record.belongs_under(&key) {
|
||||
continue;
|
||||
let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result<Vec<_>> {
|
||||
let mut records = Vec::new();
|
||||
if limit == 0 {
|
||||
return Ok(records);
|
||||
}
|
||||
let matches = record.card().is_none_or(|card| {
|
||||
let tokens = card.index_tokens();
|
||||
filter_tokens.iter().all(|token| tokens.contains(token))
|
||||
});
|
||||
if matches {
|
||||
records.push(record);
|
||||
if records.len() == limit {
|
||||
break;
|
||||
let rows = statement.query_map(
|
||||
params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)],
|
||||
|row| row.get::<_, Vec<u8>>(0),
|
||||
)?;
|
||||
for row in rows {
|
||||
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
||||
continue;
|
||||
};
|
||||
if record.is_tombstone() != tombstone || !record.belongs_under(&key) {
|
||||
continue;
|
||||
}
|
||||
let matches = record.card().is_none_or(|card| {
|
||||
let tokens = card.index_tokens();
|
||||
filter_tokens.iter().all(|token| tokens.contains(token))
|
||||
});
|
||||
if matches {
|
||||
records.push(record);
|
||||
if records.len() == limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(records)
|
||||
};
|
||||
let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?;
|
||||
let mut records = select(false, limit - tombstones.len())?;
|
||||
records.extend(tombstones);
|
||||
Ok(records)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Removes expired records and their key placements. Returns the number
|
||||
/// of records removed.
|
||||
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
let now = clamp_i64(now_ms);
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1",
|
||||
[now],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
|
||||
[clamp_i64(now_ms)],
|
||||
[now],
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_peers
|
||||
(peer_id, node_id, ticket, last_seen_ms)
|
||||
@@ -218,13 +310,20 @@ impl PodcastDhtDatabase {
|
||||
clamp_i64(contact.last_seen_ms),
|
||||
],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id IN (
|
||||
SELECT peer_id FROM podcast_dht_peers
|
||||
ORDER BY last_seen_ms DESC, peer_id
|
||||
LIMIT -1 OFFSET ?1)",
|
||||
[MAX_PERSISTED_PODCAST_DHT_PEERS as i64],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
|
||||
[peer.to_string()],
|
||||
@@ -235,7 +334,7 @@ impl PodcastDhtDatabase {
|
||||
}
|
||||
|
||||
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
let mut statement =
|
||||
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
|
||||
let rows = statement.query_map([], |row| {
|
||||
@@ -267,7 +366,7 @@ impl PodcastDhtDatabase {
|
||||
&self,
|
||||
podcast_id: PodcastId,
|
||||
) -> Result<Option<LocalPublication>> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.query_row(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
WHERE podcast_id = ?1",
|
||||
@@ -281,7 +380,7 @@ impl PodcastDhtDatabase {
|
||||
}
|
||||
|
||||
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
ORDER BY podcast_id",
|
||||
@@ -303,7 +402,7 @@ impl PodcastDhtDatabase {
|
||||
publication: LocalPublication,
|
||||
now_ms: u64,
|
||||
) -> Result<()> {
|
||||
self.call(move |conn, _| {
|
||||
self.call(move |conn, _, _| {
|
||||
let spec = publication
|
||||
.spec
|
||||
.as_ref()
|
||||
@@ -332,141 +431,235 @@ impl PodcastDhtDatabase {
|
||||
fn store_one(
|
||||
conn: &Connection,
|
||||
network_id: &NetworkId,
|
||||
limits: &StoreLimits,
|
||||
keys: &[PodcastDhtKey],
|
||||
record: &SignedPodcastRecord,
|
||||
now_ms: u64,
|
||||
) -> rusqlite::Result<usize> {
|
||||
let podcast_id = record.podcast_id().as_str().to_string();
|
||||
let publisher = record.publisher().to_string();
|
||||
let now = clamp_i64(now_ms);
|
||||
|
||||
// Every cached version of this (podcast, publisher), re-verified.
|
||||
let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new();
|
||||
let mut invalid_keys = Vec::new();
|
||||
{
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT dht_key, payload FROM podcast_dht_records
|
||||
// The cached record of this (podcast, publisher), re-verified.
|
||||
let cached: Option<(Vec<u8>, i64)> = conn
|
||||
.query_row(
|
||||
"SELECT payload, payload_bytes FROM podcast_dht_records
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
||||
Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?))
|
||||
})?;
|
||||
for row in rows {
|
||||
let (key, payload) = row?;
|
||||
let decoded = decode_valid(&payload, network_id, now_ms);
|
||||
match (<[u8; 32]>::try_from(key.as_slice()), decoded) {
|
||||
(Ok(key_bytes), Some(cached)) => {
|
||||
existing.insert(key_bytes, cached);
|
||||
}
|
||||
_ => invalid_keys.push(key),
|
||||
params![podcast_id, publisher],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (existing, existing_bytes) = match cached {
|
||||
Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) {
|
||||
Some(existing) => (Some(existing), bytes.max(0) as usize),
|
||||
None => {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
(None, 0)
|
||||
}
|
||||
},
|
||||
None => {
|
||||
// Stray placements without a record.
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
(None, 0)
|
||||
}
|
||||
}
|
||||
for key in invalid_keys {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key, podcast_id, publisher],
|
||||
)?;
|
||||
}
|
||||
};
|
||||
|
||||
// A tombstone or a newer revision cached under any key wins everywhere.
|
||||
if existing
|
||||
.values()
|
||||
.any(|cached| supersedes_everywhere(cached, record))
|
||||
// Higher revision, tombstone at an equal revision, later republish...
|
||||
let replace = match &existing {
|
||||
Some(existing) => match compare_records(existing, record) {
|
||||
std::cmp::Ordering::Greater => return Ok(0),
|
||||
std::cmp::Ordering::Equal => false,
|
||||
std::cmp::Ordering::Less => true,
|
||||
},
|
||||
None => true,
|
||||
};
|
||||
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
||||
if replace
|
||||
&& !admits_record(
|
||||
conn,
|
||||
limits,
|
||||
&publisher,
|
||||
existing.is_none(),
|
||||
existing_bytes,
|
||||
payload.len(),
|
||||
now,
|
||||
)?
|
||||
{
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
||||
let mut indexed: HashSet<[u8; 32]> = HashSet::new();
|
||||
{
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
||||
row.get::<_, Vec<u8>>(0)
|
||||
})?;
|
||||
for row in rows {
|
||||
if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) {
|
||||
indexed.insert(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
if replace {
|
||||
// Placements the new version no longer belongs under are dropped (a
|
||||
// renamed card, or a tombstone that does not list the key); the others
|
||||
// now serve the new version.
|
||||
let belongs: HashSet<[u8; 32]> = record
|
||||
.dht_keys()
|
||||
.iter()
|
||||
.map(|key| *key.as_bytes())
|
||||
.collect();
|
||||
let stale: Vec<[u8; 32]> = indexed
|
||||
.iter()
|
||||
.filter(|key| !belongs.contains(*key))
|
||||
.copied()
|
||||
.collect();
|
||||
for key in stale {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
indexed.remove(&key);
|
||||
}
|
||||
written += conn.execute(
|
||||
"UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
],
|
||||
)?;
|
||||
}
|
||||
|
||||
// New placements. A tombstone only takes a new place under the podcast-id
|
||||
// key: under a text key it may only replace a record of its owner, so
|
||||
// nobody can index tombstones under someone else's search terms.
|
||||
let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id());
|
||||
for key in keys {
|
||||
if let Some(cached) = existing.get(key.as_bytes())
|
||||
&& compare_records(cached, record).is_ge()
|
||||
{
|
||||
if indexed.contains(key.as_bytes()) {
|
||||
continue;
|
||||
}
|
||||
if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? {
|
||||
if record.is_tombstone() && *key != id_key {
|
||||
continue;
|
||||
}
|
||||
if !admits_placement(conn, limits, key, record.is_tombstone(), now)? {
|
||||
continue;
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_records
|
||||
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET
|
||||
tombstone = excluded.tombstone,
|
||||
expires_at_ms = excluded.expires_at_ms,
|
||||
payload = excluded.payload",
|
||||
"INSERT INTO podcast_dht_keys
|
||||
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
payload,
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
indexed.insert(*key.as_bytes());
|
||||
written += 1;
|
||||
}
|
||||
|
||||
// Drop versions the new record makes obsolete under other keys too.
|
||||
if written > 0 {
|
||||
for (key, cached) in &existing {
|
||||
if !keys.iter().any(|stored| stored.as_bytes() == key)
|
||||
&& supersedes_everywhere(record, cached)
|
||||
{
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
if indexed.is_empty() {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
return Ok(0);
|
||||
}
|
||||
if replace {
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_records
|
||||
(podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(podcast_id, publisher) DO UPDATE SET
|
||||
tombstone = excluded.tombstone,
|
||||
expires_at_ms = excluded.expires_at_ms,
|
||||
payload_bytes = excluded.payload_bytes,
|
||||
payload = excluded.payload",
|
||||
params![
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
payload.len() as i64,
|
||||
payload,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Ensures there is room for one more record under `key`.
|
||||
fn make_room(
|
||||
/// Whether the cache may hold the new version of a record: per-publisher and
|
||||
/// global record counts for a new owner, and the global byte budget.
|
||||
fn admits_record(
|
||||
conn: &Connection,
|
||||
key: &PodcastDhtKey,
|
||||
record: &SignedPodcastRecord,
|
||||
now_ms: u64,
|
||||
limits: &StoreLimits,
|
||||
publisher: &str,
|
||||
new_owner: bool,
|
||||
previous_bytes: usize,
|
||||
new_bytes: usize,
|
||||
now: i64,
|
||||
) -> rusqlite::Result<bool> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2",
|
||||
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|
||||
let (records, bytes): (i64, i64) = conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records
|
||||
WHERE expires_at_ms > ?1",
|
||||
[now],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1",
|
||||
[key.as_bytes().as_slice()],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY {
|
||||
let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
|
||||
if bytes.saturating_add(new_bytes) > limits.bytes {
|
||||
return Ok(false);
|
||||
}
|
||||
if !new_owner {
|
||||
return Ok(true);
|
||||
}
|
||||
let victim: Option<(String, String, i64)> = conn
|
||||
.query_row(
|
||||
"SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records
|
||||
WHERE dht_key = ?1
|
||||
ORDER BY expires_at_ms ASC, podcast_id, publisher
|
||||
LIMIT 1",
|
||||
[key.as_bytes().as_slice()],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
||||
)
|
||||
.optional()?;
|
||||
match victim {
|
||||
Some((podcast_id, publisher, expires_at_ms))
|
||||
if expires_at_ms < clamp_i64(record.expires_at_ms()) =>
|
||||
{
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_bytes().as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
_ => Ok(false),
|
||||
if records.max(0) as usize >= limits.records {
|
||||
return Ok(false);
|
||||
}
|
||||
let own: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
|
||||
params![publisher, now],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok((own.max(0) as usize) < limits.per_publisher)
|
||||
}
|
||||
|
||||
/// Whether `key` has room for one more placement. Existing placements are
|
||||
/// never evicted.
|
||||
fn admits_placement(
|
||||
conn: &Connection,
|
||||
limits: &StoreLimits,
|
||||
key: &PodcastDhtKey,
|
||||
tombstone: bool,
|
||||
now: i64,
|
||||
) -> rusqlite::Result<bool> {
|
||||
let (total, tombstones): (i64, i64) = conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND expires_at_ms > ?2",
|
||||
params![key.as_bytes().as_slice(), now],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
Ok((total.max(0) as usize) < limits.per_key
|
||||
&& (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key))
|
||||
}
|
||||
|
||||
fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_valid(
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
|
||||
use federation_net::{NetworkId, SecretKey};
|
||||
|
||||
use super::card::supersedes_everywhere;
|
||||
use crate::routing::NodeId;
|
||||
|
||||
use super::search::{aggregate_hits, rank_hits};
|
||||
use super::*;
|
||||
use crate::podcast::{FeedUrl, PodcastMeta};
|
||||
@@ -362,10 +363,6 @@ fn record_order_is_total_and_tombstones_win_ties() {
|
||||
compare_records(&edited, &older),
|
||||
compare_records(&older, &edited).reverse()
|
||||
);
|
||||
assert!(supersedes_everywhere(&grave, &older));
|
||||
assert!(supersedes_everywhere(&newer, &older));
|
||||
assert!(!supersedes_everywhere(&refresh, &older));
|
||||
assert!(!supersedes_everywhere(&older, &grave));
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------- cache
|
||||
@@ -437,7 +434,7 @@ async fn cache_converges_regardless_of_delivery_order() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() {
|
||||
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key_it_holds() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let key = secret(1);
|
||||
@@ -451,7 +448,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key()
|
||||
vec![active.clone()]
|
||||
);
|
||||
|
||||
// A tombstone stored only under the id key still removes the card under
|
||||
// A tombstone stored only under the id key still replaces the card under
|
||||
// the text keys of this node.
|
||||
let grave = tombstone(&key, 2, NOW);
|
||||
let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id());
|
||||
@@ -462,7 +459,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key()
|
||||
db.records_by_key(token_key, vec!["rust".into()], NOW, 10)
|
||||
.await
|
||||
.unwrap(),
|
||||
Vec::<SignedPodcastRecord>::new()
|
||||
vec![grave.clone()]
|
||||
);
|
||||
// An equal-revision card arriving later loses, under any key.
|
||||
let same = card(&key, 2);
|
||||
@@ -564,43 +561,247 @@ async fn cache_read_path_filters_tokens_expiry_and_corruption() {
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
db.delete_expired(NOW + ttl).await.unwrap(),
|
||||
MAX_RECORD_DHT_KEYS.min(record.dht_keys().len())
|
||||
);
|
||||
assert_eq!(db.delete_expired(NOW + ttl).await.unwrap(), 1);
|
||||
}
|
||||
|
||||
fn numbered_secret(index: usize) -> SecretKey {
|
||||
let mut bytes = [0u8; 32];
|
||||
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
|
||||
bytes[31] = 0x5a;
|
||||
SecretKey::from_bytes(&bytes)
|
||||
}
|
||||
|
||||
async fn open_db_with(
|
||||
temp: &tempfile::TempDir,
|
||||
limits: store::StoreLimits,
|
||||
) -> store::PodcastDhtDatabase {
|
||||
store::PodcastDhtDatabase::open_with_limits(
|
||||
&temp.path().join("podcast-dht.sqlite3"),
|
||||
network(),
|
||||
limits,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_bounds_records_per_key() {
|
||||
async fn cache_full_key_keeps_its_records_and_refuses_newcomers() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let token_key = PodcastDhtKey::token(&network(), "rust");
|
||||
let mut entries = Vec::new();
|
||||
for index in 0..=MAX_PODCAST_RECORDS_PER_KEY {
|
||||
let key = SecretKey::from_bytes(&{
|
||||
let mut bytes = [0u8; 32];
|
||||
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
|
||||
bytes
|
||||
});
|
||||
// Later publishers expire later, so the first one is evicted.
|
||||
let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64);
|
||||
// Later publishers expire later; that buys them nothing.
|
||||
let record = card_with(
|
||||
spec("show.xml", "Rust Radio"),
|
||||
&numbered_secret(index),
|
||||
1,
|
||||
NOW + index as u64,
|
||||
);
|
||||
entries.push((vec![token_key], record));
|
||||
}
|
||||
let first = entries[0].1.clone();
|
||||
for chunk in entries.chunks(64) {
|
||||
db.store(chunk.to_vec(), NOW).await.unwrap();
|
||||
let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone();
|
||||
// The first publisher arrived first; the rest a moment later.
|
||||
db.store(entries[..1].to_vec(), NOW).await.unwrap();
|
||||
for chunk in entries[1..].chunks(64) {
|
||||
db.store(chunk.to_vec(), NOW + 1).await.unwrap();
|
||||
}
|
||||
let cached = db
|
||||
.records_by_key(token_key, Vec::new(), NOW, usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
|
||||
assert!(!cached.contains(&first));
|
||||
// A record expiring before every cached one is not admitted.
|
||||
assert_eq!(
|
||||
db.store(vec![(vec![token_key], first)], NOW).await.unwrap(),
|
||||
0
|
||||
assert!(cached.contains(&first));
|
||||
assert!(!cached.contains(&last));
|
||||
assert_eq!(cached[0], first, "records indexed longest come first");
|
||||
// The first publisher keeps refreshing its place.
|
||||
let refresh = card_with(
|
||||
spec("show.xml", "Rust Radio"),
|
||||
&numbered_secret(0),
|
||||
1,
|
||||
NOW + 1_000,
|
||||
);
|
||||
assert_eq!(
|
||||
db.store(vec![(vec![token_key], refresh)], NOW)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sybil_tombstones_cannot_suppress_a_card() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let victim = card(&secret(1), 1);
|
||||
let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id());
|
||||
let rust = PodcastDhtKey::token(&network(), "rust");
|
||||
db.store(vec![entry(&victim)], NOW).await.unwrap();
|
||||
|
||||
// 256 throwaway identities sign tombstones of the victim's podcast that
|
||||
// list the victim card's text keys, and issue them as late as allowed.
|
||||
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
|
||||
let sybils: Vec<_> = (0..256)
|
||||
.map(|index| {
|
||||
let key = numbered_secret(index);
|
||||
let tombstone = PodcastTombstone::new(
|
||||
network(),
|
||||
key.public(),
|
||||
victim.podcast_id().clone(),
|
||||
u64::MAX,
|
||||
NOW + skew,
|
||||
victim.dht_keys(),
|
||||
);
|
||||
let record =
|
||||
SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap();
|
||||
(record.dht_keys(), record)
|
||||
})
|
||||
.collect();
|
||||
let mut written = 0;
|
||||
for chunk in sybils.chunks(MAX_PODCAST_STORE_RECORDS) {
|
||||
written += db.store(chunk.to_vec(), NOW).await.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
written, MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||
"only the id key's tombstone budget is used"
|
||||
);
|
||||
|
||||
let by_id = db
|
||||
.records_by_key(id_key, Vec::new(), NOW, MAX_PODCAST_DHT_RECORDS)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(by_id.contains(&victim));
|
||||
assert_eq!(
|
||||
by_id.iter().filter(|record| record.is_tombstone()).count(),
|
||||
MAX_PODCAST_RESPONSE_TOMBSTONES
|
||||
);
|
||||
assert_eq!(
|
||||
db.records_by_key(rust, vec!["rust".into()], NOW, MAX_PODCAST_DHT_RECORDS)
|
||||
.await
|
||||
.unwrap(),
|
||||
vec![victim.clone()],
|
||||
"no tombstone is indexed under a text key without its owner's card"
|
||||
);
|
||||
// Eleven hours later the victim's republish still lands.
|
||||
let later = NOW + 11 * 3_600_000;
|
||||
let republished = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later);
|
||||
assert_eq!(
|
||||
db.store(vec![entry(&republished)], later).await.unwrap(),
|
||||
republished.dht_keys().len()
|
||||
);
|
||||
// Sybil cards issued in the future cannot displace it either.
|
||||
let sybil_cards: Vec<_> = (300..600)
|
||||
.map(|index| {
|
||||
entry(&card_with(
|
||||
spec("show.xml", "Rust Radio"),
|
||||
&numbered_secret(index),
|
||||
u64::MAX,
|
||||
later + skew,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
for chunk in sybil_cards.chunks(MAX_PODCAST_STORE_RECORDS) {
|
||||
db.store(chunk.to_vec(), later).await.unwrap();
|
||||
}
|
||||
for key in [id_key, rust] {
|
||||
let records = db
|
||||
.records_by_key(key, vec![], later, MAX_PODCAST_DHT_RECORDS)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(records.contains(&republished));
|
||||
assert!(records.len() <= MAX_PODCAST_DHT_RECORDS);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_enforces_publisher_record_and_byte_quotas() {
|
||||
let limits = store::StoreLimits {
|
||||
per_key: 100,
|
||||
tombstones_per_key: 10,
|
||||
per_publisher: 3,
|
||||
records: 5,
|
||||
bytes: usize::MAX,
|
||||
};
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db_with(&temp, limits).await;
|
||||
let show = |index: usize, key: &SecretKey| {
|
||||
entry(&card_with(
|
||||
spec(&format!("show-{index}.xml"), "Rust Radio"),
|
||||
key,
|
||||
1,
|
||||
NOW,
|
||||
))
|
||||
};
|
||||
let heavy = secret(1);
|
||||
let stored: Vec<usize> = {
|
||||
let mut out = Vec::new();
|
||||
for index in 0..4 {
|
||||
out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap());
|
||||
}
|
||||
out
|
||||
};
|
||||
assert!(stored[..3].iter().all(|written| *written > 0));
|
||||
assert_eq!(
|
||||
stored[3], 0,
|
||||
"the fourth podcast of one publisher is refused"
|
||||
);
|
||||
// A refresh of an admitted record is always accepted.
|
||||
let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1);
|
||||
assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0);
|
||||
assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0);
|
||||
assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0);
|
||||
assert_eq!(
|
||||
db.store(vec![show(12, &secret(4))], NOW).await.unwrap(),
|
||||
0,
|
||||
"the cache holds at most five records"
|
||||
);
|
||||
|
||||
let one = card(&secret(1), 1);
|
||||
let limits = store::StoreLimits {
|
||||
bytes: one.encoded_len() * 2,
|
||||
..store::StoreLimits::default()
|
||||
};
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db_with(&temp, limits).await;
|
||||
assert!(
|
||||
db.store(vec![entry(&card(&secret(1), 1))], NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
> 0
|
||||
);
|
||||
assert!(
|
||||
db.store(vec![entry(&card(&secret(2), 1))], NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
> 0
|
||||
);
|
||||
assert_eq!(
|
||||
db.store(vec![entry(&card(&secret(3), 1))], NOW)
|
||||
.await
|
||||
.unwrap(),
|
||||
0,
|
||||
"the byte budget is full"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_stores_each_payload_once() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let db = open_db(&temp).await;
|
||||
let many: Vec<String> = (0..50).map(|index| format!("w{index}")).collect();
|
||||
let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW);
|
||||
assert_eq!(
|
||||
db.store(vec![entry(&record)], NOW).await.unwrap(),
|
||||
MAX_RECORD_DHT_KEYS
|
||||
);
|
||||
let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap();
|
||||
let payloads: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(payloads, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -634,6 +835,25 @@ async fn cache_refuses_a_newer_schema_and_keeps_local_publications() {
|
||||
assert_eq!(local[0].record, record);
|
||||
drop(db);
|
||||
|
||||
// A pre-release version-1 replica table is dropped; own publications
|
||||
// survive.
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"DROP TABLE podcast_dht_records;
|
||||
CREATE TABLE podcast_dht_records (dht_key BLOB, payload BLOB);
|
||||
INSERT INTO podcast_dht_records VALUES (x'00', x'00');
|
||||
PRAGMA user_version = 1;",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let db = store::PodcastDhtDatabase::open(&path, network())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(db.local_publications().await.unwrap().len(), 1);
|
||||
assert!(db.store(vec![entry(&record)], NOW).await.unwrap() > 0);
|
||||
drop(db);
|
||||
|
||||
rusqlite::Connection::open(&path)
|
||||
.unwrap()
|
||||
.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1)
|
||||
@@ -655,12 +875,14 @@ fn store_batches_are_bounded_and_keys_must_belong() {
|
||||
keys: record.dht_keys(),
|
||||
record: record.clone(),
|
||||
};
|
||||
validate_store_batch(vec![good.clone()], &network(), NOW).unwrap();
|
||||
assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err());
|
||||
let publisher = key.public();
|
||||
validate_store_batch(vec![good.clone()], &network(), publisher, NOW).unwrap();
|
||||
assert!(validate_store_batch(Vec::new(), &network(), publisher, NOW).is_err());
|
||||
assert!(
|
||||
validate_store_batch(
|
||||
vec![good.clone(); MAX_PODCAST_STORE_RECORDS],
|
||||
&network(),
|
||||
publisher,
|
||||
NOW
|
||||
)
|
||||
.is_ok()
|
||||
@@ -669,19 +891,87 @@ fn store_batches_are_bounded_and_keys_must_belong() {
|
||||
validate_store_batch(
|
||||
vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1],
|
||||
&network(),
|
||||
publisher,
|
||||
NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let mut foreign = good.clone();
|
||||
foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")];
|
||||
assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err());
|
||||
assert!(validate_store_batch(vec![foreign], &network(), publisher, NOW).is_err());
|
||||
let mut duplicated = good.clone();
|
||||
duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]];
|
||||
assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err());
|
||||
let mut forged = good;
|
||||
assert!(validate_store_batch(vec![duplicated], &network(), publisher, NOW).is_err());
|
||||
let mut forged = good.clone();
|
||||
resign(&mut forged.record, &secret(2));
|
||||
assert!(validate_store_batch(vec![forged], &network(), NOW).is_err());
|
||||
assert!(validate_store_batch(vec![forged], &network(), publisher, NOW).is_err());
|
||||
// A valid record of another publisher may not be stored (or replayed)
|
||||
// by anyone else.
|
||||
let error = validate_store_batch(vec![good], &network(), secret(2).public(), NOW).unwrap_err();
|
||||
assert!(error.to_string().contains("publisher"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicas_only_accept_keys_they_are_close_to() {
|
||||
let own = NodeId::from_bytes([0; 32]);
|
||||
let key = PodcastDhtKey::from_bytes([0; 32]);
|
||||
let far: Vec<NodeId> = (0..100)
|
||||
.map(|index| NodeId::from_bytes([0x80 | index as u8; 32]))
|
||||
.collect();
|
||||
assert!(is_responsible(&own, &far, &key));
|
||||
let near: Vec<NodeId> = (1..=PODCAST_STORE_RESPONSIBILITY_WINDOW)
|
||||
.map(|index| {
|
||||
let mut bytes = [0; 32];
|
||||
bytes[31] = index as u8;
|
||||
NodeId::from_bytes(bytes)
|
||||
})
|
||||
.collect();
|
||||
let own = NodeId::from_bytes([0xff; 32]);
|
||||
assert!(!is_responsible(&own, &near, &key));
|
||||
assert!(is_responsible(&own, &near[1..], &key));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inbound_requests_are_limited_per_peer() {
|
||||
let inflight = Arc::new(Mutex::new(HashMap::new()));
|
||||
let peer = secret(1).public();
|
||||
let slots: Vec<_> = (0..MAX_PODCAST_DHT_REQUESTS_PER_PEER)
|
||||
.map(|_| PeerSlot::acquire(&inflight, peer).unwrap())
|
||||
.collect();
|
||||
assert!(PeerSlot::acquire(&inflight, peer).is_none());
|
||||
assert!(PeerSlot::acquire(&inflight, secret(2).public()).is_some());
|
||||
drop(slots);
|
||||
assert!(PeerSlot::acquire(&inflight, peer).is_some());
|
||||
assert!(lock(&inflight).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookups_keep_a_bounded_number_of_owners() {
|
||||
let mut winners = HashMap::new();
|
||||
let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id());
|
||||
let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS)
|
||||
.map(|index| {
|
||||
card_with(
|
||||
spec("show.xml", "Rust Radio"),
|
||||
&numbered_secret(index),
|
||||
1,
|
||||
now_ms(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let first = records[0].clone();
|
||||
merge_records(&mut winners, records, &network(), &id_key, &[]);
|
||||
assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS);
|
||||
// A newer record of a known owner still replaces its winner.
|
||||
let newer = card_with(
|
||||
spec("show.xml", "Rust Radio"),
|
||||
&numbered_secret(0),
|
||||
2,
|
||||
now_ms(),
|
||||
);
|
||||
merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]);
|
||||
let owner = (first.podcast_id().clone(), first.publisher());
|
||||
assert_eq!(winners[&owner], newer);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -763,18 +1053,27 @@ fn music(name: &str, kind: ItemKind) -> LibraryItem {
|
||||
}
|
||||
}
|
||||
|
||||
fn moved_spec(path: &str, origin: &str, title: &str) -> PodcastCardSpec {
|
||||
let mut moved = spec(path, title);
|
||||
moved.id_feed_url = Some(feed(origin));
|
||||
moved.podcast_id = PodcastId::from_feed_url(&feed(origin));
|
||||
moved
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hits_merge_publishers_and_drop_tombstoned_ones() {
|
||||
let a = secret(1);
|
||||
let b = secret(2);
|
||||
let c = secret(3);
|
||||
let mut moved = spec("moved.xml", "Rust Radio");
|
||||
moved.id_feed_url = Some(feed("show.xml"));
|
||||
moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml"));
|
||||
let records = vec![
|
||||
card(&a, 1),
|
||||
card(&a, 2),
|
||||
card_with(moved, &b, 1, NOW + 10),
|
||||
card_with(
|
||||
moved_spec("moved.xml", "show.xml", "Rust Radio"),
|
||||
&b,
|
||||
1,
|
||||
NOW + 10,
|
||||
),
|
||||
card(&c, 1),
|
||||
tombstone(&c, 2, NOW),
|
||||
card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW),
|
||||
@@ -790,8 +1089,81 @@ fn hits_merge_publishers_and_drop_tombstoned_ones() {
|
||||
let mut expected = vec![a.public(), b.public()];
|
||||
expected.sort_by_key(|key| *key.as_bytes());
|
||||
assert_eq!(hit.publishers, expected);
|
||||
// Equal votes per feed URL: the most recently issued card represents.
|
||||
assert_eq!(hit.card.publisher, b.public());
|
||||
// The card under the origin URL represents, whoever issued later.
|
||||
assert_eq!(hit.card.publisher, a.public());
|
||||
assert_eq!(hit.feed_url, feed("show.xml"));
|
||||
assert_eq!(
|
||||
hit.claimed_feed_urls,
|
||||
vec![ClaimedFeedUrl {
|
||||
feed_url: feed("moved.xml"),
|
||||
publishers: vec![b.public()],
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sybil_cards_cannot_redirect_an_existing_podcast() {
|
||||
let genuine = card(&secret(1), 1);
|
||||
let id = genuine.podcast_id().clone();
|
||||
let evil_feed = FeedUrl::parse("https://evil.example/feed.xml").unwrap();
|
||||
let hijack = |seed: usize| {
|
||||
let mut spec = spec("show.xml", "Rust Radio (official)");
|
||||
spec.id_feed_url = Some(spec.feed_url.clone());
|
||||
spec.feed_url = evil_feed.clone();
|
||||
spec.validate().unwrap();
|
||||
assert_eq!(spec.podcast_id, id, "a hijack card claims the genuine id");
|
||||
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
|
||||
card_with(spec, &numbered_secret(seed), u64::MAX, NOW + skew)
|
||||
};
|
||||
let mut records = vec![genuine.clone()];
|
||||
records.extend((0..10).map(hijack));
|
||||
let hit = aggregate_hits(records).remove(0);
|
||||
assert_eq!(
|
||||
hit.feed_url,
|
||||
feed("show.xml"),
|
||||
"only the origin URL is offered"
|
||||
);
|
||||
assert_eq!(
|
||||
hit.card,
|
||||
*genuine.card().unwrap(),
|
||||
"the origin card represents"
|
||||
);
|
||||
assert_eq!(hit.claimed_feed_urls.len(), 1);
|
||||
assert_eq!(hit.claimed_feed_urls[0].feed_url, evil_feed);
|
||||
assert_eq!(hit.claimed_feed_urls[0].publishers.len(), 10);
|
||||
|
||||
// Without any origin card the hit still offers the origin URL, never the
|
||||
// claimed one, and ranks below podcasts that have an origin card.
|
||||
let only_claims = aggregate_hits((0..3).map(hijack)).remove(0);
|
||||
assert_eq!(only_claims.feed_url, feed("show.xml"));
|
||||
assert!(!only_claims.card.feed_url_is_origin());
|
||||
let honest = aggregate_hits(vec![card_with(
|
||||
spec("b.xml", "Rust Radio Weekly"),
|
||||
&secret(2),
|
||||
1,
|
||||
NOW,
|
||||
)])
|
||||
.remove(0);
|
||||
let ranked = rank_hits(
|
||||
vec![only_claims.clone(), honest.clone()],
|
||||
"rust",
|
||||
&podcast_query_tokens("rust"),
|
||||
);
|
||||
assert_eq!(ranked[0].podcast_id, honest.podcast_id);
|
||||
// Publisher counts do not rank.
|
||||
let crowded = aggregate_hits(
|
||||
(20..60).map(|seed| card_with(spec("c.xml", "Rust Zone"), &numbered_secret(seed), 1, NOW)),
|
||||
)
|
||||
.remove(0);
|
||||
let ranked = rank_hits(
|
||||
vec![crowded, honest.clone()],
|
||||
"rust",
|
||||
&podcast_query_tokens("rust"),
|
||||
);
|
||||
assert_eq!(
|
||||
ranked[0].podcast_id, honest.podcast_id,
|
||||
"ranked by title, not crowd"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
//! Integration test: public podcast discovery on three real loopback iroh
|
||||
//! nodes.
|
||||
//!
|
||||
//! * A publishes a podcast its user shared; B is one of A's trusted devices
|
||||
//! (in this test merely another overlay node, then offline); C is an
|
||||
//! unrelated peer outside A's trusted-device group, with no device-sync
|
||||
//! pairing.
|
||||
//! * C finds the card by text and by id; a podcast A never shared is never
|
||||
//! discoverable; after A unpublishes, the tombstone hides the card from C.
|
||||
//! * Forged, tampered, replayed, malformed, oversized and wrong-version
|
||||
//! requests are rejected by a real peer.
|
||||
//! * D, a music-DHT peer without the overlay, is never added to the overlay.
|
||||
//! * A publishes a podcast its user shared. B is another device of the same
|
||||
//! user: it applies the same merged registers (as device sync would deliver
|
||||
//! them) and publishes the same podcast under its own key. C is an
|
||||
//! unrelated peer outside the user's trusted-device group, with no
|
||||
//! device-sync pairing.
|
||||
//! * C finds the podcast by text and by id, merged over publishers {A, B};
|
||||
//! once B unshares and goes offline, C still finds A's card, so B is
|
||||
//! irrelevant to discovery. A podcast the user never shared (held by both
|
||||
//! A and B) is never discoverable; after A unpublishes, the tombstone hides
|
||||
//! the card from C.
|
||||
//! * Forged, foreign, tampered, replayed, malformed, oversized and
|
||||
//! wrong-version requests are rejected by a real peer.
|
||||
//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but
|
||||
//! never answers on the overlay ALPN, so it never becomes an overlay contact
|
||||
//! and is never asked for records or to store them.
|
||||
//! * `sync_shares` stays fail-safe for revocation and holds back URLs that
|
||||
//! look private (single node).
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
@@ -214,14 +222,15 @@ async fn scenario() {
|
||||
(a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
// D never answers on the overlay ALPN, so it never becomes an overlay
|
||||
// contact and is never queried or asked to store records.
|
||||
// D is dialed by bootstrap but never answers on the overlay ALPN, so it
|
||||
// never becomes an overlay contact: it is never asked for records or to
|
||||
// store them.
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 2);
|
||||
|
||||
// A capability manifest without podcast_dht evicts a peer; one with it
|
||||
// lets the peer be learned again.
|
||||
let manifest = CapabilityManifest::frid("furumi", "test");
|
||||
let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht();
|
||||
a.dht
|
||||
.note_peer_capabilities(
|
||||
c.service.endpoint_id(),
|
||||
@@ -260,11 +269,13 @@ async fn scenario() {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, true, 2)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
PodcastShareState {
|
||||
subscription: private.clone(),
|
||||
share: None,
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
];
|
||||
let stats = a.dht.sync_shares(states.clone()).await.unwrap();
|
||||
@@ -294,7 +305,54 @@ async fn scenario() {
|
||||
assert_eq!(published.len(), 1);
|
||||
let first_card = published[0].record.clone();
|
||||
|
||||
// B (A's trusted device) is not needed for discovery.
|
||||
// B, the same user's other device, holds the same merged registers and
|
||||
// publishes the same podcast under its own key; the private podcast
|
||||
// stays private there too.
|
||||
let b_states = vec![
|
||||
PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, true, 2)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
PodcastShareState {
|
||||
subscription: private.clone(),
|
||||
share: None,
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
];
|
||||
let stats = b.dht.sync_shares(b_states).await.unwrap();
|
||||
assert_eq!((stats.published, stats.unpublished), (1, 0));
|
||||
let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()];
|
||||
both.sort_by_key(|key| *key.as_bytes());
|
||||
let hits = wait_for("C sees both publishers", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(hits.len() == 1 && hits[0].publishers == both).then_some(hits)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||
assert!(hits[0].claimed_feed_urls.is_empty());
|
||||
|
||||
// B unshares (for example a stale register on B) and goes offline: B is
|
||||
// not needed for discovery, A's card is still found.
|
||||
let stats = b
|
||||
.dht
|
||||
.sync_shares(vec![PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, false, 3)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.unpublished, 1);
|
||||
let a_only = vec![a.service.endpoint_id()];
|
||||
wait_for("B's tombstone reaches C's lookup", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(hits.len() == 1 && hits[0].publishers == a_only).then_some(())
|
||||
})
|
||||
.await;
|
||||
b.stop().await;
|
||||
|
||||
let hits = wait_for("C finds the card by text", async || {
|
||||
@@ -305,6 +363,7 @@ async fn scenario() {
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].podcast_id, shared.podcast_id);
|
||||
assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]);
|
||||
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||
assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url);
|
||||
assert_eq!(hits[0].card.title, "Rust Radio Weekly");
|
||||
assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens");
|
||||
@@ -350,7 +409,11 @@ async fn scenario() {
|
||||
let response = store_request(&c, &a, forged).await;
|
||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||
|
||||
// Tampered: A's genuine card with a different feed URL.
|
||||
// Foreign: A's genuine card stored by C. Only publishers store records.
|
||||
let response = store_request(&c, &a, first_card.clone()).await;
|
||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||
|
||||
// Tampered: A's genuine card with a different description.
|
||||
let mut tampered = first_card.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
||||
card.description = Some("Now with malware".into());
|
||||
@@ -388,6 +451,7 @@ async fn scenario() {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, false, 3)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
}];
|
||||
let stats = a.dht.sync_shares(unshare).await.unwrap();
|
||||
assert_eq!(stats.unpublished, 1);
|
||||
@@ -407,10 +471,12 @@ async fn scenario() {
|
||||
.is_none()
|
||||
);
|
||||
|
||||
// Replaying the old card, even to the node that stores the tombstone,
|
||||
// does not bring it back.
|
||||
let response = store_request(&c, &a, first_card).await;
|
||||
// A replaying its own old card to a replica that stores the tombstone
|
||||
// does not bring it back; nobody else may replay it at all.
|
||||
let response = store_request(&a, &c, first_card.clone()).await;
|
||||
assert!(response.ok && response.stored == 0, "{response:?}");
|
||||
let response = store_request(&c, &a, first_card).await;
|
||||
assert!(!response.ok, "{response:?}");
|
||||
assert!(search(&c, "rust radio").await.is_empty());
|
||||
assert!(search(&a, "rust radio").await.is_empty());
|
||||
|
||||
@@ -419,3 +485,84 @@ async fn scenario() {
|
||||
d_events.abort();
|
||||
d.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn sync_shares_is_fail_safe_and_holds_back_private_urls() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, single_node_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn single_node_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-sync-shares");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let state =
|
||||
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
|
||||
subscription: subscription.clone(),
|
||||
share: Some(share(subscription, shared, generation)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
};
|
||||
let first = subscription("first.xml", "First Show");
|
||||
let second = subscription("second.xml", "Second Show");
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&first, true, 2), state(&second, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.published, 2);
|
||||
assert!(stats.failed.is_empty() && stats.held_back.is_empty());
|
||||
|
||||
// In one call the user unshares the first podcast while the second
|
||||
// one's card cannot be built (the URL its id derives from is unknown).
|
||||
let mut broken = second.clone();
|
||||
broken.podcast.feed_url = feed("elsewhere.xml");
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.failed.len(), 1);
|
||||
assert_eq!(stats.failed[0].podcast_id, second.podcast_id);
|
||||
assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast");
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert!(published.iter().all(|publication| !publication.active));
|
||||
assert!(
|
||||
published
|
||||
.iter()
|
||||
.all(|publication| publication.record.is_tombstone())
|
||||
);
|
||||
// Republishing re-sends tombstones only; nothing comes back.
|
||||
a.dht.republish().await.unwrap();
|
||||
assert!(search(&a, "first show").await.is_empty());
|
||||
assert!(search(&a, "second show").await.is_empty());
|
||||
|
||||
// A feed URL that looks private is held back until the user confirms.
|
||||
let mut member = subscription("member.xml?token=abc", "Member Show");
|
||||
member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url);
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&member, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.published, 0);
|
||||
assert_eq!(stats.held_back.len(), 1);
|
||||
assert_eq!(stats.held_back[0].podcast_id, member.podcast_id);
|
||||
assert!(!stats.held_back[0].warnings.is_empty());
|
||||
assert!(search(&a, "member show").await.is_empty());
|
||||
let mut confirmed = state(&member, true, 2);
|
||||
confirmed.privacy_confirmed = true;
|
||||
let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap();
|
||||
assert_eq!(stats.published, 1);
|
||||
assert_eq!(search(&a, "member show").await.len(), 1);
|
||||
// Once published with the user's consent, the same URLs are kept.
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&member, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0));
|
||||
|
||||
a.stop().await;
|
||||
}
|
||||
Reference in new issue
Block a user