Harden podcast discovery against sybil and hijack records
Replicas now accept records only from their publisher, keep one payload per (podcast, publisher), store placements only for keys they are close to, and admit records within per-key, per-tombstone, per-publisher and total budgets without ever evicting existing records. Tombstones cannot claim text keys without their owner's card. Search offers only the feed URL the podcast id hashes from and reports moved-feed URLs as claims; publisher counts no longer rank. sync_shares handles each podcast on its own, always revokes, and holds back cards with privacy warnings. Lookups and inbound requests per peer are bounded, persisted contacts are capped, and the podcast_dht capability is opt-in. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fef155d6ef
commit
81ccd924d0
7 files changed
+1276
-281
No files matched your search
@@ -56,7 +56,13 @@ pub struct CapabilityManifest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CapabilityManifest {
|
impl CapabilityManifest {
|
||||||
/// Creates a manifest containing every protocol owned by Frid.
|
/// Creates a manifest containing every protocol owned by Frid that a
|
||||||
|
/// music-DHT application serves.
|
||||||
|
///
|
||||||
|
/// The public podcast-discovery overlay is opt-in: applications that run
|
||||||
|
/// [`crate::podcast_dht::PodcastDht`] add it with
|
||||||
|
/// [`Self::with_podcast_dht`], so an application that does not serve the
|
||||||
|
/// overlay never advertises it.
|
||||||
pub fn frid(application: impl Into<String>, application_version: impl Into<String>) -> Self {
|
pub fn frid(application: impl Into<String>, application_version: impl Into<String>) -> Self {
|
||||||
let mut protocols = BTreeMap::new();
|
let mut protocols = BTreeMap::new();
|
||||||
protocols.insert(
|
protocols.insert(
|
||||||
@@ -81,10 +87,6 @@ impl CapabilityManifest {
|
|||||||
SIMILARITY_DHT_ID.to_string(),
|
SIMILARITY_DHT_ID.to_string(),
|
||||||
crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION,
|
crate::similarity_lsh::SIMILARITY_DHT_PROTOCOL_VERSION,
|
||||||
);
|
);
|
||||||
protocols.insert(
|
|
||||||
PODCAST_DHT_ID.to_string(),
|
|
||||||
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
|
||||||
);
|
|
||||||
protocols.insert(
|
protocols.insert(
|
||||||
DEVICE_SYNC_ID.to_string(),
|
DEVICE_SYNC_ID.to_string(),
|
||||||
crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION,
|
crate::device_sync::DEVICE_SYNC_PROTOCOL_VERSION,
|
||||||
@@ -111,6 +113,17 @@ impl CapabilityManifest {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Advertises the public podcast-discovery overlay
|
||||||
|
/// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||||
|
/// Call it only when the application serves
|
||||||
|
/// [`crate::podcast_dht::PODCAST_DHT_ALPN`].
|
||||||
|
pub fn with_podcast_dht(self) -> Self {
|
||||||
|
self.with_protocol(
|
||||||
|
PODCAST_DHT_ID,
|
||||||
|
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Version of protocol `id` advertised by the peer, if any.
|
/// Version of protocol `id` advertised by the peer, if any.
|
||||||
pub fn protocol_version(&self, id: &str) -> Option<u16> {
|
pub fn protocol_version(&self, id: &str) -> Option<u16> {
|
||||||
self.protocols.get(id).copied()
|
self.protocols.get(id).copied()
|
||||||
@@ -237,7 +250,6 @@ mod tests {
|
|||||||
CATALOG_ID,
|
CATALOG_ID,
|
||||||
SIMILARITY_ID,
|
SIMILARITY_ID,
|
||||||
SIMILARITY_DHT_ID,
|
SIMILARITY_DHT_ID,
|
||||||
PODCAST_DHT_ID,
|
|
||||||
DEVICE_SYNC_ID,
|
DEVICE_SYNC_ID,
|
||||||
JAM_ID,
|
JAM_ID,
|
||||||
] {
|
] {
|
||||||
@@ -247,13 +259,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn podcast_dht_capability_is_advertised_and_removable() {
|
fn podcast_dht_capability_is_opt_in_and_removable() {
|
||||||
let manifest = CapabilityManifest::frid("furumi", "1.2.3");
|
let plain = CapabilityManifest::frid("furumi", "1.2.3");
|
||||||
|
assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None);
|
||||||
|
assert!(!plain.supports_podcast_dht());
|
||||||
|
let manifest = plain.with_podcast_dht();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
manifest.protocol_version(PODCAST_DHT_ID),
|
manifest.protocol_version(PODCAST_DHT_ID),
|
||||||
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||||
);
|
);
|
||||||
assert!(manifest.supports_podcast_dht());
|
assert!(manifest.supports_podcast_dht());
|
||||||
|
manifest.validate().unwrap();
|
||||||
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
|
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
|
||||||
assert!(!without.supports_podcast_dht());
|
assert!(!without.supports_podcast_dht());
|
||||||
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
|
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
|
||||||
|
|||||||
@@ -396,6 +396,19 @@ impl PodcastCard {
|
|||||||
card_index_tokens(&self.title, self.author.as_deref())
|
card_index_tokens(&self.title, self.author.as_deref())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url`
|
||||||
|
/// when the feed has not moved). A valid card's id hashes from it, so it
|
||||||
|
/// is the only URL of the card bound to the podcast.
|
||||||
|
pub fn origin_feed_url(&self) -> &FeedUrl {
|
||||||
|
self.id_feed_url.as_ref().unwrap_or(&self.feed_url)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether `feed_url` is the URL the id is derived from. Otherwise
|
||||||
|
/// `feed_url` is an unverified claim of the publisher that the feed moved.
|
||||||
|
pub fn feed_url_is_origin(&self) -> bool {
|
||||||
|
self.id_feed_url.is_none()
|
||||||
|
}
|
||||||
|
|
||||||
/// Canonical signing input (see `PODCASTS.md`).
|
/// Canonical signing input (see `PODCASTS.md`).
|
||||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||||
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
|
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
|
||||||
@@ -735,18 +748,6 @@ pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord)
|
|||||||
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
|
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether `winner` makes `other` obsolete everywhere, not just under one key:
|
|
||||||
/// `other` is a card of a lower revision, or `winner` is a tombstone of an
|
|
||||||
/// equal or higher revision.
|
|
||||||
pub(crate) fn supersedes_everywhere(
|
|
||||||
winner: &SignedPodcastRecord,
|
|
||||||
other: &SignedPodcastRecord,
|
|
||||||
) -> bool {
|
|
||||||
!other.is_tombstone()
|
|
||||||
&& (other.revision() < winner.revision()
|
|
||||||
|| (winner.is_tombstone() && other.revision() <= winner.revision()))
|
|
||||||
}
|
|
||||||
|
|
||||||
struct Canonical(Vec<u8>);
|
struct Canonical(Vec<u8>);
|
||||||
|
|
||||||
impl Canonical {
|
impl Canonical {
|
||||||
|
|||||||
@@ -27,6 +27,13 @@
|
|||||||
//! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with
|
//! [`crate::similarity_dht::SimilarityDht`]: declare [`PODCAST_DHT_ALPN`] with
|
||||||
//! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take
|
//! [`crate::MusicDhtConfigBuilder::schema_independent_stream_protocol`], take
|
||||||
//! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`].
|
//! its acceptor and spawn [`PodcastDht::serve`] and [`PodcastDht::maintenance`].
|
||||||
|
//! Advertise the overlay with [`CapabilityManifest::with_podcast_dht`] (it is
|
||||||
|
//! opt-in) and pass peer manifests to [`PodcastDht::note_peer_capabilities`].
|
||||||
|
//!
|
||||||
|
//! Only the URL a podcast id hashes from is bound to the podcast; search hits
|
||||||
|
//! offer that URL ([`PodcastSearchHit::feed_url`]) and report moved-feed URLs
|
||||||
|
//! as unverified claims. The publisher key is the device's transport identity
|
||||||
|
//! and therefore linkable to its network address and music-DHT records.
|
||||||
|
|
||||||
mod card;
|
mod card;
|
||||||
mod privacy;
|
mod privacy;
|
||||||
@@ -36,7 +43,7 @@ mod store;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||||
@@ -69,10 +76,16 @@ pub use privacy::{
|
|||||||
FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning,
|
FeedUrlPrivacyWarning, PRIVATE_FEED_HOSTS, SECRET_QUERY_WORDS, feed_url_privacy_warning,
|
||||||
};
|
};
|
||||||
pub use search::{
|
pub use search::{
|
||||||
FederatedSearchHit, MAX_HIT_PUBLISHERS, MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit,
|
ClaimedFeedUrl, FederatedSearchHit, MAX_HIT_FEED_URL_CLAIMS, MAX_HIT_PUBLISHERS,
|
||||||
SearchHitKind, merge_search_results, podcast_query_tokens,
|
MAX_PODCAST_QUERY_TOKENS, PodcastSearchHit, SearchHitKind, merge_search_results,
|
||||||
|
podcast_query_tokens,
|
||||||
|
};
|
||||||
|
pub use store::{
|
||||||
|
MAX_PERSISTED_PODCAST_DHT_PEERS, MAX_PODCAST_CACHE_BYTES, MAX_PODCAST_CACHE_RECORDS,
|
||||||
|
MAX_PODCAST_RECORDS_PER_KEY, MAX_PODCAST_RECORDS_PER_PUBLISHER,
|
||||||
|
MAX_PODCAST_RESPONSE_TOMBSTONES, MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||||
|
PODCAST_DHT_CACHE_SCHEMA_VERSION,
|
||||||
};
|
};
|
||||||
pub use store::{MAX_PODCAST_RECORDS_PER_KEY, PODCAST_DHT_CACHE_SCHEMA_VERSION};
|
|
||||||
|
|
||||||
use card::publication_error;
|
use card::publication_error;
|
||||||
use search::{aggregate_hits, rank_hits};
|
use search::{aggregate_hits, rank_hits};
|
||||||
@@ -105,6 +118,16 @@ pub const PODCAST_BOOTSTRAP_INTERVAL: Duration = Duration::from_secs(10 * 60);
|
|||||||
/// Maximum inbound requests processed concurrently; further streams are
|
/// Maximum inbound requests processed concurrently; further streams are
|
||||||
/// dropped.
|
/// dropped.
|
||||||
pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64;
|
pub const MAX_CONCURRENT_PODCAST_DHT_REQUESTS: usize = 64;
|
||||||
|
/// Maximum inbound requests of one peer processed concurrently; further
|
||||||
|
/// streams of that peer are dropped.
|
||||||
|
pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8;
|
||||||
|
/// Maximum distinct `(podcast_id, publisher)` records one lookup keeps;
|
||||||
|
/// records of further owners are ignored.
|
||||||
|
pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512;
|
||||||
|
/// A node stores a placement only when fewer than this many of its known
|
||||||
|
/// overlay contacts are closer to the key than itself (`2 * K`: the `K`
|
||||||
|
/// closest with slack for differing routing views).
|
||||||
|
pub const PODCAST_STORE_RESPONSIBILITY_WINDOW: usize = 2 * K;
|
||||||
/// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is
|
/// How long a peer whose capability manifest lacks [`PODCAST_DHT_ID`] is
|
||||||
/// not contacted.
|
/// not contacted.
|
||||||
pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60);
|
pub const UNSUPPORTED_PEER_BACKOFF: Duration = Duration::from_secs(60 * 60);
|
||||||
@@ -228,7 +251,7 @@ pub struct PodcastDhtPublishStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Result of [`PodcastDht::sync_shares`].
|
/// Result of [`PodcastDht::sync_shares`].
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
pub struct PodcastDhtSyncStats {
|
pub struct PodcastDhtSyncStats {
|
||||||
/// Cards published with a new revision, or refreshed because they used
|
/// Cards published with a new revision, or refreshed because they used
|
||||||
/// half their lifetime.
|
/// half their lifetime.
|
||||||
@@ -237,10 +260,38 @@ pub struct PodcastDhtSyncStats {
|
|||||||
pub unchanged: usize,
|
pub unchanged: usize,
|
||||||
/// Podcasts unpublished with a tombstone.
|
/// Podcasts unpublished with a tombstone.
|
||||||
pub unpublished: usize,
|
pub unpublished: usize,
|
||||||
|
/// Shared podcasts not published because their card has privacy
|
||||||
|
/// warnings the user has not confirmed. The UI asks, then calls
|
||||||
|
/// [`PodcastDht::publish`] or `sync_shares` with
|
||||||
|
/// [`PodcastShareState::privacy_confirmed`]. An earlier publication of
|
||||||
|
/// the podcast stays as it was.
|
||||||
|
pub held_back: Vec<PodcastPrivacyHold>,
|
||||||
|
/// Podcasts whose card could not be built or signed. They are not
|
||||||
|
/// published, and an earlier publication of them is unpublished.
|
||||||
|
pub failed: Vec<PodcastSyncFailure>,
|
||||||
/// Network statistics of the pushed records.
|
/// Network statistics of the pushed records.
|
||||||
pub push: PodcastDhtPublishStats,
|
pub push: PodcastDhtPublishStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A shared podcast held back by [`PodcastDht::sync_shares`] until the user
|
||||||
|
/// confirms publishing URLs that look private.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct PodcastPrivacyHold {
|
||||||
|
/// Podcast.
|
||||||
|
pub podcast_id: PodcastId,
|
||||||
|
/// Warnings of [`PodcastCardSpec::privacy_warnings`].
|
||||||
|
pub warnings: Vec<FeedUrlPrivacyWarning>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A podcast [`PodcastDht::sync_shares`] could not publish or unpublish.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct PodcastSyncFailure {
|
||||||
|
/// Podcast.
|
||||||
|
pub podcast_id: PodcastId,
|
||||||
|
/// Diagnostic.
|
||||||
|
pub error: String,
|
||||||
|
}
|
||||||
|
|
||||||
/// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`].
|
/// Personal registers of one podcast, as input to [`PodcastDht::sync_shares`].
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct PodcastShareState {
|
pub struct PodcastShareState {
|
||||||
@@ -251,6 +302,11 @@ pub struct PodcastShareState {
|
|||||||
/// Card to publish when shared; `None` derives it with
|
/// Card to publish when shared; `None` derives it with
|
||||||
/// [`PodcastCardSpec::from_subscription`].
|
/// [`PodcastCardSpec::from_subscription`].
|
||||||
pub card: Option<PodcastCardSpec>,
|
pub card: Option<PodcastCardSpec>,
|
||||||
|
/// Whether the user confirmed publishing this card's feed URLs despite
|
||||||
|
/// its [`PodcastCardSpec::privacy_warnings`]. Without it, a card with
|
||||||
|
/// warnings is only published when this device already publishes the
|
||||||
|
/// same feed URLs.
|
||||||
|
pub privacy_confirmed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PodcastShareState {
|
impl PodcastShareState {
|
||||||
@@ -258,6 +314,22 @@ impl PodcastShareState {
|
|||||||
pub fn is_shared(&self) -> bool {
|
pub fn is_shared(&self) -> bool {
|
||||||
effective_share(Some(&self.subscription), self.share.as_ref())
|
effective_share(Some(&self.subscription), self.share.as_ref())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The bounded, validated card [`PodcastDht::sync_shares`] would
|
||||||
|
/// publish: [`Self::card`] or one derived from the subscription.
|
||||||
|
pub fn card_spec(&self) -> Result<PodcastCardSpec> {
|
||||||
|
let card = match &self.card {
|
||||||
|
Some(card) => card.bounded(),
|
||||||
|
None => PodcastCardSpec::from_subscription(&self.subscription)?,
|
||||||
|
};
|
||||||
|
if card.podcast_id != self.subscription.podcast_id {
|
||||||
|
return Err(publication_error(
|
||||||
|
"card and subscription describe different podcasts",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
card.validate()?;
|
||||||
|
Ok(card)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// This device's publication state of one podcast.
|
/// This device's publication state of one podcast.
|
||||||
@@ -284,6 +356,7 @@ pub struct PodcastDht {
|
|||||||
failed: Mutex<HashMap<EndpointId, u64>>,
|
failed: Mutex<HashMap<EndpointId, u64>>,
|
||||||
publish_gate: Semaphore,
|
publish_gate: Semaphore,
|
||||||
serve_limit: Arc<Semaphore>,
|
serve_limit: Arc<Semaphore>,
|
||||||
|
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for PodcastDht {
|
impl std::fmt::Debug for PodcastDht {
|
||||||
@@ -330,6 +403,7 @@ impl PodcastDht {
|
|||||||
failed: Mutex::new(HashMap::new()),
|
failed: Mutex::new(HashMap::new()),
|
||||||
publish_gate: Semaphore::new(1),
|
publish_gate: Semaphore::new(1),
|
||||||
serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)),
|
serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)),
|
||||||
|
inflight: Arc::new(Mutex::new(HashMap::new())),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -341,6 +415,10 @@ impl PodcastDht {
|
|||||||
/// Serves authenticated overlay requests until `acceptor` closes.
|
/// Serves authenticated overlay requests until `acceptor` closes.
|
||||||
pub async fn serve(self: Arc<Self>, mut acceptor: StreamAcceptor) {
|
pub async fn serve(self: Arc<Self>, mut acceptor: StreamAcceptor) {
|
||||||
while let Some(stream) = acceptor.accept().await {
|
while let Some(stream) = acceptor.accept().await {
|
||||||
|
let Some(slot) = PeerSlot::acquire(&self.inflight, stream.peer_id) else {
|
||||||
|
debug!(peer = %stream.peer_id, "podcast DHT request dropped: peer limit");
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else {
|
let Ok(permit) = Arc::clone(&self.serve_limit).try_acquire_owned() else {
|
||||||
debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded");
|
debug!(peer = %stream.peer_id, "podcast DHT request dropped: overloaded");
|
||||||
continue;
|
continue;
|
||||||
@@ -348,6 +426,7 @@ impl PodcastDht {
|
|||||||
let this = Arc::clone(&self);
|
let this = Arc::clone(&self);
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _permit = permit;
|
let _permit = permit;
|
||||||
|
let _slot = slot;
|
||||||
let peer = stream.peer_id;
|
let peer = stream.peer_id;
|
||||||
match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await {
|
match tokio::time::timeout(SERVE_TIMEOUT, this.serve_one(stream)).await {
|
||||||
Ok(Ok(())) => {}
|
Ok(Ok(())) => {}
|
||||||
@@ -421,7 +500,10 @@ impl PodcastDht {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Discovers overlay peers among the routing table and the music DHT
|
/// Discovers overlay peers among the routing table and the music DHT
|
||||||
/// contacts.
|
/// contacts with one `FindNode` each. Music-DHT contacts whose capability
|
||||||
|
/// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are
|
||||||
|
/// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become
|
||||||
|
/// overlay contacts when they answer.
|
||||||
pub async fn bootstrap(&self) -> Result<()> {
|
pub async fn bootstrap(&self) -> Result<()> {
|
||||||
let known_feature: HashSet<EndpointId> = lock(&self.routing)
|
let known_feature: HashSet<EndpointId> = lock(&self.routing)
|
||||||
.contacts()
|
.contacts()
|
||||||
@@ -521,44 +603,92 @@ impl PodcastDht {
|
|||||||
/// [`Self::republish`]). Every other podcast this device published,
|
/// [`Self::republish`]). Every other podcast this device published,
|
||||||
/// including podcasts absent from `states`, is unpublished. The call is
|
/// including podcasts absent from `states`, is unpublished. The call is
|
||||||
/// idempotent: repeating it publishes nothing new.
|
/// idempotent: repeating it publishes nothing new.
|
||||||
|
///
|
||||||
|
/// Each podcast is handled on its own, so revocation is fail-safe: a
|
||||||
|
/// shared podcast whose card cannot be built is reported in
|
||||||
|
/// [`PodcastDhtSyncStats::failed`] and unpublished, and never prevents
|
||||||
|
/// other podcasts from being unpublished. A card with
|
||||||
|
/// [`PodcastCardSpec::privacy_warnings`] is held back
|
||||||
|
/// ([`PodcastDhtSyncStats::held_back`]) unless
|
||||||
|
/// [`PodcastShareState::privacy_confirmed`] is set or this device already
|
||||||
|
/// publishes the same feed URLs.
|
||||||
pub async fn sync_shares(&self, states: Vec<PodcastShareState>) -> Result<PodcastDhtSyncStats> {
|
pub async fn sync_shares(&self, states: Vec<PodcastShareState>) -> Result<PodcastDhtSyncStats> {
|
||||||
let mut desired: HashMap<PodcastId, PodcastCardSpec> = HashMap::new();
|
let _permit = self.publish_permit().await?;
|
||||||
for state in states {
|
let local = self.database.local_publications().await?;
|
||||||
|
let active: HashMap<&PodcastId, &PodcastCardSpec> = local
|
||||||
|
.iter()
|
||||||
|
.filter_map(|publication| {
|
||||||
|
publication
|
||||||
|
.spec
|
||||||
|
.as_ref()
|
||||||
|
.map(|spec| (&publication.podcast_id, spec))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut stats = PodcastDhtSyncStats::default();
|
||||||
|
let mut desired: BTreeMap<PodcastId, PodcastCardSpec> = BTreeMap::new();
|
||||||
|
let mut held: HashSet<PodcastId> = HashSet::new();
|
||||||
|
for state in &states {
|
||||||
if !state.is_shared() {
|
if !state.is_shared() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let card = match state.card {
|
let podcast_id = state.subscription.podcast_id.clone();
|
||||||
Some(card) => card.bounded(),
|
let card = match state.card_spec() {
|
||||||
None => PodcastCardSpec::from_subscription(&state.subscription)?,
|
Ok(card) => card,
|
||||||
|
Err(error) => {
|
||||||
|
debug!(%podcast_id, %error, "podcast card cannot be published");
|
||||||
|
stats.failed.push(PodcastSyncFailure {
|
||||||
|
podcast_id,
|
||||||
|
error: error.to_string(),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
if card.podcast_id != state.subscription.podcast_id {
|
let warnings = card.privacy_warnings();
|
||||||
return Err(publication_error(
|
let already_published = active.get(&podcast_id).is_some_and(|published| {
|
||||||
"card and subscription describe different podcasts",
|
published.feed_url == card.feed_url && published.id_feed_url == card.id_feed_url
|
||||||
));
|
});
|
||||||
|
if !warnings.is_empty() && !state.privacy_confirmed && !already_published {
|
||||||
|
held.insert(podcast_id.clone());
|
||||||
|
stats.held_back.push(PodcastPrivacyHold {
|
||||||
|
podcast_id,
|
||||||
|
warnings,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
card.validate()?;
|
desired.insert(podcast_id, card);
|
||||||
desired.insert(card.podcast_id.clone(), card);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let _permit = self.publish_permit().await?;
|
|
||||||
let local = self.database.local_publications().await?;
|
|
||||||
let mut stats = PodcastDhtSyncStats::default();
|
|
||||||
let mut records = Vec::new();
|
let mut records = Vec::new();
|
||||||
for publication in &local {
|
for publication in &local {
|
||||||
if publication.spec.is_some() && !desired.contains_key(&publication.podcast_id) {
|
let podcast_id = &publication.podcast_id;
|
||||||
records.extend(self.unpublish_locked(&publication.podcast_id).await?);
|
if publication.spec.is_none()
|
||||||
stats.unpublished += 1;
|
|| desired.contains_key(podcast_id)
|
||||||
|
|| held.contains(podcast_id)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match self.unpublish_locked(podcast_id).await {
|
||||||
|
Ok(record) => {
|
||||||
|
records.extend(record);
|
||||||
|
stats.unpublished += 1;
|
||||||
|
}
|
||||||
|
Err(error) => stats.failed.push(PodcastSyncFailure {
|
||||||
|
podcast_id: podcast_id.clone(),
|
||||||
|
error: error.to_string(),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let mut desired: Vec<_> = desired.into_values().collect();
|
for (podcast_id, card) in desired {
|
||||||
desired.sort_by(|left, right| left.podcast_id.cmp(&right.podcast_id));
|
match self.sign_card_locked(card, false).await {
|
||||||
for card in desired {
|
Ok(Some(record)) => {
|
||||||
match self.sign_card_locked(card, false).await? {
|
|
||||||
Some(record) => {
|
|
||||||
records.push(record);
|
records.push(record);
|
||||||
stats.published += 1;
|
stats.published += 1;
|
||||||
}
|
}
|
||||||
None => stats.unchanged += 1,
|
Ok(None) => stats.unchanged += 1,
|
||||||
|
Err(error) => stats.failed.push(PodcastSyncFailure {
|
||||||
|
podcast_id,
|
||||||
|
error: error.to_string(),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
stats.push = self.push(records).await?;
|
stats.push = self.push(records).await?;
|
||||||
@@ -613,8 +743,8 @@ impl PodcastDht {
|
|||||||
/// normalized token of `query` ([`podcast_query_tokens`]).
|
/// normalized token of `query` ([`podcast_query_tokens`]).
|
||||||
///
|
///
|
||||||
/// Hits are merged by `podcast_id` over publishers and ranked: exact
|
/// Hits are merged by `podcast_id` over publishers and ranked: exact
|
||||||
/// normalized title first, then more publishers. Partial results are
|
/// normalized title first, then hits with a card under the origin feed
|
||||||
/// returned when peers fail.
|
/// URL. Partial results are returned when peers fail.
|
||||||
pub async fn search(&self, query: &str, limit: usize) -> Result<Vec<PodcastSearchHit>> {
|
pub async fn search(&self, query: &str, limit: usize) -> Result<Vec<PodcastSearchHit>> {
|
||||||
let tokens = podcast_query_tokens(query);
|
let tokens = podcast_query_tokens(query);
|
||||||
if tokens.is_empty() || limit == 0 {
|
if tokens.is_empty() || limit == 0 {
|
||||||
@@ -966,8 +1096,25 @@ impl PodcastDht {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
PodcastDhtOperation::StoreBatch { records } => {
|
PodcastDhtOperation::StoreBatch { records } => {
|
||||||
let entries = validate_store_batch(records, &self.network_id, now_ms())?;
|
let entries =
|
||||||
let stored = self.database.store(entries, now_ms()).await?;
|
validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?;
|
||||||
|
let nodes: Vec<NodeId> = lock(&self.routing)
|
||||||
|
.contacts()
|
||||||
|
.into_iter()
|
||||||
|
.map(|contact| contact.node_id)
|
||||||
|
.collect();
|
||||||
|
let entries: Vec<_> = entries
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(mut keys, record)| {
|
||||||
|
keys.retain(|key| is_responsible(&self.own_contact.node_id, &nodes, key));
|
||||||
|
(!keys.is_empty()).then_some((keys, record))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let stored = if entries.is_empty() {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
self.database.store(entries, now_ms()).await?
|
||||||
|
};
|
||||||
Ok(PodcastDhtResponse::success(
|
Ok(PodcastDhtResponse::success(
|
||||||
Vec::new(),
|
Vec::new(),
|
||||||
Vec::new(),
|
Vec::new(),
|
||||||
@@ -1099,11 +1246,17 @@ impl PodcastDht {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Validates a store batch received from a peer: bounds, every record's
|
/// Validates a store batch received from `requester`: bounds, that every
|
||||||
/// signature and lifetime, and that every key belongs to its record.
|
/// record is published by the requester itself, every record's signature and
|
||||||
|
/// lifetime, and that every key belongs to its record.
|
||||||
|
///
|
||||||
|
/// Replicas never forward records, so only a record's publisher stores it.
|
||||||
|
/// Nobody can therefore replay or inject another device's records, and
|
||||||
|
/// per-publisher quotas apply to the storing peer.
|
||||||
fn validate_store_batch(
|
fn validate_store_batch(
|
||||||
entries: Vec<PodcastStoreEntry>,
|
entries: Vec<PodcastStoreEntry>,
|
||||||
network_id: &NetworkId,
|
network_id: &NetworkId,
|
||||||
|
requester: EndpointId,
|
||||||
now_ms: u64,
|
now_ms: u64,
|
||||||
) -> Result<Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>> {
|
) -> Result<Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>> {
|
||||||
if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS {
|
if entries.is_empty() || entries.len() > MAX_PODCAST_STORE_RECORDS {
|
||||||
@@ -1117,6 +1270,11 @@ fn validate_store_batch(
|
|||||||
if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() {
|
if keys.is_empty() || keys.len() > MAX_RECORD_DHT_KEYS || unique.len() != keys.len() {
|
||||||
return Err(protocol_error("invalid podcast DHT store keys"));
|
return Err(protocol_error("invalid podcast DHT store keys"));
|
||||||
}
|
}
|
||||||
|
if entry.record.publisher() != requester {
|
||||||
|
return Err(protocol_error(
|
||||||
|
"podcast records must be stored by their publisher",
|
||||||
|
));
|
||||||
|
}
|
||||||
entry.record.verify_at(network_id, now_ms)?;
|
entry.record.verify_at(network_id, now_ms)?;
|
||||||
let valid_keys = entry.record.dht_keys();
|
let valid_keys = entry.record.dht_keys();
|
||||||
if keys.iter().any(|key| !valid_keys.contains(key)) {
|
if keys.iter().any(|key| !valid_keys.contains(key)) {
|
||||||
@@ -1128,6 +1286,18 @@ fn validate_store_batch(
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a node is among the nodes that should store `key`: fewer than
|
||||||
|
/// [`PODCAST_STORE_RESPONSIBILITY_WINDOW`] known contacts are closer.
|
||||||
|
fn is_responsible(own: &NodeId, contacts: &[NodeId], key: &PodcastDhtKey) -> bool {
|
||||||
|
let own_distance = distance(own.as_bytes(), key.as_bytes());
|
||||||
|
contacts
|
||||||
|
.iter()
|
||||||
|
.filter(|node| distance(node.as_bytes(), key.as_bytes()) < own_distance)
|
||||||
|
.take(PODCAST_STORE_RESPONSIBILITY_WINDOW)
|
||||||
|
.count()
|
||||||
|
< PODCAST_STORE_RESPONSIBILITY_WINDOW
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
|
fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
|
||||||
if tokens.len() > MAX_PODCAST_QUERY_TOKENS
|
if tokens.len() > MAX_PODCAST_QUERY_TOKENS
|
||||||
|| tokens
|
|| tokens
|
||||||
@@ -1139,7 +1309,8 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adds verified records found under `key` to the per-owner winners.
|
/// Adds verified records found under `key` to the per-owner winners, keeping
|
||||||
|
/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners.
|
||||||
fn merge_records(
|
fn merge_records(
|
||||||
winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>,
|
winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>,
|
||||||
records: Vec<SignedPodcastRecord>,
|
records: Vec<SignedPodcastRecord>,
|
||||||
@@ -1161,6 +1332,7 @@ fn merge_records(
|
|||||||
let owner = (record.podcast_id().clone(), record.publisher());
|
let owner = (record.podcast_id().clone(), record.publisher());
|
||||||
match winners.get(&owner) {
|
match winners.get(&owner) {
|
||||||
Some(current) if compare_records(current, &record).is_ge() => {}
|
Some(current) if compare_records(current, &record).is_ge() => {}
|
||||||
|
None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {}
|
||||||
_ => {
|
_ => {
|
||||||
winners.insert(owner, record);
|
winners.insert(owner, record);
|
||||||
}
|
}
|
||||||
@@ -1168,6 +1340,42 @@ fn merge_records(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One in-flight inbound request of a peer; released on drop.
|
||||||
|
struct PeerSlot {
|
||||||
|
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
|
||||||
|
peer: EndpointId,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PeerSlot {
|
||||||
|
fn acquire(
|
||||||
|
inflight: &Arc<Mutex<HashMap<EndpointId, usize>>>,
|
||||||
|
peer: EndpointId,
|
||||||
|
) -> Option<Self> {
|
||||||
|
let mut map = lock(inflight);
|
||||||
|
let count = map.entry(peer).or_default();
|
||||||
|
if *count >= MAX_PODCAST_DHT_REQUESTS_PER_PEER {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
*count += 1;
|
||||||
|
Some(Self {
|
||||||
|
inflight: Arc::clone(inflight),
|
||||||
|
peer,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for PeerSlot {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let mut map = lock(&self.inflight);
|
||||||
|
if let Some(count) = map.get_mut(&self.peer) {
|
||||||
|
*count = count.saturating_sub(1);
|
||||||
|
if *count == 0 {
|
||||||
|
map.remove(&self.peer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_contact(
|
fn validate_contact(
|
||||||
contact: &NodeContact,
|
contact: &NodeContact,
|
||||||
network_id: &NetworkId,
|
network_id: &NetworkId,
|
||||||
|
|||||||
@@ -8,15 +8,17 @@ use federation_net::EndpointId;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::normalization::{normalize_name, tokenize};
|
use crate::normalization::{normalize_name, tokenize};
|
||||||
use crate::podcast::PodcastId;
|
use crate::podcast::{FeedUrl, PodcastId};
|
||||||
use crate::record::{ItemKind, LibraryItem};
|
use crate::record::{ItemKind, LibraryItem};
|
||||||
|
|
||||||
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
|
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
|
||||||
|
|
||||||
/// Maximum distinct query tokens used by a podcast text search.
|
/// Maximum distinct query tokens used by a podcast text search.
|
||||||
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
|
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
|
||||||
/// Maximum publishers reported per hit.
|
/// Maximum publishers reported per hit and per claimed feed URL.
|
||||||
pub const MAX_HIT_PUBLISHERS: usize = 64;
|
pub const MAX_HIT_PUBLISHERS: usize = 64;
|
||||||
|
/// Maximum claimed moved feed URLs reported per hit.
|
||||||
|
pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8;
|
||||||
|
|
||||||
/// Kind of a federated search hit.
|
/// Kind of a federated search hit.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
@@ -64,18 +66,35 @@ impl From<ItemKind> for SearchHitKind {
|
|||||||
///
|
///
|
||||||
/// Cards of the same podcast published by several devices are merged by
|
/// Cards of the same podcast published by several devices are merged by
|
||||||
/// `podcast_id`. The hit carries no personal state of any publisher.
|
/// `podcast_id`. The hit carries no personal state of any publisher.
|
||||||
|
///
|
||||||
|
/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every
|
||||||
|
/// other field is a claim of some publisher device, and device identities are
|
||||||
|
/// free, so neither the metadata nor the number of publishers is a trust
|
||||||
|
/// signal.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct PodcastSearchHit {
|
pub struct PodcastSearchHit {
|
||||||
/// Podcast.
|
/// Podcast.
|
||||||
pub podcast_id: PodcastId,
|
pub podcast_id: PodcastId,
|
||||||
/// Representative card: the most common `feed_url` among the
|
/// The URL `podcast_id` is derived from
|
||||||
/// publishers' current cards, then the most recently issued card, then
|
/// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with
|
||||||
/// the greatest publisher key.
|
/// this URL; follow the feed's own HTTP redirects and
|
||||||
|
/// `itunes:new-feed-url` from there.
|
||||||
|
pub feed_url: FeedUrl,
|
||||||
|
/// Representative card for display: a card whose `feed_url` is the
|
||||||
|
/// origin URL when one exists, then the title most publishers agree on,
|
||||||
|
/// then the smallest publisher key. Its metadata is a publisher's claim.
|
||||||
pub card: PodcastCard,
|
pub card: PodcastCard,
|
||||||
/// Sorted distinct publisher devices with a current valid card, at most
|
/// Sorted distinct publisher devices with a current valid card, at most
|
||||||
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
|
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
|
||||||
/// one user may publish from several devices.
|
/// one user may publish from several devices, and anyone can mint
|
||||||
|
/// devices.
|
||||||
pub publishers: Vec<EndpointId>,
|
pub publishers: Vec<EndpointId>,
|
||||||
|
/// Feed URLs other than [`Self::feed_url`] that publishers claim the
|
||||||
|
/// podcast moved to, most claimed first, at most
|
||||||
|
/// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this
|
||||||
|
/// podcast or record it as an alias unless the feed at
|
||||||
|
/// [`Self::feed_url`] itself redirects there.
|
||||||
|
pub claimed_feed_urls: Vec<ClaimedFeedUrl>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PodcastSearchHit {
|
impl PodcastSearchHit {
|
||||||
@@ -85,6 +104,16 @@ impl PodcastSearchHit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An unverified moved-feed URL claimed by some publishers of a podcast.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct ClaimedFeedUrl {
|
||||||
|
/// Claimed current feed URL.
|
||||||
|
pub feed_url: FeedUrl,
|
||||||
|
/// Sorted distinct publisher devices claiming it, at most
|
||||||
|
/// [`MAX_HIT_PUBLISHERS`].
|
||||||
|
pub publishers: Vec<EndpointId>,
|
||||||
|
}
|
||||||
|
|
||||||
/// A music or podcast result of a federated search.
|
/// A music or podcast result of a federated search.
|
||||||
///
|
///
|
||||||
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
|
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
|
||||||
@@ -184,37 +213,66 @@ pub(crate) fn aggregate_hits(
|
|||||||
by_podcast
|
by_podcast
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(podcast_id, cards)| {
|
.map(|(podcast_id, cards)| {
|
||||||
let mut feed_votes: HashMap<&str, usize> = HashMap::new();
|
let mut title_votes: HashMap<String, usize> = HashMap::new();
|
||||||
for card in &cards {
|
for card in &cards {
|
||||||
*feed_votes.entry(card.feed_url.as_str()).or_default() += 1;
|
*title_votes.entry(normalize_name(&card.title)).or_default() += 1;
|
||||||
}
|
}
|
||||||
|
// Publisher-controlled times and revisions never break ties.
|
||||||
let card = cards
|
let card = cards
|
||||||
.iter()
|
.iter()
|
||||||
.max_by_key(|card| {
|
.max_by_key(|card| {
|
||||||
(
|
(
|
||||||
feed_votes[card.feed_url.as_str()],
|
card.feed_url_is_origin(),
|
||||||
card.issued_at_ms,
|
title_votes[&normalize_name(&card.title)],
|
||||||
*card.publisher.as_bytes(),
|
Reverse(*card.publisher.as_bytes()),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.cloned()
|
.cloned()
|
||||||
.expect("a group has at least one card");
|
.expect("a group has at least one card");
|
||||||
let mut publishers: Vec<EndpointId> = cards.iter().map(|card| card.publisher).collect();
|
let mut claims: BTreeMap<&FeedUrl, Vec<EndpointId>> = BTreeMap::new();
|
||||||
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
for card in cards.iter().filter(|card| !card.feed_url_is_origin()) {
|
||||||
publishers.dedup();
|
claims
|
||||||
publishers.truncate(MAX_HIT_PUBLISHERS);
|
.entry(&card.feed_url)
|
||||||
|
.or_default()
|
||||||
|
.push(card.publisher);
|
||||||
|
}
|
||||||
|
let mut claimed_feed_urls: Vec<ClaimedFeedUrl> = claims
|
||||||
|
.into_iter()
|
||||||
|
.map(|(feed_url, publishers)| ClaimedFeedUrl {
|
||||||
|
feed_url: feed_url.clone(),
|
||||||
|
publishers: sorted_publishers(publishers),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
claimed_feed_urls.sort_by(|left, right| {
|
||||||
|
right
|
||||||
|
.publishers
|
||||||
|
.len()
|
||||||
|
.cmp(&left.publishers.len())
|
||||||
|
.then_with(|| left.feed_url.cmp(&right.feed_url))
|
||||||
|
});
|
||||||
|
claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS);
|
||||||
PodcastSearchHit {
|
PodcastSearchHit {
|
||||||
podcast_id,
|
podcast_id,
|
||||||
|
feed_url: card.origin_feed_url().clone(),
|
||||||
|
publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()),
|
||||||
card,
|
card,
|
||||||
publishers,
|
claimed_feed_urls,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sorted_publishers(mut publishers: Vec<EndpointId>) -> Vec<EndpointId> {
|
||||||
|
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
||||||
|
publishers.dedup();
|
||||||
|
publishers.truncate(MAX_HIT_PUBLISHERS);
|
||||||
|
publishers
|
||||||
|
}
|
||||||
|
|
||||||
/// Keeps hits whose card is indexed under every query token and ranks
|
/// Keeps hits whose card is indexed under every query token and ranks
|
||||||
/// them: exact normalized title first, then more publishers, then title and
|
/// them: exact normalized title first, then hits with a card published under
|
||||||
/// id.
|
/// the origin feed URL, then title and id. The publisher count is not used:
|
||||||
|
/// anyone can inflate it with new device identities.
|
||||||
pub(crate) fn rank_hits(
|
pub(crate) fn rank_hits(
|
||||||
mut hits: Vec<PodcastSearchHit>,
|
mut hits: Vec<PodcastSearchHit>,
|
||||||
query: &str,
|
query: &str,
|
||||||
@@ -229,7 +287,7 @@ pub(crate) fn rank_hits(
|
|||||||
let title = normalize_name(&hit.card.title);
|
let title = normalize_name(&hit.card.title);
|
||||||
(
|
(
|
||||||
title != normalized_query,
|
title != normalized_query,
|
||||||
Reverse(hit.publishers.len()),
|
!hit.card.feed_url_is_origin(),
|
||||||
title,
|
title,
|
||||||
hit.podcast_id.clone(),
|
hit.podcast_id.clone(),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
//! (like the similarity-routing cache). Every record read from it is decoded
|
//! (like the similarity-routing cache). Every record read from it is decoded
|
||||||
//! and verified again before it is used or served.
|
//! and verified again before it is used or served.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashSet;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||||
@@ -17,32 +17,66 @@ use crate::error::{MusicDhtError, Result};
|
|||||||
use crate::podcast::PodcastId;
|
use crate::podcast::PodcastId;
|
||||||
use crate::routing::{NodeContact, NodeId};
|
use crate::routing::{NodeContact, NodeId};
|
||||||
|
|
||||||
use super::card::{
|
use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records};
|
||||||
PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records, supersedes_everywhere,
|
|
||||||
};
|
|
||||||
|
|
||||||
/// `PRAGMA user_version` of the podcast-discovery cache schema.
|
/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1
|
||||||
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 1;
|
/// (pre-release) stored one payload per key; its replica table is dropped on
|
||||||
/// Maximum distinct `(podcast_id, publisher)` records cached under one key.
|
/// open because the cache is replaceable. Own publications are kept.
|
||||||
/// When full, the record expiring first is evicted if the new one outlives it.
|
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2;
|
||||||
|
/// Maximum records (cards and tombstones) indexed under one key. A full key
|
||||||
|
/// admits no new `(podcast_id, publisher)` until a record expires: existing
|
||||||
|
/// records are never evicted for newcomers, and their owners keep refreshing
|
||||||
|
/// them in place.
|
||||||
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256;
|
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256;
|
||||||
|
/// Maximum tombstones newly indexed under one key. A tombstone is admitted
|
||||||
|
/// under a text key only in place of a record of the same
|
||||||
|
/// `(podcast_id, publisher)` already indexed there, so this budget applies to
|
||||||
|
/// the podcast-id key, where tombstones of unknown cards land.
|
||||||
|
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32;
|
||||||
|
/// Maximum tombstones in one `FindValue` response; cards fill the rest.
|
||||||
|
pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32;
|
||||||
|
/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a
|
||||||
|
/// replica.
|
||||||
|
pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256;
|
||||||
|
/// Maximum `(podcast_id, publisher)` records in a replica cache.
|
||||||
|
pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096;
|
||||||
|
/// Maximum total encoded size of the records in a replica cache.
|
||||||
|
pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024;
|
||||||
|
/// Maximum persisted overlay contacts; the least recently seen are dropped.
|
||||||
|
pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024;
|
||||||
|
|
||||||
const SCHEMA: &str = "
|
const SCHEMA: &str = "
|
||||||
CREATE TABLE IF NOT EXISTS podcast_dht_records (
|
CREATE TABLE IF NOT EXISTS podcast_dht_records (
|
||||||
dht_key BLOB NOT NULL,
|
podcast_id TEXT NOT NULL,
|
||||||
podcast_id TEXT NOT NULL,
|
publisher TEXT NOT NULL,
|
||||||
publisher TEXT NOT NULL,
|
tombstone INTEGER NOT NULL,
|
||||||
tombstone INTEGER NOT NULL,
|
expires_at_ms INTEGER NOT NULL,
|
||||||
expires_at_ms INTEGER NOT NULL,
|
payload_bytes INTEGER NOT NULL,
|
||||||
payload BLOB NOT NULL,
|
payload BLOB NOT NULL,
|
||||||
PRIMARY KEY (dht_key, podcast_id, publisher)
|
PRIMARY KEY (podcast_id, publisher)
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
|
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
|
||||||
ON podcast_dht_records(expires_at_ms);
|
ON podcast_dht_records(expires_at_ms);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_owner
|
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher
|
||||||
ON podcast_dht_records(podcast_id, publisher);
|
ON podcast_dht_records(publisher);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS podcast_dht_keys (
|
||||||
|
dht_key BLOB NOT NULL,
|
||||||
|
podcast_id TEXT NOT NULL,
|
||||||
|
publisher TEXT NOT NULL,
|
||||||
|
tombstone INTEGER NOT NULL,
|
||||||
|
expires_at_ms INTEGER NOT NULL,
|
||||||
|
first_stored_at_ms INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (dht_key, podcast_id, publisher)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner
|
||||||
|
ON podcast_dht_keys(podcast_id, publisher);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry
|
||||||
|
ON podcast_dht_keys(expires_at_ms);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
|
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
|
||||||
peer_id TEXT PRIMARY KEY,
|
peer_id TEXT PRIMARY KEY,
|
||||||
@@ -60,6 +94,28 @@ CREATE TABLE IF NOT EXISTS podcast_dht_local (
|
|||||||
);
|
);
|
||||||
";
|
";
|
||||||
|
|
||||||
|
/// Admission limits of a replica cache.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) struct StoreLimits {
|
||||||
|
pub per_key: usize,
|
||||||
|
pub tombstones_per_key: usize,
|
||||||
|
pub per_publisher: usize,
|
||||||
|
pub records: usize,
|
||||||
|
pub bytes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for StoreLimits {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
per_key: MAX_PODCAST_RECORDS_PER_KEY,
|
||||||
|
tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||||
|
per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER,
|
||||||
|
records: MAX_PODCAST_CACHE_RECORDS,
|
||||||
|
bytes: MAX_PODCAST_CACHE_BYTES,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// This device's latest publication of one podcast.
|
/// This device's latest publication of one podcast.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub(crate) struct LocalPublication {
|
pub(crate) struct LocalPublication {
|
||||||
@@ -76,6 +132,7 @@ pub(crate) struct LocalPublication {
|
|||||||
pub(crate) struct PodcastDhtDatabase {
|
pub(crate) struct PodcastDhtDatabase {
|
||||||
conn: Arc<Mutex<Connection>>,
|
conn: Arc<Mutex<Connection>>,
|
||||||
network_id: NetworkId,
|
network_id: NetworkId,
|
||||||
|
limits: StoreLimits,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for PodcastDhtDatabase {
|
impl std::fmt::Debug for PodcastDhtDatabase {
|
||||||
@@ -88,6 +145,14 @@ impl std::fmt::Debug for PodcastDhtDatabase {
|
|||||||
|
|
||||||
impl PodcastDhtDatabase {
|
impl PodcastDhtDatabase {
|
||||||
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
|
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
|
||||||
|
Self::open_with_limits(path, network_id, StoreLimits::default()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn open_with_limits(
|
||||||
|
path: &Path,
|
||||||
|
network_id: NetworkId,
|
||||||
|
limits: StoreLimits,
|
||||||
|
) -> Result<Self> {
|
||||||
let path = path.to_path_buf();
|
let path = path.to_path_buf();
|
||||||
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
|
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
|
||||||
let conn = Connection::open(&path).map_err(|error| {
|
let conn = Connection::open(&path).map_err(|error| {
|
||||||
@@ -101,6 +166,11 @@ impl PodcastDhtDatabase {
|
|||||||
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
|
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
if version == 1 {
|
||||||
|
// Replicas are replaceable; the old layout is dropped.
|
||||||
|
conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;")
|
||||||
|
.map_err(database_error)?;
|
||||||
|
}
|
||||||
conn.execute_batch(SCHEMA).map_err(|error| {
|
conn.execute_batch(SCHEMA).map_err(|error| {
|
||||||
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
|
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
|
||||||
})?;
|
})?;
|
||||||
@@ -113,37 +183,39 @@ impl PodcastDhtDatabase {
|
|||||||
Ok(Self {
|
Ok(Self {
|
||||||
conn: Arc::new(Mutex::new(conn)),
|
conn: Arc::new(Mutex::new(conn)),
|
||||||
network_id,
|
network_id,
|
||||||
|
limits,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call<F, T>(&self, operation: F) -> Result<T>
|
async fn call<F, T>(&self, operation: F) -> Result<T>
|
||||||
where
|
where
|
||||||
F: FnOnce(&Connection, &NetworkId) -> rusqlite::Result<T> + Send + 'static,
|
F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result<T> + Send + 'static,
|
||||||
T: Send + 'static,
|
T: Send + 'static,
|
||||||
{
|
{
|
||||||
let conn = Arc::clone(&self.conn);
|
let conn = Arc::clone(&self.conn);
|
||||||
let network_id = self.network_id;
|
let network_id = self.network_id;
|
||||||
|
let limits = self.limits;
|
||||||
tokio::task::spawn_blocking(move || {
|
tokio::task::spawn_blocking(move || {
|
||||||
let guard = lock(&conn);
|
let guard = lock(&conn);
|
||||||
operation(&guard, &network_id).map_err(database_error)
|
operation(&guard, &network_id, &limits).map_err(database_error)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
|
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stores records under the given keys. The caller has verified every
|
/// Stores records under the given keys. The caller has verified every
|
||||||
/// record and checked that each key belongs to it. Returns the number of
|
/// record, checked that each key belongs to it and that the sender is its
|
||||||
/// `(key, record)` rows written.
|
/// publisher. Returns the number of `(key, record)` placements written.
|
||||||
pub async fn store(
|
pub async fn store(
|
||||||
&self,
|
&self,
|
||||||
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
|
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
|
||||||
now_ms: u64,
|
now_ms: u64,
|
||||||
) -> Result<usize> {
|
) -> Result<usize> {
|
||||||
self.call(move |conn, network_id| {
|
self.call(move |conn, network_id, limits| {
|
||||||
let transaction = conn.unchecked_transaction()?;
|
let transaction = conn.unchecked_transaction()?;
|
||||||
let mut written = 0;
|
let mut written = 0;
|
||||||
for (keys, record) in entries {
|
for (keys, record) in entries {
|
||||||
written += store_one(&transaction, network_id, &keys, &record, now_ms)?;
|
written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?;
|
||||||
}
|
}
|
||||||
transaction.commit()?;
|
transaction.commit()?;
|
||||||
Ok(written)
|
Ok(written)
|
||||||
@@ -151,9 +223,11 @@ impl PodcastDhtDatabase {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Valid, unexpired records under `key`, newest expiry first. Cards must
|
/// Valid, unexpired records under `key`: at most
|
||||||
/// contain every token in `filter_tokens`; tombstones are always
|
/// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale
|
||||||
/// returned so stale replicas elsewhere can be overruled.
|
/// replicas elsewhere can be overruled) and cards containing every token
|
||||||
|
/// in `filter_tokens` up to `limit` records in all. Records indexed longest
|
||||||
|
/// come first. Cards precede tombstones.
|
||||||
pub async fn records_by_key(
|
pub async fn records_by_key(
|
||||||
&self,
|
&self,
|
||||||
key: PodcastDhtKey,
|
key: PodcastDhtKey,
|
||||||
@@ -161,52 +235,70 @@ impl PodcastDhtDatabase {
|
|||||||
now_ms: u64,
|
now_ms: u64,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
) -> Result<Vec<SignedPodcastRecord>> {
|
) -> Result<Vec<SignedPodcastRecord>> {
|
||||||
self.call(move |conn, network_id| {
|
self.call(move |conn, network_id, _| {
|
||||||
let mut statement = conn.prepare(
|
let mut statement = conn.prepare(
|
||||||
"SELECT payload FROM podcast_dht_records
|
"SELECT r.payload FROM podcast_dht_keys k
|
||||||
WHERE dht_key = ?1 AND expires_at_ms > ?2
|
JOIN podcast_dht_records r
|
||||||
ORDER BY expires_at_ms DESC, podcast_id, publisher",
|
ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher
|
||||||
|
WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3
|
||||||
|
ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher",
|
||||||
)?;
|
)?;
|
||||||
let rows = statement.query_map(
|
let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result<Vec<_>> {
|
||||||
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|
let mut records = Vec::new();
|
||||||
|row| row.get::<_, Vec<u8>>(0),
|
if limit == 0 {
|
||||||
)?;
|
return Ok(records);
|
||||||
let mut records = Vec::new();
|
|
||||||
for row in rows {
|
|
||||||
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
if !record.belongs_under(&key) {
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
let matches = record.card().is_none_or(|card| {
|
let rows = statement.query_map(
|
||||||
let tokens = card.index_tokens();
|
params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)],
|
||||||
filter_tokens.iter().all(|token| tokens.contains(token))
|
|row| row.get::<_, Vec<u8>>(0),
|
||||||
});
|
)?;
|
||||||
if matches {
|
for row in rows {
|
||||||
records.push(record);
|
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
||||||
if records.len() == limit {
|
continue;
|
||||||
break;
|
};
|
||||||
|
if record.is_tombstone() != tombstone || !record.belongs_under(&key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let matches = record.card().is_none_or(|card| {
|
||||||
|
let tokens = card.index_tokens();
|
||||||
|
filter_tokens.iter().all(|token| tokens.contains(token))
|
||||||
|
});
|
||||||
|
if matches {
|
||||||
|
records.push(record);
|
||||||
|
if records.len() == limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
Ok(records)
|
||||||
|
};
|
||||||
|
let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?;
|
||||||
|
let mut records = select(false, limit - tombstones.len())?;
|
||||||
|
records.extend(tombstones);
|
||||||
Ok(records)
|
Ok(records)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Removes expired records and their key placements. Returns the number
|
||||||
|
/// of records removed.
|
||||||
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
|
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
|
let now = clamp_i64(now_ms);
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1",
|
||||||
|
[now],
|
||||||
|
)?;
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
|
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
|
||||||
[clamp_i64(now_ms)],
|
[now],
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
|
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO podcast_dht_peers
|
"INSERT OR REPLACE INTO podcast_dht_peers
|
||||||
(peer_id, node_id, ticket, last_seen_ms)
|
(peer_id, node_id, ticket, last_seen_ms)
|
||||||
@@ -218,13 +310,20 @@ impl PodcastDhtDatabase {
|
|||||||
clamp_i64(contact.last_seen_ms),
|
clamp_i64(contact.last_seen_ms),
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM podcast_dht_peers WHERE peer_id IN (
|
||||||
|
SELECT peer_id FROM podcast_dht_peers
|
||||||
|
ORDER BY last_seen_ms DESC, peer_id
|
||||||
|
LIMIT -1 OFFSET ?1)",
|
||||||
|
[MAX_PERSISTED_PODCAST_DHT_PEERS as i64],
|
||||||
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
|
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
|
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
|
||||||
[peer.to_string()],
|
[peer.to_string()],
|
||||||
@@ -235,7 +334,7 @@ impl PodcastDhtDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
|
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
let mut statement =
|
let mut statement =
|
||||||
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
|
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
|
||||||
let rows = statement.query_map([], |row| {
|
let rows = statement.query_map([], |row| {
|
||||||
@@ -267,7 +366,7 @@ impl PodcastDhtDatabase {
|
|||||||
&self,
|
&self,
|
||||||
podcast_id: PodcastId,
|
podcast_id: PodcastId,
|
||||||
) -> Result<Option<LocalPublication>> {
|
) -> Result<Option<LocalPublication>> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||||
WHERE podcast_id = ?1",
|
WHERE podcast_id = ?1",
|
||||||
@@ -281,7 +380,7 @@ impl PodcastDhtDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
|
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
let mut statement = conn.prepare(
|
let mut statement = conn.prepare(
|
||||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||||
ORDER BY podcast_id",
|
ORDER BY podcast_id",
|
||||||
@@ -303,7 +402,7 @@ impl PodcastDhtDatabase {
|
|||||||
publication: LocalPublication,
|
publication: LocalPublication,
|
||||||
now_ms: u64,
|
now_ms: u64,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.call(move |conn, _| {
|
self.call(move |conn, _, _| {
|
||||||
let spec = publication
|
let spec = publication
|
||||||
.spec
|
.spec
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -332,141 +431,235 @@ impl PodcastDhtDatabase {
|
|||||||
fn store_one(
|
fn store_one(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
network_id: &NetworkId,
|
network_id: &NetworkId,
|
||||||
|
limits: &StoreLimits,
|
||||||
keys: &[PodcastDhtKey],
|
keys: &[PodcastDhtKey],
|
||||||
record: &SignedPodcastRecord,
|
record: &SignedPodcastRecord,
|
||||||
now_ms: u64,
|
now_ms: u64,
|
||||||
) -> rusqlite::Result<usize> {
|
) -> rusqlite::Result<usize> {
|
||||||
let podcast_id = record.podcast_id().as_str().to_string();
|
let podcast_id = record.podcast_id().as_str().to_string();
|
||||||
let publisher = record.publisher().to_string();
|
let publisher = record.publisher().to_string();
|
||||||
|
let now = clamp_i64(now_ms);
|
||||||
|
|
||||||
// Every cached version of this (podcast, publisher), re-verified.
|
// The cached record of this (podcast, publisher), re-verified.
|
||||||
let mut existing: HashMap<[u8; 32], SignedPodcastRecord> = HashMap::new();
|
let cached: Option<(Vec<u8>, i64)> = conn
|
||||||
let mut invalid_keys = Vec::new();
|
.query_row(
|
||||||
{
|
"SELECT payload, payload_bytes FROM podcast_dht_records
|
||||||
let mut statement = conn.prepare(
|
|
||||||
"SELECT dht_key, payload FROM podcast_dht_records
|
|
||||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||||
)?;
|
params![podcast_id, publisher],
|
||||||
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?))
|
)
|
||||||
})?;
|
.optional()?;
|
||||||
for row in rows {
|
let (existing, existing_bytes) = match cached {
|
||||||
let (key, payload) = row?;
|
Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) {
|
||||||
let decoded = decode_valid(&payload, network_id, now_ms);
|
Some(existing) => (Some(existing), bytes.max(0) as usize),
|
||||||
match (<[u8; 32]>::try_from(key.as_slice()), decoded) {
|
None => {
|
||||||
(Ok(key_bytes), Some(cached)) => {
|
delete_owner(conn, &podcast_id, &publisher)?;
|
||||||
existing.insert(key_bytes, cached);
|
(None, 0)
|
||||||
}
|
|
||||||
_ => invalid_keys.push(key),
|
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
None => {
|
||||||
|
// Stray placements without a record.
|
||||||
|
delete_owner(conn, &podcast_id, &publisher)?;
|
||||||
|
(None, 0)
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
for key in invalid_keys {
|
|
||||||
conn.execute(
|
|
||||||
"DELETE FROM podcast_dht_records
|
|
||||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
|
||||||
params![key, podcast_id, publisher],
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// A tombstone or a newer revision cached under any key wins everywhere.
|
// Higher revision, tombstone at an equal revision, later republish...
|
||||||
if existing
|
let replace = match &existing {
|
||||||
.values()
|
Some(existing) => match compare_records(existing, record) {
|
||||||
.any(|cached| supersedes_everywhere(cached, record))
|
std::cmp::Ordering::Greater => return Ok(0),
|
||||||
|
std::cmp::Ordering::Equal => false,
|
||||||
|
std::cmp::Ordering::Less => true,
|
||||||
|
},
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
||||||
|
if replace
|
||||||
|
&& !admits_record(
|
||||||
|
conn,
|
||||||
|
limits,
|
||||||
|
&publisher,
|
||||||
|
existing.is_none(),
|
||||||
|
existing_bytes,
|
||||||
|
payload.len(),
|
||||||
|
now,
|
||||||
|
)?
|
||||||
{
|
{
|
||||||
return Ok(0);
|
return Ok(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
let mut indexed: HashSet<[u8; 32]> = HashSet::new();
|
||||||
|
{
|
||||||
|
let mut statement = conn.prepare(
|
||||||
|
"SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||||
|
)?;
|
||||||
|
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
||||||
|
row.get::<_, Vec<u8>>(0)
|
||||||
|
})?;
|
||||||
|
for row in rows {
|
||||||
|
if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) {
|
||||||
|
indexed.insert(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut written = 0;
|
let mut written = 0;
|
||||||
|
if replace {
|
||||||
|
// Placements the new version no longer belongs under are dropped (a
|
||||||
|
// renamed card, or a tombstone that does not list the key); the others
|
||||||
|
// now serve the new version.
|
||||||
|
let belongs: HashSet<[u8; 32]> = record
|
||||||
|
.dht_keys()
|
||||||
|
.iter()
|
||||||
|
.map(|key| *key.as_bytes())
|
||||||
|
.collect();
|
||||||
|
let stale: Vec<[u8; 32]> = indexed
|
||||||
|
.iter()
|
||||||
|
.filter(|key| !belongs.contains(*key))
|
||||||
|
.copied()
|
||||||
|
.collect();
|
||||||
|
for key in stale {
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM podcast_dht_keys
|
||||||
|
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||||
|
params![key.as_slice(), podcast_id, publisher],
|
||||||
|
)?;
|
||||||
|
indexed.remove(&key);
|
||||||
|
}
|
||||||
|
written += conn.execute(
|
||||||
|
"UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4
|
||||||
|
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||||
|
params![
|
||||||
|
podcast_id,
|
||||||
|
publisher,
|
||||||
|
record.is_tombstone(),
|
||||||
|
clamp_i64(record.expires_at_ms()),
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// New placements. A tombstone only takes a new place under the podcast-id
|
||||||
|
// key: under a text key it may only replace a record of its owner, so
|
||||||
|
// nobody can index tombstones under someone else's search terms.
|
||||||
|
let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id());
|
||||||
for key in keys {
|
for key in keys {
|
||||||
if let Some(cached) = existing.get(key.as_bytes())
|
if indexed.contains(key.as_bytes()) {
|
||||||
&& compare_records(cached, record).is_ge()
|
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if !existing.contains_key(key.as_bytes()) && !make_room(conn, key, record, now_ms)? {
|
if record.is_tombstone() && *key != id_key {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !admits_placement(conn, limits, key, record.is_tombstone(), now)? {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO podcast_dht_records
|
"INSERT INTO podcast_dht_keys
|
||||||
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, payload)
|
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
ON CONFLICT(dht_key, podcast_id, publisher) DO UPDATE SET
|
|
||||||
tombstone = excluded.tombstone,
|
|
||||||
expires_at_ms = excluded.expires_at_ms,
|
|
||||||
payload = excluded.payload",
|
|
||||||
params![
|
params![
|
||||||
key.as_bytes().as_slice(),
|
key.as_bytes().as_slice(),
|
||||||
podcast_id,
|
podcast_id,
|
||||||
publisher,
|
publisher,
|
||||||
record.is_tombstone(),
|
record.is_tombstone(),
|
||||||
clamp_i64(record.expires_at_ms()),
|
clamp_i64(record.expires_at_ms()),
|
||||||
payload,
|
now,
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
|
indexed.insert(*key.as_bytes());
|
||||||
written += 1;
|
written += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop versions the new record makes obsolete under other keys too.
|
if indexed.is_empty() {
|
||||||
if written > 0 {
|
delete_owner(conn, &podcast_id, &publisher)?;
|
||||||
for (key, cached) in &existing {
|
return Ok(0);
|
||||||
if !keys.iter().any(|stored| stored.as_bytes() == key)
|
}
|
||||||
&& supersedes_everywhere(record, cached)
|
if replace {
|
||||||
{
|
conn.execute(
|
||||||
conn.execute(
|
"INSERT INTO podcast_dht_records
|
||||||
"DELETE FROM podcast_dht_records
|
(podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload)
|
||||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||||
params![key.as_slice(), podcast_id, publisher],
|
ON CONFLICT(podcast_id, publisher) DO UPDATE SET
|
||||||
)?;
|
tombstone = excluded.tombstone,
|
||||||
}
|
expires_at_ms = excluded.expires_at_ms,
|
||||||
}
|
payload_bytes = excluded.payload_bytes,
|
||||||
|
payload = excluded.payload",
|
||||||
|
params![
|
||||||
|
podcast_id,
|
||||||
|
publisher,
|
||||||
|
record.is_tombstone(),
|
||||||
|
clamp_i64(record.expires_at_ms()),
|
||||||
|
payload.len() as i64,
|
||||||
|
payload,
|
||||||
|
],
|
||||||
|
)?;
|
||||||
}
|
}
|
||||||
Ok(written)
|
Ok(written)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ensures there is room for one more record under `key`.
|
/// Whether the cache may hold the new version of a record: per-publisher and
|
||||||
fn make_room(
|
/// global record counts for a new owner, and the global byte budget.
|
||||||
|
fn admits_record(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
key: &PodcastDhtKey,
|
limits: &StoreLimits,
|
||||||
record: &SignedPodcastRecord,
|
publisher: &str,
|
||||||
now_ms: u64,
|
new_owner: bool,
|
||||||
|
previous_bytes: usize,
|
||||||
|
new_bytes: usize,
|
||||||
|
now: i64,
|
||||||
) -> rusqlite::Result<bool> {
|
) -> rusqlite::Result<bool> {
|
||||||
conn.execute(
|
let (records, bytes): (i64, i64) = conn.query_row(
|
||||||
"DELETE FROM podcast_dht_records WHERE dht_key = ?1 AND expires_at_ms <= ?2",
|
"SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records
|
||||||
params![key.as_bytes().as_slice(), clamp_i64(now_ms)],
|
WHERE expires_at_ms > ?1",
|
||||||
|
[now],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
)?;
|
)?;
|
||||||
let count: i64 = conn.query_row(
|
let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
|
||||||
"SELECT COUNT(*) FROM podcast_dht_records WHERE dht_key = ?1",
|
if bytes.saturating_add(new_bytes) > limits.bytes {
|
||||||
[key.as_bytes().as_slice()],
|
return Ok(false);
|
||||||
|row| row.get(0),
|
}
|
||||||
)?;
|
if !new_owner {
|
||||||
if (count as usize) < MAX_PODCAST_RECORDS_PER_KEY {
|
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
}
|
}
|
||||||
let victim: Option<(String, String, i64)> = conn
|
if records.max(0) as usize >= limits.records {
|
||||||
.query_row(
|
return Ok(false);
|
||||||
"SELECT podcast_id, publisher, expires_at_ms FROM podcast_dht_records
|
|
||||||
WHERE dht_key = ?1
|
|
||||||
ORDER BY expires_at_ms ASC, podcast_id, publisher
|
|
||||||
LIMIT 1",
|
|
||||||
[key.as_bytes().as_slice()],
|
|
||||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
|
||||||
)
|
|
||||||
.optional()?;
|
|
||||||
match victim {
|
|
||||||
Some((podcast_id, publisher, expires_at_ms))
|
|
||||||
if expires_at_ms < clamp_i64(record.expires_at_ms()) =>
|
|
||||||
{
|
|
||||||
conn.execute(
|
|
||||||
"DELETE FROM podcast_dht_records
|
|
||||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
|
||||||
params![key.as_bytes().as_slice(), podcast_id, publisher],
|
|
||||||
)?;
|
|
||||||
Ok(true)
|
|
||||||
}
|
|
||||||
_ => Ok(false),
|
|
||||||
}
|
}
|
||||||
|
let own: i64 = conn.query_row(
|
||||||
|
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
|
||||||
|
params![publisher, now],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
Ok((own.max(0) as usize) < limits.per_publisher)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether `key` has room for one more placement. Existing placements are
|
||||||
|
/// never evicted.
|
||||||
|
fn admits_placement(
|
||||||
|
conn: &Connection,
|
||||||
|
limits: &StoreLimits,
|
||||||
|
key: &PodcastDhtKey,
|
||||||
|
tombstone: bool,
|
||||||
|
now: i64,
|
||||||
|
) -> rusqlite::Result<bool> {
|
||||||
|
let (total, tombstones): (i64, i64) = conn.query_row(
|
||||||
|
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
|
||||||
|
WHERE dht_key = ?1 AND expires_at_ms > ?2",
|
||||||
|
params![key.as_bytes().as_slice(), now],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
|
)?;
|
||||||
|
Ok((total.max(0) as usize) < limits.per_key
|
||||||
|
&& (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> {
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||||
|
params![podcast_id, publisher],
|
||||||
|
)?;
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2",
|
||||||
|
params![podcast_id, publisher],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decode_valid(
|
fn decode_valid(
|
||||||
|
|||||||
@@ -3,7 +3,8 @@
|
|||||||
|
|
||||||
use federation_net::{NetworkId, SecretKey};
|
use federation_net::{NetworkId, SecretKey};
|
||||||
|
|
||||||
use super::card::supersedes_everywhere;
|
use crate::routing::NodeId;
|
||||||
|
|
||||||
use super::search::{aggregate_hits, rank_hits};
|
use super::search::{aggregate_hits, rank_hits};
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::podcast::{FeedUrl, PodcastMeta};
|
use crate::podcast::{FeedUrl, PodcastMeta};
|
||||||
@@ -362,10 +363,6 @@ fn record_order_is_total_and_tombstones_win_ties() {
|
|||||||
compare_records(&edited, &older),
|
compare_records(&edited, &older),
|
||||||
compare_records(&older, &edited).reverse()
|
compare_records(&older, &edited).reverse()
|
||||||
);
|
);
|
||||||
assert!(supersedes_everywhere(&grave, &older));
|
|
||||||
assert!(supersedes_everywhere(&newer, &older));
|
|
||||||
assert!(!supersedes_everywhere(&refresh, &older));
|
|
||||||
assert!(!supersedes_everywhere(&older, &grave));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------------------------------------------------- cache
|
// --------------------------------------------------------------------- cache
|
||||||
@@ -437,7 +434,7 @@ async fn cache_converges_regardless_of_delivery_order() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key() {
|
async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key_it_holds() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let db = open_db(&temp).await;
|
let db = open_db(&temp).await;
|
||||||
let key = secret(1);
|
let key = secret(1);
|
||||||
@@ -451,7 +448,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key()
|
|||||||
vec![active.clone()]
|
vec![active.clone()]
|
||||||
);
|
);
|
||||||
|
|
||||||
// A tombstone stored only under the id key still removes the card under
|
// A tombstone stored only under the id key still replaces the card under
|
||||||
// the text keys of this node.
|
// the text keys of this node.
|
||||||
let grave = tombstone(&key, 2, NOW);
|
let grave = tombstone(&key, 2, NOW);
|
||||||
let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id());
|
let id_key = PodcastDhtKey::podcast(&network(), grave.podcast_id());
|
||||||
@@ -462,7 +459,7 @@ async fn cache_tombstone_supersedes_lower_and_equal_revisions_under_every_key()
|
|||||||
db.records_by_key(token_key, vec!["rust".into()], NOW, 10)
|
db.records_by_key(token_key, vec!["rust".into()], NOW, 10)
|
||||||
.await
|
.await
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
Vec::<SignedPodcastRecord>::new()
|
vec![grave.clone()]
|
||||||
);
|
);
|
||||||
// An equal-revision card arriving later loses, under any key.
|
// An equal-revision card arriving later loses, under any key.
|
||||||
let same = card(&key, 2);
|
let same = card(&key, 2);
|
||||||
@@ -564,43 +561,247 @@ async fn cache_read_path_filters_tokens_expiry_and_corruption() {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
.is_empty()
|
.is_empty()
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(db.delete_expired(NOW + ttl).await.unwrap(), 1);
|
||||||
db.delete_expired(NOW + ttl).await.unwrap(),
|
}
|
||||||
MAX_RECORD_DHT_KEYS.min(record.dht_keys().len())
|
|
||||||
);
|
fn numbered_secret(index: usize) -> SecretKey {
|
||||||
|
let mut bytes = [0u8; 32];
|
||||||
|
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
|
||||||
|
bytes[31] = 0x5a;
|
||||||
|
SecretKey::from_bytes(&bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn open_db_with(
|
||||||
|
temp: &tempfile::TempDir,
|
||||||
|
limits: store::StoreLimits,
|
||||||
|
) -> store::PodcastDhtDatabase {
|
||||||
|
store::PodcastDhtDatabase::open_with_limits(
|
||||||
|
&temp.path().join("podcast-dht.sqlite3"),
|
||||||
|
network(),
|
||||||
|
limits,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cache_bounds_records_per_key() {
|
async fn cache_full_key_keeps_its_records_and_refuses_newcomers() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let db = open_db(&temp).await;
|
let db = open_db(&temp).await;
|
||||||
let token_key = PodcastDhtKey::token(&network(), "rust");
|
let token_key = PodcastDhtKey::token(&network(), "rust");
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
for index in 0..=MAX_PODCAST_RECORDS_PER_KEY {
|
for index in 0..=MAX_PODCAST_RECORDS_PER_KEY {
|
||||||
let key = SecretKey::from_bytes(&{
|
// Later publishers expire later; that buys them nothing.
|
||||||
let mut bytes = [0u8; 32];
|
let record = card_with(
|
||||||
bytes[..8].copy_from_slice(&(index as u64 + 1).to_le_bytes());
|
spec("show.xml", "Rust Radio"),
|
||||||
bytes
|
&numbered_secret(index),
|
||||||
});
|
1,
|
||||||
// Later publishers expire later, so the first one is evicted.
|
NOW + index as u64,
|
||||||
let record = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW + index as u64);
|
);
|
||||||
entries.push((vec![token_key], record));
|
entries.push((vec![token_key], record));
|
||||||
}
|
}
|
||||||
let first = entries[0].1.clone();
|
let first = entries[0].1.clone();
|
||||||
for chunk in entries.chunks(64) {
|
let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone();
|
||||||
db.store(chunk.to_vec(), NOW).await.unwrap();
|
// The first publisher arrived first; the rest a moment later.
|
||||||
|
db.store(entries[..1].to_vec(), NOW).await.unwrap();
|
||||||
|
for chunk in entries[1..].chunks(64) {
|
||||||
|
db.store(chunk.to_vec(), NOW + 1).await.unwrap();
|
||||||
}
|
}
|
||||||
let cached = db
|
let cached = db
|
||||||
.records_by_key(token_key, Vec::new(), NOW, usize::MAX)
|
.records_by_key(token_key, Vec::new(), NOW, usize::MAX)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
|
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
|
||||||
assert!(!cached.contains(&first));
|
assert!(cached.contains(&first));
|
||||||
// A record expiring before every cached one is not admitted.
|
assert!(!cached.contains(&last));
|
||||||
assert_eq!(
|
assert_eq!(cached[0], first, "records indexed longest come first");
|
||||||
db.store(vec![(vec![token_key], first)], NOW).await.unwrap(),
|
// The first publisher keeps refreshing its place.
|
||||||
0
|
let refresh = card_with(
|
||||||
|
spec("show.xml", "Rust Radio"),
|
||||||
|
&numbered_secret(0),
|
||||||
|
1,
|
||||||
|
NOW + 1_000,
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
db.store(vec![(vec![token_key], refresh)], NOW)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn sybil_tombstones_cannot_suppress_a_card() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let db = open_db(&temp).await;
|
||||||
|
let victim = card(&secret(1), 1);
|
||||||
|
let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id());
|
||||||
|
let rust = PodcastDhtKey::token(&network(), "rust");
|
||||||
|
db.store(vec![entry(&victim)], NOW).await.unwrap();
|
||||||
|
|
||||||
|
// 256 throwaway identities sign tombstones of the victim's podcast that
|
||||||
|
// list the victim card's text keys, and issue them as late as allowed.
|
||||||
|
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
|
||||||
|
let sybils: Vec<_> = (0..256)
|
||||||
|
.map(|index| {
|
||||||
|
let key = numbered_secret(index);
|
||||||
|
let tombstone = PodcastTombstone::new(
|
||||||
|
network(),
|
||||||
|
key.public(),
|
||||||
|
victim.podcast_id().clone(),
|
||||||
|
u64::MAX,
|
||||||
|
NOW + skew,
|
||||||
|
victim.dht_keys(),
|
||||||
|
);
|
||||||
|
let record =
|
||||||
|
SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap();
|
||||||
|
(record.dht_keys(), record)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut written = 0;
|
||||||
|
for chunk in sybils.chunks(MAX_PODCAST_STORE_RECORDS) {
|
||||||
|
written += db.store(chunk.to_vec(), NOW).await.unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
written, MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||||
|
"only the id key's tombstone budget is used"
|
||||||
|
);
|
||||||
|
|
||||||
|
let by_id = db
|
||||||
|
.records_by_key(id_key, Vec::new(), NOW, MAX_PODCAST_DHT_RECORDS)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(by_id.contains(&victim));
|
||||||
|
assert_eq!(
|
||||||
|
by_id.iter().filter(|record| record.is_tombstone()).count(),
|
||||||
|
MAX_PODCAST_RESPONSE_TOMBSTONES
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
db.records_by_key(rust, vec!["rust".into()], NOW, MAX_PODCAST_DHT_RECORDS)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
vec![victim.clone()],
|
||||||
|
"no tombstone is indexed under a text key without its owner's card"
|
||||||
|
);
|
||||||
|
// Eleven hours later the victim's republish still lands.
|
||||||
|
let later = NOW + 11 * 3_600_000;
|
||||||
|
let republished = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later);
|
||||||
|
assert_eq!(
|
||||||
|
db.store(vec![entry(&republished)], later).await.unwrap(),
|
||||||
|
republished.dht_keys().len()
|
||||||
|
);
|
||||||
|
// Sybil cards issued in the future cannot displace it either.
|
||||||
|
let sybil_cards: Vec<_> = (300..600)
|
||||||
|
.map(|index| {
|
||||||
|
entry(&card_with(
|
||||||
|
spec("show.xml", "Rust Radio"),
|
||||||
|
&numbered_secret(index),
|
||||||
|
u64::MAX,
|
||||||
|
later + skew,
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
for chunk in sybil_cards.chunks(MAX_PODCAST_STORE_RECORDS) {
|
||||||
|
db.store(chunk.to_vec(), later).await.unwrap();
|
||||||
|
}
|
||||||
|
for key in [id_key, rust] {
|
||||||
|
let records = db
|
||||||
|
.records_by_key(key, vec![], later, MAX_PODCAST_DHT_RECORDS)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(records.contains(&republished));
|
||||||
|
assert!(records.len() <= MAX_PODCAST_DHT_RECORDS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cache_enforces_publisher_record_and_byte_quotas() {
|
||||||
|
let limits = store::StoreLimits {
|
||||||
|
per_key: 100,
|
||||||
|
tombstones_per_key: 10,
|
||||||
|
per_publisher: 3,
|
||||||
|
records: 5,
|
||||||
|
bytes: usize::MAX,
|
||||||
|
};
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let db = open_db_with(&temp, limits).await;
|
||||||
|
let show = |index: usize, key: &SecretKey| {
|
||||||
|
entry(&card_with(
|
||||||
|
spec(&format!("show-{index}.xml"), "Rust Radio"),
|
||||||
|
key,
|
||||||
|
1,
|
||||||
|
NOW,
|
||||||
|
))
|
||||||
|
};
|
||||||
|
let heavy = secret(1);
|
||||||
|
let stored: Vec<usize> = {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for index in 0..4 {
|
||||||
|
out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap());
|
||||||
|
}
|
||||||
|
out
|
||||||
|
};
|
||||||
|
assert!(stored[..3].iter().all(|written| *written > 0));
|
||||||
|
assert_eq!(
|
||||||
|
stored[3], 0,
|
||||||
|
"the fourth podcast of one publisher is refused"
|
||||||
|
);
|
||||||
|
// A refresh of an admitted record is always accepted.
|
||||||
|
let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1);
|
||||||
|
assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0);
|
||||||
|
assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0);
|
||||||
|
assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0);
|
||||||
|
assert_eq!(
|
||||||
|
db.store(vec![show(12, &secret(4))], NOW).await.unwrap(),
|
||||||
|
0,
|
||||||
|
"the cache holds at most five records"
|
||||||
|
);
|
||||||
|
|
||||||
|
let one = card(&secret(1), 1);
|
||||||
|
let limits = store::StoreLimits {
|
||||||
|
bytes: one.encoded_len() * 2,
|
||||||
|
..store::StoreLimits::default()
|
||||||
|
};
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let db = open_db_with(&temp, limits).await;
|
||||||
|
assert!(
|
||||||
|
db.store(vec![entry(&card(&secret(1), 1))], NOW)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
> 0
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
db.store(vec![entry(&card(&secret(2), 1))], NOW)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
> 0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
db.store(vec![entry(&card(&secret(3), 1))], NOW)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
0,
|
||||||
|
"the byte budget is full"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cache_stores_each_payload_once() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let db = open_db(&temp).await;
|
||||||
|
let many: Vec<String> = (0..50).map(|index| format!("w{index}")).collect();
|
||||||
|
let record = card_with(spec("show.xml", &many.join(" ")), &secret(1), 1, NOW);
|
||||||
|
assert_eq!(
|
||||||
|
db.store(vec![entry(&record)], NOW).await.unwrap(),
|
||||||
|
MAX_RECORD_DHT_KEYS
|
||||||
|
);
|
||||||
|
let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap();
|
||||||
|
let payloads: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(payloads, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -634,6 +835,25 @@ async fn cache_refuses_a_newer_schema_and_keeps_local_publications() {
|
|||||||
assert_eq!(local[0].record, record);
|
assert_eq!(local[0].record, record);
|
||||||
drop(db);
|
drop(db);
|
||||||
|
|
||||||
|
// A pre-release version-1 replica table is dropped; own publications
|
||||||
|
// survive.
|
||||||
|
{
|
||||||
|
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||||
|
conn.execute_batch(
|
||||||
|
"DROP TABLE podcast_dht_records;
|
||||||
|
CREATE TABLE podcast_dht_records (dht_key BLOB, payload BLOB);
|
||||||
|
INSERT INTO podcast_dht_records VALUES (x'00', x'00');
|
||||||
|
PRAGMA user_version = 1;",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let db = store::PodcastDhtDatabase::open(&path, network())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(db.local_publications().await.unwrap().len(), 1);
|
||||||
|
assert!(db.store(vec![entry(&record)], NOW).await.unwrap() > 0);
|
||||||
|
drop(db);
|
||||||
|
|
||||||
rusqlite::Connection::open(&path)
|
rusqlite::Connection::open(&path)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1)
|
.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION + 1)
|
||||||
@@ -655,12 +875,14 @@ fn store_batches_are_bounded_and_keys_must_belong() {
|
|||||||
keys: record.dht_keys(),
|
keys: record.dht_keys(),
|
||||||
record: record.clone(),
|
record: record.clone(),
|
||||||
};
|
};
|
||||||
validate_store_batch(vec![good.clone()], &network(), NOW).unwrap();
|
let publisher = key.public();
|
||||||
assert!(validate_store_batch(Vec::new(), &network(), NOW).is_err());
|
validate_store_batch(vec![good.clone()], &network(), publisher, NOW).unwrap();
|
||||||
|
assert!(validate_store_batch(Vec::new(), &network(), publisher, NOW).is_err());
|
||||||
assert!(
|
assert!(
|
||||||
validate_store_batch(
|
validate_store_batch(
|
||||||
vec![good.clone(); MAX_PODCAST_STORE_RECORDS],
|
vec![good.clone(); MAX_PODCAST_STORE_RECORDS],
|
||||||
&network(),
|
&network(),
|
||||||
|
publisher,
|
||||||
NOW
|
NOW
|
||||||
)
|
)
|
||||||
.is_ok()
|
.is_ok()
|
||||||
@@ -669,19 +891,87 @@ fn store_batches_are_bounded_and_keys_must_belong() {
|
|||||||
validate_store_batch(
|
validate_store_batch(
|
||||||
vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1],
|
vec![good.clone(); MAX_PODCAST_STORE_RECORDS + 1],
|
||||||
&network(),
|
&network(),
|
||||||
|
publisher,
|
||||||
NOW
|
NOW
|
||||||
)
|
)
|
||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
let mut foreign = good.clone();
|
let mut foreign = good.clone();
|
||||||
foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")];
|
foreign.keys = vec![PodcastDhtKey::token(&network(), "jazz")];
|
||||||
assert!(validate_store_batch(vec![foreign], &network(), NOW).is_err());
|
assert!(validate_store_batch(vec![foreign], &network(), publisher, NOW).is_err());
|
||||||
let mut duplicated = good.clone();
|
let mut duplicated = good.clone();
|
||||||
duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]];
|
duplicated.keys = vec![duplicated.keys[0], duplicated.keys[0]];
|
||||||
assert!(validate_store_batch(vec![duplicated], &network(), NOW).is_err());
|
assert!(validate_store_batch(vec![duplicated], &network(), publisher, NOW).is_err());
|
||||||
let mut forged = good;
|
let mut forged = good.clone();
|
||||||
resign(&mut forged.record, &secret(2));
|
resign(&mut forged.record, &secret(2));
|
||||||
assert!(validate_store_batch(vec![forged], &network(), NOW).is_err());
|
assert!(validate_store_batch(vec![forged], &network(), publisher, NOW).is_err());
|
||||||
|
// A valid record of another publisher may not be stored (or replayed)
|
||||||
|
// by anyone else.
|
||||||
|
let error = validate_store_batch(vec![good], &network(), secret(2).public(), NOW).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("publisher"), "{error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replicas_only_accept_keys_they_are_close_to() {
|
||||||
|
let own = NodeId::from_bytes([0; 32]);
|
||||||
|
let key = PodcastDhtKey::from_bytes([0; 32]);
|
||||||
|
let far: Vec<NodeId> = (0..100)
|
||||||
|
.map(|index| NodeId::from_bytes([0x80 | index as u8; 32]))
|
||||||
|
.collect();
|
||||||
|
assert!(is_responsible(&own, &far, &key));
|
||||||
|
let near: Vec<NodeId> = (1..=PODCAST_STORE_RESPONSIBILITY_WINDOW)
|
||||||
|
.map(|index| {
|
||||||
|
let mut bytes = [0; 32];
|
||||||
|
bytes[31] = index as u8;
|
||||||
|
NodeId::from_bytes(bytes)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let own = NodeId::from_bytes([0xff; 32]);
|
||||||
|
assert!(!is_responsible(&own, &near, &key));
|
||||||
|
assert!(is_responsible(&own, &near[1..], &key));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn inbound_requests_are_limited_per_peer() {
|
||||||
|
let inflight = Arc::new(Mutex::new(HashMap::new()));
|
||||||
|
let peer = secret(1).public();
|
||||||
|
let slots: Vec<_> = (0..MAX_PODCAST_DHT_REQUESTS_PER_PEER)
|
||||||
|
.map(|_| PeerSlot::acquire(&inflight, peer).unwrap())
|
||||||
|
.collect();
|
||||||
|
assert!(PeerSlot::acquire(&inflight, peer).is_none());
|
||||||
|
assert!(PeerSlot::acquire(&inflight, secret(2).public()).is_some());
|
||||||
|
drop(slots);
|
||||||
|
assert!(PeerSlot::acquire(&inflight, peer).is_some());
|
||||||
|
assert!(lock(&inflight).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lookups_keep_a_bounded_number_of_owners() {
|
||||||
|
let mut winners = HashMap::new();
|
||||||
|
let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id());
|
||||||
|
let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS)
|
||||||
|
.map(|index| {
|
||||||
|
card_with(
|
||||||
|
spec("show.xml", "Rust Radio"),
|
||||||
|
&numbered_secret(index),
|
||||||
|
1,
|
||||||
|
now_ms(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let first = records[0].clone();
|
||||||
|
merge_records(&mut winners, records, &network(), &id_key, &[]);
|
||||||
|
assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS);
|
||||||
|
// A newer record of a known owner still replaces its winner.
|
||||||
|
let newer = card_with(
|
||||||
|
spec("show.xml", "Rust Radio"),
|
||||||
|
&numbered_secret(0),
|
||||||
|
2,
|
||||||
|
now_ms(),
|
||||||
|
);
|
||||||
|
merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]);
|
||||||
|
let owner = (first.podcast_id().clone(), first.publisher());
|
||||||
|
assert_eq!(winners[&owner], newer);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -763,18 +1053,27 @@ fn music(name: &str, kind: ItemKind) -> LibraryItem {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn moved_spec(path: &str, origin: &str, title: &str) -> PodcastCardSpec {
|
||||||
|
let mut moved = spec(path, title);
|
||||||
|
moved.id_feed_url = Some(feed(origin));
|
||||||
|
moved.podcast_id = PodcastId::from_feed_url(&feed(origin));
|
||||||
|
moved
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hits_merge_publishers_and_drop_tombstoned_ones() {
|
fn hits_merge_publishers_and_drop_tombstoned_ones() {
|
||||||
let a = secret(1);
|
let a = secret(1);
|
||||||
let b = secret(2);
|
let b = secret(2);
|
||||||
let c = secret(3);
|
let c = secret(3);
|
||||||
let mut moved = spec("moved.xml", "Rust Radio");
|
|
||||||
moved.id_feed_url = Some(feed("show.xml"));
|
|
||||||
moved.podcast_id = PodcastId::from_feed_url(&feed("show.xml"));
|
|
||||||
let records = vec![
|
let records = vec![
|
||||||
card(&a, 1),
|
card(&a, 1),
|
||||||
card(&a, 2),
|
card(&a, 2),
|
||||||
card_with(moved, &b, 1, NOW + 10),
|
card_with(
|
||||||
|
moved_spec("moved.xml", "show.xml", "Rust Radio"),
|
||||||
|
&b,
|
||||||
|
1,
|
||||||
|
NOW + 10,
|
||||||
|
),
|
||||||
card(&c, 1),
|
card(&c, 1),
|
||||||
tombstone(&c, 2, NOW),
|
tombstone(&c, 2, NOW),
|
||||||
card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW),
|
card_with(spec("other.xml", "Rust Hour"), &a, 1, NOW),
|
||||||
@@ -790,8 +1089,81 @@ fn hits_merge_publishers_and_drop_tombstoned_ones() {
|
|||||||
let mut expected = vec![a.public(), b.public()];
|
let mut expected = vec![a.public(), b.public()];
|
||||||
expected.sort_by_key(|key| *key.as_bytes());
|
expected.sort_by_key(|key| *key.as_bytes());
|
||||||
assert_eq!(hit.publishers, expected);
|
assert_eq!(hit.publishers, expected);
|
||||||
// Equal votes per feed URL: the most recently issued card represents.
|
// The card under the origin URL represents, whoever issued later.
|
||||||
assert_eq!(hit.card.publisher, b.public());
|
assert_eq!(hit.card.publisher, a.public());
|
||||||
|
assert_eq!(hit.feed_url, feed("show.xml"));
|
||||||
|
assert_eq!(
|
||||||
|
hit.claimed_feed_urls,
|
||||||
|
vec![ClaimedFeedUrl {
|
||||||
|
feed_url: feed("moved.xml"),
|
||||||
|
publishers: vec![b.public()],
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sybil_cards_cannot_redirect_an_existing_podcast() {
|
||||||
|
let genuine = card(&secret(1), 1);
|
||||||
|
let id = genuine.podcast_id().clone();
|
||||||
|
let evil_feed = FeedUrl::parse("https://evil.example/feed.xml").unwrap();
|
||||||
|
let hijack = |seed: usize| {
|
||||||
|
let mut spec = spec("show.xml", "Rust Radio (official)");
|
||||||
|
spec.id_feed_url = Some(spec.feed_url.clone());
|
||||||
|
spec.feed_url = evil_feed.clone();
|
||||||
|
spec.validate().unwrap();
|
||||||
|
assert_eq!(spec.podcast_id, id, "a hijack card claims the genuine id");
|
||||||
|
let skew = PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64;
|
||||||
|
card_with(spec, &numbered_secret(seed), u64::MAX, NOW + skew)
|
||||||
|
};
|
||||||
|
let mut records = vec![genuine.clone()];
|
||||||
|
records.extend((0..10).map(hijack));
|
||||||
|
let hit = aggregate_hits(records).remove(0);
|
||||||
|
assert_eq!(
|
||||||
|
hit.feed_url,
|
||||||
|
feed("show.xml"),
|
||||||
|
"only the origin URL is offered"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
hit.card,
|
||||||
|
*genuine.card().unwrap(),
|
||||||
|
"the origin card represents"
|
||||||
|
);
|
||||||
|
assert_eq!(hit.claimed_feed_urls.len(), 1);
|
||||||
|
assert_eq!(hit.claimed_feed_urls[0].feed_url, evil_feed);
|
||||||
|
assert_eq!(hit.claimed_feed_urls[0].publishers.len(), 10);
|
||||||
|
|
||||||
|
// Without any origin card the hit still offers the origin URL, never the
|
||||||
|
// claimed one, and ranks below podcasts that have an origin card.
|
||||||
|
let only_claims = aggregate_hits((0..3).map(hijack)).remove(0);
|
||||||
|
assert_eq!(only_claims.feed_url, feed("show.xml"));
|
||||||
|
assert!(!only_claims.card.feed_url_is_origin());
|
||||||
|
let honest = aggregate_hits(vec![card_with(
|
||||||
|
spec("b.xml", "Rust Radio Weekly"),
|
||||||
|
&secret(2),
|
||||||
|
1,
|
||||||
|
NOW,
|
||||||
|
)])
|
||||||
|
.remove(0);
|
||||||
|
let ranked = rank_hits(
|
||||||
|
vec![only_claims.clone(), honest.clone()],
|
||||||
|
"rust",
|
||||||
|
&podcast_query_tokens("rust"),
|
||||||
|
);
|
||||||
|
assert_eq!(ranked[0].podcast_id, honest.podcast_id);
|
||||||
|
// Publisher counts do not rank.
|
||||||
|
let crowded = aggregate_hits(
|
||||||
|
(20..60).map(|seed| card_with(spec("c.xml", "Rust Zone"), &numbered_secret(seed), 1, NOW)),
|
||||||
|
)
|
||||||
|
.remove(0);
|
||||||
|
let ranked = rank_hits(
|
||||||
|
vec![crowded, honest.clone()],
|
||||||
|
"rust",
|
||||||
|
&podcast_query_tokens("rust"),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ranked[0].podcast_id, honest.podcast_id,
|
||||||
|
"ranked by title, not crowd"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1,15 +1,23 @@
|
|||||||
//! Integration test: public podcast discovery on three real loopback iroh
|
//! Integration test: public podcast discovery on three real loopback iroh
|
||||||
//! nodes.
|
//! nodes.
|
||||||
//!
|
//!
|
||||||
//! * A publishes a podcast its user shared; B is one of A's trusted devices
|
//! * A publishes a podcast its user shared. B is another device of the same
|
||||||
//! (in this test merely another overlay node, then offline); C is an
|
//! user: it applies the same merged registers (as device sync would deliver
|
||||||
//! unrelated peer outside A's trusted-device group, with no device-sync
|
//! them) and publishes the same podcast under its own key. C is an
|
||||||
//! pairing.
|
//! unrelated peer outside the user's trusted-device group, with no
|
||||||
//! * C finds the card by text and by id; a podcast A never shared is never
|
//! device-sync pairing.
|
||||||
//! discoverable; after A unpublishes, the tombstone hides the card from C.
|
//! * C finds the podcast by text and by id, merged over publishers {A, B};
|
||||||
//! * Forged, tampered, replayed, malformed, oversized and wrong-version
|
//! once B unshares and goes offline, C still finds A's card, so B is
|
||||||
//! requests are rejected by a real peer.
|
//! irrelevant to discovery. A podcast the user never shared (held by both
|
||||||
//! * D, a music-DHT peer without the overlay, is never added to the overlay.
|
//! A and B) is never discoverable; after A unpublishes, the tombstone hides
|
||||||
|
//! the card from C.
|
||||||
|
//! * Forged, foreign, tampered, replayed, malformed, oversized and
|
||||||
|
//! wrong-version requests are rejected by a real peer.
|
||||||
|
//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but
|
||||||
|
//! never answers on the overlay ALPN, so it never becomes an overlay contact
|
||||||
|
//! and is never asked for records or to store them.
|
||||||
|
//! * `sync_shares` stays fail-safe for revocation and holds back URLs that
|
||||||
|
//! look private (single node).
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -214,14 +222,15 @@ async fn scenario() {
|
|||||||
(a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(())
|
(a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(())
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
// D never answers on the overlay ALPN, so it never becomes an overlay
|
// D is dialed by bootstrap but never answers on the overlay ALPN, so it
|
||||||
// contact and is never queried or asked to store records.
|
// never becomes an overlay contact: it is never asked for records or to
|
||||||
|
// store them.
|
||||||
a.dht.bootstrap().await.unwrap();
|
a.dht.bootstrap().await.unwrap();
|
||||||
assert_eq!(a.dht.known_peers(), 2);
|
assert_eq!(a.dht.known_peers(), 2);
|
||||||
|
|
||||||
// A capability manifest without podcast_dht evicts a peer; one with it
|
// A capability manifest without podcast_dht evicts a peer; one with it
|
||||||
// lets the peer be learned again.
|
// lets the peer be learned again.
|
||||||
let manifest = CapabilityManifest::frid("furumi", "test");
|
let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht();
|
||||||
a.dht
|
a.dht
|
||||||
.note_peer_capabilities(
|
.note_peer_capabilities(
|
||||||
c.service.endpoint_id(),
|
c.service.endpoint_id(),
|
||||||
@@ -260,11 +269,13 @@ async fn scenario() {
|
|||||||
subscription: shared.clone(),
|
subscription: shared.clone(),
|
||||||
share: Some(share(&shared, true, 2)),
|
share: Some(share(&shared, true, 2)),
|
||||||
card: None,
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
},
|
},
|
||||||
PodcastShareState {
|
PodcastShareState {
|
||||||
subscription: private.clone(),
|
subscription: private.clone(),
|
||||||
share: None,
|
share: None,
|
||||||
card: None,
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
let stats = a.dht.sync_shares(states.clone()).await.unwrap();
|
let stats = a.dht.sync_shares(states.clone()).await.unwrap();
|
||||||
@@ -294,7 +305,54 @@ async fn scenario() {
|
|||||||
assert_eq!(published.len(), 1);
|
assert_eq!(published.len(), 1);
|
||||||
let first_card = published[0].record.clone();
|
let first_card = published[0].record.clone();
|
||||||
|
|
||||||
// B (A's trusted device) is not needed for discovery.
|
// B, the same user's other device, holds the same merged registers and
|
||||||
|
// publishes the same podcast under its own key; the private podcast
|
||||||
|
// stays private there too.
|
||||||
|
let b_states = vec![
|
||||||
|
PodcastShareState {
|
||||||
|
subscription: shared.clone(),
|
||||||
|
share: Some(share(&shared, true, 2)),
|
||||||
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
|
},
|
||||||
|
PodcastShareState {
|
||||||
|
subscription: private.clone(),
|
||||||
|
share: None,
|
||||||
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let stats = b.dht.sync_shares(b_states).await.unwrap();
|
||||||
|
assert_eq!((stats.published, stats.unpublished), (1, 0));
|
||||||
|
let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()];
|
||||||
|
both.sort_by_key(|key| *key.as_bytes());
|
||||||
|
let hits = wait_for("C sees both publishers", async || {
|
||||||
|
let hits = search(&c, "rust radio").await;
|
||||||
|
(hits.len() == 1 && hits[0].publishers == both).then_some(hits)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||||
|
assert!(hits[0].claimed_feed_urls.is_empty());
|
||||||
|
|
||||||
|
// B unshares (for example a stale register on B) and goes offline: B is
|
||||||
|
// not needed for discovery, A's card is still found.
|
||||||
|
let stats = b
|
||||||
|
.dht
|
||||||
|
.sync_shares(vec![PodcastShareState {
|
||||||
|
subscription: shared.clone(),
|
||||||
|
share: Some(share(&shared, false, 3)),
|
||||||
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
|
}])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stats.unpublished, 1);
|
||||||
|
let a_only = vec![a.service.endpoint_id()];
|
||||||
|
wait_for("B's tombstone reaches C's lookup", async || {
|
||||||
|
let hits = search(&c, "rust radio").await;
|
||||||
|
(hits.len() == 1 && hits[0].publishers == a_only).then_some(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
b.stop().await;
|
b.stop().await;
|
||||||
|
|
||||||
let hits = wait_for("C finds the card by text", async || {
|
let hits = wait_for("C finds the card by text", async || {
|
||||||
@@ -305,6 +363,7 @@ async fn scenario() {
|
|||||||
assert_eq!(hits.len(), 1);
|
assert_eq!(hits.len(), 1);
|
||||||
assert_eq!(hits[0].podcast_id, shared.podcast_id);
|
assert_eq!(hits[0].podcast_id, shared.podcast_id);
|
||||||
assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]);
|
assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]);
|
||||||
|
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||||
assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url);
|
assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url);
|
||||||
assert_eq!(hits[0].card.title, "Rust Radio Weekly");
|
assert_eq!(hits[0].card.title, "Rust Radio Weekly");
|
||||||
assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens");
|
assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens");
|
||||||
@@ -350,7 +409,11 @@ async fn scenario() {
|
|||||||
let response = store_request(&c, &a, forged).await;
|
let response = store_request(&c, &a, forged).await;
|
||||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||||
|
|
||||||
// Tampered: A's genuine card with a different feed URL.
|
// Foreign: A's genuine card stored by C. Only publishers store records.
|
||||||
|
let response = store_request(&c, &a, first_card.clone()).await;
|
||||||
|
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||||
|
|
||||||
|
// Tampered: A's genuine card with a different description.
|
||||||
let mut tampered = first_card.clone();
|
let mut tampered = first_card.clone();
|
||||||
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
||||||
card.description = Some("Now with malware".into());
|
card.description = Some("Now with malware".into());
|
||||||
@@ -388,6 +451,7 @@ async fn scenario() {
|
|||||||
subscription: shared.clone(),
|
subscription: shared.clone(),
|
||||||
share: Some(share(&shared, false, 3)),
|
share: Some(share(&shared, false, 3)),
|
||||||
card: None,
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
}];
|
}];
|
||||||
let stats = a.dht.sync_shares(unshare).await.unwrap();
|
let stats = a.dht.sync_shares(unshare).await.unwrap();
|
||||||
assert_eq!(stats.unpublished, 1);
|
assert_eq!(stats.unpublished, 1);
|
||||||
@@ -407,10 +471,12 @@ async fn scenario() {
|
|||||||
.is_none()
|
.is_none()
|
||||||
);
|
);
|
||||||
|
|
||||||
// Replaying the old card, even to the node that stores the tombstone,
|
// A replaying its own old card to a replica that stores the tombstone
|
||||||
// does not bring it back.
|
// does not bring it back; nobody else may replay it at all.
|
||||||
let response = store_request(&c, &a, first_card).await;
|
let response = store_request(&a, &c, first_card.clone()).await;
|
||||||
assert!(response.ok && response.stored == 0, "{response:?}");
|
assert!(response.ok && response.stored == 0, "{response:?}");
|
||||||
|
let response = store_request(&c, &a, first_card).await;
|
||||||
|
assert!(!response.ok, "{response:?}");
|
||||||
assert!(search(&c, "rust radio").await.is_empty());
|
assert!(search(&c, "rust radio").await.is_empty());
|
||||||
assert!(search(&a, "rust radio").await.is_empty());
|
assert!(search(&a, "rust radio").await.is_empty());
|
||||||
|
|
||||||
@@ -419,3 +485,84 @@ async fn scenario() {
|
|||||||
d_events.abort();
|
d_events.abort();
|
||||||
d.shutdown().await.unwrap();
|
d.shutdown().await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn sync_shares_is_fail_safe_and_holds_back_private_urls() {
|
||||||
|
tokio::time::timeout(TEST_TIMEOUT, single_node_scenario())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn single_node_scenario() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let network = NetworkId::from_name("podcast-dht-sync-shares");
|
||||||
|
let a = Node::start(&temp.path().join("a"), network).await;
|
||||||
|
let state =
|
||||||
|
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
|
||||||
|
subscription: subscription.clone(),
|
||||||
|
share: Some(share(subscription, shared, generation)),
|
||||||
|
card: None,
|
||||||
|
privacy_confirmed: false,
|
||||||
|
};
|
||||||
|
let first = subscription("first.xml", "First Show");
|
||||||
|
let second = subscription("second.xml", "Second Show");
|
||||||
|
let stats = a
|
||||||
|
.dht
|
||||||
|
.sync_shares(vec![state(&first, true, 2), state(&second, true, 2)])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stats.published, 2);
|
||||||
|
assert!(stats.failed.is_empty() && stats.held_back.is_empty());
|
||||||
|
|
||||||
|
// In one call the user unshares the first podcast while the second
|
||||||
|
// one's card cannot be built (the URL its id derives from is unknown).
|
||||||
|
let mut broken = second.clone();
|
||||||
|
broken.podcast.feed_url = feed("elsewhere.xml");
|
||||||
|
let stats = a
|
||||||
|
.dht
|
||||||
|
.sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stats.failed.len(), 1);
|
||||||
|
assert_eq!(stats.failed[0].podcast_id, second.podcast_id);
|
||||||
|
assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast");
|
||||||
|
let published = a.dht.published().await.unwrap();
|
||||||
|
assert!(published.iter().all(|publication| !publication.active));
|
||||||
|
assert!(
|
||||||
|
published
|
||||||
|
.iter()
|
||||||
|
.all(|publication| publication.record.is_tombstone())
|
||||||
|
);
|
||||||
|
// Republishing re-sends tombstones only; nothing comes back.
|
||||||
|
a.dht.republish().await.unwrap();
|
||||||
|
assert!(search(&a, "first show").await.is_empty());
|
||||||
|
assert!(search(&a, "second show").await.is_empty());
|
||||||
|
|
||||||
|
// A feed URL that looks private is held back until the user confirms.
|
||||||
|
let mut member = subscription("member.xml?token=abc", "Member Show");
|
||||||
|
member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url);
|
||||||
|
let stats = a
|
||||||
|
.dht
|
||||||
|
.sync_shares(vec![state(&member, true, 2)])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stats.published, 0);
|
||||||
|
assert_eq!(stats.held_back.len(), 1);
|
||||||
|
assert_eq!(stats.held_back[0].podcast_id, member.podcast_id);
|
||||||
|
assert!(!stats.held_back[0].warnings.is_empty());
|
||||||
|
assert!(search(&a, "member show").await.is_empty());
|
||||||
|
let mut confirmed = state(&member, true, 2);
|
||||||
|
confirmed.privacy_confirmed = true;
|
||||||
|
let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap();
|
||||||
|
assert_eq!(stats.published, 1);
|
||||||
|
assert_eq!(search(&a, "member show").await.len(), 1);
|
||||||
|
// Once published with the user's consent, the same URLs are kept.
|
||||||
|
let stats = a
|
||||||
|
.dht
|
||||||
|
.sync_shares(vec![state(&member, true, 2)])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0));
|
||||||
|
|
||||||
|
a.stop().await;
|
||||||
|
}
|
||||||
Reference in new issue
Block a user